def objectByUid(self, uid, screenname=None, userNameCache=None):
try:
defer.returnValue(Object(self.uidToObjectId[uid]))
except KeyError:
results = yield Object.query(
self.endpoint, '%s = %s' % (ftwitter.idTag.getPath(), uid))
nResults = len(results)
if nResults:
if nResults > 1:
msg = ('User with Twitter id %d exists %d times '
'in FluidDB! Ignoring.' % (uid, nResults))
log.err(msg)
raise Exception(msg)
else:
o = results[0]
self.add(o.uuid, uid, screenname)
log.msg('Found FluidDB object for Twitter user %d.' % uid)
else:
about = '%s:uid:%d' % (TWITTER_USERNAME, uid)
o = yield Object.create(self.endpoint, about)
if screenname is None:
assert userNameCache is not None
screenname = yield userNameCache.screennameByUid(uid)
log.msg('Made new object for Twitter user %r (uid %d).' %
(screenname, uid))
# TODO: what happens if something goes wrong here?
yield defer.gatherResults([
o.set(self.endpoint, ftwitter.idTag, int(uid)),
o.set(self.endpoint, ftwitter.screennameTag, screenname),
])
log.msg('Set id/created tags on obj for Twitter user %r' %
screenname)
self.add(o.uuid, uid, screenname)
defer.returnValue(o)
def test_AboutTagSearchTerms(self):
"""
Creates various objects with a phrase as about tag. Then query FluidDB
using the "matches" operator on one of the words of the phrase.
"""
aboutTags = [u'Haruhi Suzumiya (涼宮ハルヒ) is the',
u'general name for a series of light novels written',
u'by Nagaru Tanigawa and illustrated by Noizi Ito',
u'and subsequently adapted into other media. The story',
u'follows the title character, Haruhi Suzumiya']
objectsByAbout = {}
for about in aboutTags:
obj = yield Object.create(self.fluiddb, about)
objectsByAbout[about] = obj.uuid
# Force Solr Commit
yield self.solr.commit()
for about in aboutTags:
# chose any word of the phrase
term = random.choice(about.split())
query = 'fluiddb/about matches "%s"' % term
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(objectsByAbout[about], results)
def test_AboutTagSearchTerms(self):
"""
Creates various objects with a phrase as about tag. Then query FluidDB
using the "matches" operator on one of the words of the phrase.
"""
aboutTags = [
u'Haruhi Suzumiya (涼宮ハルヒ) is the',
u'general name for a series of light novels written',
u'by Nagaru Tanigawa and illustrated by Noizi Ito',
u'and subsequently adapted into other media. The story',
u'follows the title character, Haruhi Suzumiya'
]
objectsByAbout = {}
for about in aboutTags:
obj = yield Object.create(self.fluiddb, about)
objectsByAbout[about] = obj.uuid
# Force Solr Commit
yield self.solr.commit()
for about in aboutTags:
# chose any word of the phrase
term = random.choice(about.split())
query = 'fluiddb/about matches "%s"' % term
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(objectsByAbout[about], results)
def findUsersByEmail(endpoint, email):
query = '%s = "%s"' % (sep.join(paths.emailPath()), email)
def _parseResult(objs):
return [User.fromObject(endpoint, obj) for obj in objs]
d = Object.query(endpoint, query)
return d.addCallback(_parseResult)
def findUsersByEmail(endpoint, email):
query = '%s = "%s"' % (sep.join(paths.emailPath()), email)
def _parseResult(objs):
return [User.fromObject(endpoint, obj) for obj in objs]
d = Object.query(endpoint, query)
return d.addCallback(_parseResult)
def test_aboutTagSearch(self):
"""
Creates a simple object with an about tag and then tries to get the
object using a query with the "matches" operator.
"""
aboutTag = 'description'
obj = yield Object.create(self.fluiddb, aboutTag)
# force Solr Commit
yield self.solr.commit()
query = 'fluiddb/about matches "%s"' % aboutTag
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(obj.uuid, results)
def test_aboutTagSearch(self):
"""
Creates a simple object with an about tag and then tries to get the
object using a query with the "matches" operator.
"""
aboutTag = 'description'
obj = yield Object.create(self.fluiddb, aboutTag)
# force Solr Commit
yield self.solr.commit()
query = 'fluiddb/about matches "%s"' % aboutTag
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(obj.uuid, results)
def test_AboutTagComplexSearchQuery(self):
"""
Creates a simple object with an about tag and then tries to get the
object using a composed query (with more than one operator) involving
the "matches" operator.
"""
aboutTag = 'my testing about tag'
obj = yield Object.create(self.fluiddb, aboutTag)
# force Solr Commit
yield self.solr.commit()
query = u'has fluiddb/about and fluiddb/about matches "%s"' % aboutTag
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(obj.uuid, results)
def test_AboutTagComplexSearchQuery(self):
"""
Creates a simple object with an about tag and then tries to get the
object using a composed query (with more than one operator) involving
the "matches" operator.
"""
aboutTag = 'my testing about tag'
obj = yield Object.create(self.fluiddb, aboutTag)
# force Solr Commit
yield self.solr.commit()
query = u'has fluiddb/about and fluiddb/about matches "%s"' % aboutTag
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(obj.uuid, results)
def objectIdByScreenname(self, screenname):
try:
return defer.succeed(self.screennameToObjectId[screenname.lower()])
except KeyError:
def _cb(results):
nResults = len(results)
if nResults == 0:
raise Exception(
'Screenname %r not known to FluidDB' % screenname)
elif nResults == 1:
objectId = results[0].uuid
self.add(objectId, screenname=screenname)
return objectId
else:
log.err('ERROR: Twitter screenname %r found %d times '
'in FluidDB! ObjectIds = %r' %
(screenname, nResults, results))
# Don't crash: just return the first object id found.
return results[0]
d = Object.query(self.endpoint, '%s = "%s"' %
(ftwitter.screennameTag.getPath(), screenname))
d.addCallback(_cb)
return d
def fluidinfoQuery(endpoint, query):
d = Object.query(endpoint, query)
d.addCallback(lambda results: [r.uuid for r in results])
d.addErrback(_queryErr, query)
return d
def cb_intermediateQuery(users, screennames, cache, endpoint, queryTree):
# Make sure all users exist, that there were no other errors in
# fetching them, and that none of them are protected.
notFound = []
otherError = []
for name, user in users.items():
if isinstance(user, failure.Failure):
if user.check(error.Error):
status = int(user.value.status)
if status == http.NOT_FOUND:
notFound.append(name)
else:
log.msg(user)
otherError.append(name)
else:
log.msg(user)
otherError.append(name)
if notFound:
raise NonExistentScreennames(notFound)
if otherError:
raise ScreennameErrors(otherError)
protected = [name for name in users if users[name]['protected']]
if protected:
raise ProtectedScreennames(protected)
# Make sure to use defaults.FRIENDS_LIMIT here, not to import
# that value. That's because we can change the value (in the
# defaults module) using the admin interface.
tooMany = [(name, users[name]['friends_count']) for name in users if
(users[name]['friends_count'] > defaults.FRIENDS_LIMIT and
not cache.adderCache.added(name))]
if tooMany:
raise TooManyFriends(tooMany)
# Enqueue screennames of users that are not yet known.
unknown = [name for name in users if not cache.adderCache.known(name)]
if unknown:
for name in unknown:
cache.adderCache.put(name, users[name]['friends_count'])
# Get the status of all the queried screennames.
statusSummary = cache.adderCache.statusSummary(screennames)
# Raise if not all queried screennames are added.
if len(statusSummary['added']) != len(screennames):
raise UnaddedScreennames(statusSummary)
# We're good to go.
queryStr = query.queryTreeToString(
queryTree, TWITTER_USERNAME, TWITTER_FRIENDS_NAMESPACE_NAME)
try:
result = cache.queryCache.lookupQueryResult(queryStr)
except KeyError:
def _cacheResult(result, queryStr, cache, screennames):
cache.queryCache.storeQueryResult(queryStr, result, screennames)
return result
log.msg('Cache miss on query %r.' % queryStr)
d = Object.query(endpoint, queryStr)
d.addCallback(lambda results: [r.uuid for r in results])
d.addCallback(_cacheResult, queryStr, cache, screennames)
return d
else:
log.msg('Query cache hit (size %d) for %r.' % (len(result), queryStr))
return defer.succeed(result)
