def objectByUid(self, uid, screenname=None, userNameCache=None): try: defer.returnValue(Object(self.uidToObjectId[uid])) except KeyError: results = yield Object.query( self.endpoint, '%s = %s' % (ftwitter.idTag.getPath(), uid)) nResults = len(results) if nResults: if nResults > 1: msg = ('User with Twitter id %d exists %d times ' 'in FluidDB! Ignoring.' % (uid, nResults)) log.err(msg) raise Exception(msg) else: o = results[0] self.add(o.uuid, uid, screenname) log.msg('Found FluidDB object for Twitter user %d.' % uid) else: about = '%s:uid:%d' % (TWITTER_USERNAME, uid) o = yield Object.create(self.endpoint, about) if screenname is None: assert userNameCache is not None screenname = yield userNameCache.screennameByUid(uid) log.msg('Made new object for Twitter user %r (uid %d).' % (screenname, uid)) # TODO: what happens if something goes wrong here? yield defer.gatherResults([ o.set(self.endpoint, ftwitter.idTag, int(uid)), o.set(self.endpoint, ftwitter.screennameTag, screenname), ]) log.msg('Set id/created tags on obj for Twitter user %r' % screenname) self.add(o.uuid, uid, screenname) defer.returnValue(o)
def test_AboutTagSearchTerms(self): """ Creates various objects with a phrase as about tag. Then query FluidDB using the "matches" operator on one of the words of the phrase. """ aboutTags = [u'Haruhi Suzumiya (涼宮ハルヒ) is the', u'general name for a series of light novels written', u'by Nagaru Tanigawa and illustrated by Noizi Ito', u'and subsequently adapted into other media. The story', u'follows the title character, Haruhi Suzumiya'] objectsByAbout = {} for about in aboutTags: obj = yield Object.create(self.fluiddb, about) objectsByAbout[about] = obj.uuid # Force Solr Commit yield self.solr.commit() for about in aboutTags: # chose any word of the phrase term = random.choice(about.split()) query = 'fluiddb/about matches "%s"' % term results = yield Object.query(self.fluiddb, query) results = [o.uuid for o in results] self.assertTrue(len(results) > 0) self.assertIn(objectsByAbout[about], results)
def test_AboutTagSearchTerms(self): """ Creates various objects with a phrase as about tag. Then query FluidDB using the "matches" operator on one of the words of the phrase. """ aboutTags = [ u'Haruhi Suzumiya (涼宮ハルヒ) is the', u'general name for a series of light novels written', u'by Nagaru Tanigawa and illustrated by Noizi Ito', u'and subsequently adapted into other media. The story', u'follows the title character, Haruhi Suzumiya' ] objectsByAbout = {} for about in aboutTags: obj = yield Object.create(self.fluiddb, about) objectsByAbout[about] = obj.uuid # Force Solr Commit yield self.solr.commit() for about in aboutTags: # chose any word of the phrase term = random.choice(about.split()) query = 'fluiddb/about matches "%s"' % term results = yield Object.query(self.fluiddb, query) results = [o.uuid for o in results] self.assertTrue(len(results) > 0) self.assertIn(objectsByAbout[about], results)
def findUsersByEmail(endpoint, email): query = '%s = "%s"' % (sep.join(paths.emailPath()), email) def _parseResult(objs): return [User.fromObject(endpoint, obj) for obj in objs] d = Object.query(endpoint, query) return d.addCallback(_parseResult)
def test_aboutTagSearch(self): """ Creates a simple object with an about tag and then tries to get the object using a query with the "matches" operator. """ aboutTag = 'description' obj = yield Object.create(self.fluiddb, aboutTag) # force Solr Commit yield self.solr.commit() query = 'fluiddb/about matches "%s"' % aboutTag results = yield Object.query(self.fluiddb, query) results = [o.uuid for o in results] self.assertTrue(len(results) > 0) self.assertIn(obj.uuid, results)
def test_AboutTagComplexSearchQuery(self): """ Creates a simple object with an about tag and then tries to get the object using a composed query (with more than one operator) involving the "matches" operator. """ aboutTag = 'my testing about tag' obj = yield Object.create(self.fluiddb, aboutTag) # force Solr Commit yield self.solr.commit() query = u'has fluiddb/about and fluiddb/about matches "%s"' % aboutTag results = yield Object.query(self.fluiddb, query) results = [o.uuid for o in results] self.assertTrue(len(results) > 0) self.assertIn(obj.uuid, results)
def objectIdByScreenname(self, screenname): try: return defer.succeed(self.screennameToObjectId[screenname.lower()]) except KeyError: def _cb(results): nResults = len(results) if nResults == 0: raise Exception( 'Screenname %r not known to FluidDB' % screenname) elif nResults == 1: objectId = results[0].uuid self.add(objectId, screenname=screenname) return objectId else: log.err('ERROR: Twitter screenname %r found %d times ' 'in FluidDB! ObjectIds = %r' % (screenname, nResults, results)) # Don't crash: just return the first object id found. return results[0] d = Object.query(self.endpoint, '%s = "%s"' % (ftwitter.screennameTag.getPath(), screenname)) d.addCallback(_cb) return d
def fluidinfoQuery(endpoint, query): d = Object.query(endpoint, query) d.addCallback(lambda results: [r.uuid for r in results]) d.addErrback(_queryErr, query) return d
def cb_intermediateQuery(users, screennames, cache, endpoint, queryTree): # Make sure all users exist, that there were no other errors in # fetching them, and that none of them are protected. notFound = [] otherError = [] for name, user in users.items(): if isinstance(user, failure.Failure): if user.check(error.Error): status = int(user.value.status) if status == http.NOT_FOUND: notFound.append(name) else: log.msg(user) otherError.append(name) else: log.msg(user) otherError.append(name) if notFound: raise NonExistentScreennames(notFound) if otherError: raise ScreennameErrors(otherError) protected = [name for name in users if users[name]['protected']] if protected: raise ProtectedScreennames(protected) # Make sure to use defaults.FRIENDS_LIMIT here, not to import # that value. That's because we can change the value (in the # defaults module) using the admin interface. tooMany = [(name, users[name]['friends_count']) for name in users if (users[name]['friends_count'] > defaults.FRIENDS_LIMIT and not cache.adderCache.added(name))] if tooMany: raise TooManyFriends(tooMany) # Enqueue screennames of users that are not yet known. unknown = [name for name in users if not cache.adderCache.known(name)] if unknown: for name in unknown: cache.adderCache.put(name, users[name]['friends_count']) # Get the status of all the queried screennames. statusSummary = cache.adderCache.statusSummary(screennames) # Raise if not all queried screennames are added. if len(statusSummary['added']) != len(screennames): raise UnaddedScreennames(statusSummary) # We're good to go. queryStr = query.queryTreeToString( queryTree, TWITTER_USERNAME, TWITTER_FRIENDS_NAMESPACE_NAME) try: result = cache.queryCache.lookupQueryResult(queryStr) except KeyError: def _cacheResult(result, queryStr, cache, screennames): cache.queryCache.storeQueryResult(queryStr, result, screennames) return result log.msg('Cache miss on query %r.' % queryStr) d = Object.query(endpoint, queryStr) d.addCallback(lambda results: [r.uuid for r in results]) d.addCallback(_cacheResult, queryStr, cache, screennames) return d else: log.msg('Query cache hit (size %d) for %r.' % (len(result), queryStr)) return defer.succeed(result)