def objectByUid(self, uid, screenname=None, userNameCache=None):
try:
defer.returnValue(Object(self.uidToObjectId[uid]))
except KeyError:
results = yield Object.query(
self.endpoint, '%s = %s' % (ftwitter.idTag.getPath(), uid))
nResults = len(results)
if nResults:
if nResults > 1:
msg = ('User with Twitter id %d exists %d times '
'in FluidDB! Ignoring.' % (uid, nResults))
log.err(msg)
raise Exception(msg)
else:
o = results[0]
self.add(o.uuid, uid, screenname)
log.msg('Found FluidDB object for Twitter user %d.' % uid)
else:
about = '%s:uid:%d' % (TWITTER_USERNAME, uid)
o = yield Object.create(self.endpoint, about)
if screenname is None:
assert userNameCache is not None
screenname = yield userNameCache.screennameByUid(uid)
log.msg('Made new object for Twitter user %r (uid %d).' %
(screenname, uid))
# TODO: what happens if something goes wrong here?
yield defer.gatherResults([
o.set(self.endpoint, ftwitter.idTag, int(uid)),
o.set(self.endpoint, ftwitter.screennameTag, screenname),
])
log.msg('Set id/created tags on obj for Twitter user %r' %
screenname)
self.add(o.uuid, uid, screenname)
defer.returnValue(o)
def checkTagOnObject(objectId):
o = Object(objectId)
tag = Tag(TWITTER_USERNAME, TWITTER_FRIENDS_NAMESPACE_NAME,
screenname1.lower())
d = o.get(endpoint, tag)
d.addCallback(lambda _: True)
return d
def test_AboutTagSearchTerms(self):
"""
Creates various objects with a phrase as about tag. Then query FluidDB
using the "matches" operator on one of the words of the phrase.
"""
aboutTags = [u'Haruhi Suzumiya (涼宮ハルヒ) is the',
u'general name for a series of light novels written',
u'by Nagaru Tanigawa and illustrated by Noizi Ito',
u'and subsequently adapted into other media. The story',
u'follows the title character, Haruhi Suzumiya']
objectsByAbout = {}
for about in aboutTags:
obj = yield Object.create(self.fluiddb, about)
objectsByAbout[about] = obj.uuid
# Force Solr Commit
yield self.solr.commit()
for about in aboutTags:
# chose any word of the phrase
term = random.choice(about.split())
query = 'fluiddb/about matches "%s"' % term
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(objectsByAbout[about], results)
def test_AboutTagSearchTerms(self):
"""
Creates various objects with a phrase as about tag. Then query FluidDB
using the "matches" operator on one of the words of the phrase.
"""
aboutTags = [
u'Haruhi Suzumiya (涼宮ハルヒ) is the',
u'general name for a series of light novels written',
u'by Nagaru Tanigawa and illustrated by Noizi Ito',
u'and subsequently adapted into other media. The story',
u'follows the title character, Haruhi Suzumiya'
]
objectsByAbout = {}
for about in aboutTags:
obj = yield Object.create(self.fluiddb, about)
objectsByAbout[about] = obj.uuid
# Force Solr Commit
yield self.solr.commit()
for about in aboutTags:
# chose any word of the phrase
term = random.choice(about.split())
query = 'fluiddb/about matches "%s"' % term
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(objectsByAbout[about], results)
def addFriend(friendName, thisIndex, totalToAdd):
log.msg('About to mark user %r as a friend %d/%d of %r.' %
(friendName, thisIndex, totalToAdd, screenname))
d = cache.oidUidScreennameCache.objectIdByScreenname(friendName)
d.addErrback(log.err)
objectId = yield d
log.msg('Marking user %r as a friend %d/%d of %r' %
(friendName, thisIndex, totalToAdd, screenname))
if objectId is not None:
o = Object(objectId)
yield o.set(endpoint, friendTag, None)
log.msg('Marked user %r as a friend %d/%d of %r' %
(friendName, thisIndex, totalToAdd, screenname))
_tagFriendDone()
def test_aboutTagIndexation(self):
"""
Create some objects in FluidDB with an about tag and then search
in the Solr index to check if the the tags are indexed.
"""
aboutTags = [u'short', u'longer about description', u'ワンピース']
# First add the objects to FluidDB
objectsByAbout = {}
for about in aboutTags:
obj = yield Object.create(self.fluiddb, about)
objectsByAbout[about] = obj.uuid
yield self.solr.commit()
# Then search the objects in Solr
for about in aboutTags:
r = yield self.solr.search('path:%s AND value_fts:"%s"' %
(self.aboutTag, about))
self.assertTrue(r.results.numFound > 0)
# checks that at least one of the returning object ids is one
# of the objects we just added
self.assertTrue(
any(obj['fluiddb/id'] in objectsByAbout.values()
for obj in r.results.docs))
defer.returnValue(None)
def test_aboutTagsTermSearch(self):
"""
Adds some objects to fluidDB with about tags. Then search terms (words)
in the about tags.
"""
aboutTags = [u'One Piece (ワンピース Wan Pīsu?) is a Japanese',
u'manga series written and illustrated by Eiichiro Oda',
u'that has been serialized in Weekly Shōnen Jump since',
u'August 4, 1997. The individual chapters are being',
u'published in tankōbon volumes by Shueisha']
# First add the objects to FluidDB
objectsByAbout = {}
for about in aboutTags:
obj = yield Object.create(self.fluiddb, about)
objectsByAbout[about] = obj.uuid
yield self.solr.commit()
# Then search the objects in Solr
for about in aboutTags:
term = random.choice(about.split())
r = yield self.solr.search('path:%s AND value_fts:"%s"' %
(self.aboutTag, term))
self.assertTrue(r.results.numFound > 0)
# checks that at least one of the returning object ids is one
# of the objects we just added
self.assertTrue(any(obj['fluiddb/id'] in objectsByAbout.values()
for obj in r.results.docs))
defer.returnValue(None)
def testTagValueFromAbout(self):
"""
A GET on /about/xxx/namespace/tag where xxx is the fluiddb/about tag of
an object and namespace/tag is the path of the tag we want to get its
value.
"""
creds = _getCreds()
endpoint = Endpoint(self._endpoint, creds=creds)
about = 'testTagValueFromAbout'
value = 'value'
# First, we create an object and attach a tag to it.
object = yield Object.create(endpoint, about)
tag = Tag(u'fluiddb', u'testing', u'test1')
yield object.set(endpoint, tag, value)
try:
URI = '{endpoint}/about/{value}/{tag}'
URI = URI.format(endpoint=self._endpoint, value=about,
tag='fluiddb/testing/test1')
headers = Headers({'authorization': [creds.encode()]})
agent = Agent(reactor)
response = yield agent.request('GET', URI, headers)
self.assertEqual(response.code, http.OK)
d = defer.Deferred()
bodyGetter = ResponseGetter(d)
response.deliverBody(bodyGetter)
body = yield d
self.assertEqual(json.loads(body), value)
finally:
yield object.delete(endpoint, tag)
def test_aboutTagIndexation(self):
"""
Create some objects in FluidDB with an about tag and then search
in the Solr index to check if the the tags are indexed.
"""
aboutTags = [u'short',
u'longer about description',
u'ワンピース']
# First add the objects to FluidDB
objectsByAbout = {}
for about in aboutTags:
obj = yield Object.create(self.fluiddb, about)
objectsByAbout[about] = obj.uuid
yield self.solr.commit()
# Then search the objects in Solr
for about in aboutTags:
r = yield self.solr.search('path:%s AND value_fts:"%s"'
% (self.aboutTag, about))
self.assertTrue(r.results.numFound > 0)
# checks that at least one of the returning object ids is one
# of the objects we just added
self.assertTrue(any(obj['fluiddb/id'] in objectsByAbout.values()
for obj in r.results.docs))
defer.returnValue(None)
def testTagValueFromAbout(self):
"""
A GET on /about/xxx/namespace/tag where xxx is the fluiddb/about tag of
an object and namespace/tag is the path of the tag we want to get its
value.
"""
creds = _getCreds()
endpoint = Endpoint(self._endpoint, creds=creds)
about = 'testTagValueFromAbout'
value = 'value'
# First, we create an object and attach a tag to it.
object = yield Object.create(endpoint, about)
tag = Tag(u'fluiddb', u'testing', u'test1')
yield object.set(endpoint, tag, value)
try:
URI = '{endpoint}/about/{value}/{tag}'
URI = URI.format(endpoint=self._endpoint,
value=about,
tag='fluiddb/testing/test1')
headers = Headers({'authorization': [creds.encode()]})
agent = Agent(reactor)
response = yield agent.request('GET', URI, headers)
self.assertEqual(response.code, http.OK)
d = defer.Deferred()
bodyGetter = ResponseGetter(d)
response.deliverBody(bodyGetter)
body = yield d
self.assertEqual(json.loads(body), value)
finally:
yield object.delete(endpoint, tag)
def findUsersByEmail(endpoint, email):
query = '%s = "%s"' % (sep.join(paths.emailPath()), email)
def _parseResult(objs):
return [User.fromObject(endpoint, obj) for obj in objs]
d = Object.query(endpoint, query)
return d.addCallback(_parseResult)
def findUsersByEmail(endpoint, email):
query = '%s = "%s"' % (sep.join(paths.emailPath()), email)
def _parseResult(objs):
return [User.fromObject(endpoint, obj) for obj in objs]
d = Object.query(endpoint, query)
return d.addCallback(_parseResult)
def test_aboutTagSearch(self):
"""
Creates a simple object with an about tag and then tries to get the
object using a query with the "matches" operator.
"""
aboutTag = 'description'
obj = yield Object.create(self.fluiddb, aboutTag)
# force Solr Commit
yield self.solr.commit()
query = 'fluiddb/about matches "%s"' % aboutTag
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(obj.uuid, results)
def test_aboutTagSearch(self):
"""
Creates a simple object with an about tag and then tries to get the
object using a query with the "matches" operator.
"""
aboutTag = 'description'
obj = yield Object.create(self.fluiddb, aboutTag)
# force Solr Commit
yield self.solr.commit()
query = 'fluiddb/about matches "%s"' % aboutTag
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(obj.uuid, results)
def test_AboutTagComplexSearchQuery(self):
"""
Creates a simple object with an about tag and then tries to get the
object using a composed query (with more than one operator) involving
the "matches" operator.
"""
aboutTag = 'my testing about tag'
obj = yield Object.create(self.fluiddb, aboutTag)
# force Solr Commit
yield self.solr.commit()
query = u'has fluiddb/about and fluiddb/about matches "%s"' % aboutTag
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(obj.uuid, results)
def test_AboutTagComplexSearchQuery(self):
"""
Creates a simple object with an about tag and then tries to get the
object using a composed query (with more than one operator) involving
the "matches" operator.
"""
aboutTag = 'my testing about tag'
obj = yield Object.create(self.fluiddb, aboutTag)
# force Solr Commit
yield self.solr.commit()
query = u'has fluiddb/about and fluiddb/about matches "%s"' % aboutTag
results = yield Object.query(self.fluiddb, query)
results = [o.uuid for o in results]
self.assertTrue(len(results) > 0)
self.assertIn(obj.uuid, results)
def objectIdByScreenname(self, screenname):
try:
return defer.succeed(self.screennameToObjectId[screenname.lower()])
except KeyError:
def _cb(results):
nResults = len(results)
if nResults == 0:
raise Exception(
'Screenname %r not known to FluidDB' % screenname)
elif nResults == 1:
objectId = results[0].uuid
self.add(objectId, screenname=screenname)
return objectId
else:
log.err('ERROR: Twitter screenname %r found %d times '
'in FluidDB! ObjectIds = %r' %
(screenname, nResults, results))
# Don't crash: just return the first object id found.
return results[0]
d = Object.query(self.endpoint, '%s = "%s"' %
(ftwitter.screennameTag.getPath(), screenname))
d.addCallback(_cb)
return d
def test_aboutTagsTermSearch(self):
"""
Adds some objects to fluidDB with about tags. Then search terms (words)
in the about tags.
"""
aboutTags = [
u'One Piece (ワンピース Wan Pīsu?) is a Japanese',
u'manga series written and illustrated by Eiichiro Oda',
u'that has been serialized in Weekly Shōnen Jump since',
u'August 4, 1997. The individual chapters are being',
u'published in tankōbon volumes by Shueisha'
]
# First add the objects to FluidDB
objectsByAbout = {}
for about in aboutTags:
obj = yield Object.create(self.fluiddb, about)
objectsByAbout[about] = obj.uuid
yield self.solr.commit()
# Then search the objects in Solr
for about in aboutTags:
term = random.choice(about.split())
r = yield self.solr.search('path:%s AND value_fts:"%s"' %
(self.aboutTag, term))
self.assertTrue(r.results.numFound > 0)
# checks that at least one of the returning object ids is one
# of the objects we just added
self.assertTrue(
any(obj['fluiddb/id'] in objectsByAbout.values()
for obj in r.results.docs))
defer.returnValue(None)
def cb_intermediateQuery(users, screennames, cache, endpoint, queryTree):
# Make sure all users exist, that there were no other errors in
# fetching them, and that none of them are protected.
notFound = []
otherError = []
for name, user in users.items():
if isinstance(user, failure.Failure):
if user.check(error.Error):
status = int(user.value.status)
if status == http.NOT_FOUND:
notFound.append(name)
else:
log.msg(user)
otherError.append(name)
else:
log.msg(user)
otherError.append(name)
if notFound:
raise NonExistentScreennames(notFound)
if otherError:
raise ScreennameErrors(otherError)
protected = [name for name in users if users[name]['protected']]
if protected:
raise ProtectedScreennames(protected)
# Make sure to use defaults.FRIENDS_LIMIT here, not to import
# that value. That's because we can change the value (in the
# defaults module) using the admin interface.
tooMany = [(name, users[name]['friends_count']) for name in users if
(users[name]['friends_count'] > defaults.FRIENDS_LIMIT and
not cache.adderCache.added(name))]
if tooMany:
raise TooManyFriends(tooMany)
# Enqueue screennames of users that are not yet known.
unknown = [name for name in users if not cache.adderCache.known(name)]
if unknown:
for name in unknown:
cache.adderCache.put(name, users[name]['friends_count'])
# Get the status of all the queried screennames.
statusSummary = cache.adderCache.statusSummary(screennames)
# Raise if not all queried screennames are added.
if len(statusSummary['added']) != len(screennames):
raise UnaddedScreennames(statusSummary)
# We're good to go.
queryStr = query.queryTreeToString(
queryTree, TWITTER_USERNAME, TWITTER_FRIENDS_NAMESPACE_NAME)
try:
result = cache.queryCache.lookupQueryResult(queryStr)
except KeyError:
def _cacheResult(result, queryStr, cache, screennames):
cache.queryCache.storeQueryResult(queryStr, result, screennames)
return result
log.msg('Cache miss on query %r.' % queryStr)
d = Object.query(endpoint, queryStr)
d.addCallback(lambda results: [r.uuid for r in results])
d.addCallback(_cacheResult, queryStr, cache, screennames)
return d
else:
log.msg('Query cache hit (size %d) for %r.' % (len(result), queryStr))
return defer.succeed(result)
def _parseResult(result):
uuid = self.uuid = result[u'id']
self.name = result[u'name']
o = Object(uuid)
d = o.get(endpoint, Tag(*map(unicode, paths.emailPath())))
return d.addCallback(_addEmail)
def fluidinfoQuery(endpoint, query):
d = Object.query(endpoint, query)
d.addCallback(lambda results: [r.uuid for r in results])
d.addErrback(_queryErr, query)
return d
def _parseResult(result):
uuid = self.uuid = result[u'id']
self.name = result[u'name']
o = Object(uuid)
d = o.get(endpoint, Tag(*map(unicode, paths.emailPath())))
return d.addCallback(_addEmail)
def tagIt(objectId, ratio, rank):
o = Object(objectId)
return defer.gatherResults([o.set(endpoint, ratioTag, float(ratio)),
o.set(endpoint, top100Tag, rank)])
