def create(endpoint, username):
ns = Namespace(username)
for name, desc in (
(HACKERNEWS_TOP100_TAG_NAME, 'Hacker News top 100 Tweeter.'),
(HACKERNEWS_RATIO_TAG_NAME, 'Hacker News top Tweeter followers %.'),
):
d = ns.createTag(endpoint, name, desc, False)
addCallbacks(d, 'Created tag %r.' % name)
yield d
def create(endpoint):
ns = Namespace(TWITTER_USERNAME)
d = ns.createChild(endpoint, TWITTER_FRIENDS_NAMESPACE_NAME,
'Holds tags for users to indicate friends.')
addCallbacks(d, 'Created friends namespace %r.' %
TWITTER_FRIENDS_NAMESPACE_NAME)
yield d
d = ns.createChild(endpoint, TWITTER_FOLLOWERS_NAMESPACE_NAME,
'Holds tags for users to indicate followers.')
addCallbacks(d, 'Created followers namespace %r.' %
TWITTER_FOLLOWERS_NAMESPACE_NAME)
yield d
d = ns.createChild(endpoint, TWITTER_USERS_NAMESPACE_NAME,
'Holds tags for info about Twitter users.')
addCallbacks(d, 'Created users namespace %r.' %
TWITTER_USERS_NAMESPACE_NAME)
yield d
# createChild (above) will return a Namespace when txFluidDB gets fixed.
userNs = ns.child(TWITTER_USERS_NAMESPACE_NAME)
for name, desc in (
(TWITTER_ID_TAG_NAME, 'Twitter user id.'),
(TWITTER_SCREENNAME_TAG_NAME, 'Twitter screen name.'),
(TWITTER_UPDATED_AT_TAG_NAME,
'Time (in seconds) of last update of this user in Fluidinfo.'),
(TWITTER_N_FRIENDS_TAG_NAME, 'Number of friends of a Twitter user.'),
(TWITTER_N_FOLLOWERS_TAG_NAME,
'Number of followers of a Twitter user.'),
(TWITTER_N_STATUSES_TAG_NAME,
'Number of status updates of a Twitter user.')):
d = userNs.createTag(endpoint, name, desc, False)
addCallbacks(d, 'Created tag %r.' % name)
yield d
def addUserByScreenname(cache, endpoint, userJob):
# We must at least create the user.
userJob.workToDo = WORK_TO_CREATE_A_FRIEND
userJob.workDone = 0
screenname = userJob.screenname
log.msg('Adding user %r' % screenname)
def catchUnknownScreenname(fail):
fail.trap(error.Error)
if int(fail.value.status) != http.NOT_FOUND:
return fail
return defer.fail(UnknownScreenname(screenname))
def catchProtectedScreenname(fail):
fail.trap(error.Error)
if int(fail.value.status) != http.UNAUTHORIZED:
return fail
return defer.fail(ProtectedScreenname(screenname))
d = cache.friendsIdCache[screenname]
d.addErrback(catchUnknownScreenname)
d.addErrback(catchProtectedScreenname)
friendUids = yield d
log.msg('Got %d friends for user %r' % (len(friendUids), screenname))
# Make a tag for this new user to mark their friends with.
ns = Namespace(TWITTER_USERNAME, TWITTER_FRIENDS_NAMESPACE_NAME)
d = ns.createTag(endpoint, screenname.lower(),
"A tag used to mark %s's Twitter friends." % screenname, False)
# TODO: check the X-FluidDB-Error-Class header in the errback to make
# sure it really got a namespace already exists error.
d.addErrback(_ignoreHTTPStatus, http.PRECONDITION_FAILED)
yield d
# Note: the call to createTag (above) will return a Tag instance when
# txFluidDB gets fixed.
friendTag = Tag(TWITTER_USERNAME, TWITTER_FRIENDS_NAMESPACE_NAME,
screenname.lower())
friendTagPath = friendTag.getPath()
log.msg('Created Twitter friends tag %s' % friendTagPath)
def _madeUserDone(userObject, user):
userJob.workDone += WORK_TO_CREATE_A_FRIEND
cache.extraTwitterTagsPool.add(
addExtraTwitterTags(endpoint, userObject, user))
return userObject
def _madeUserErr(failure):
userJob.workDone += WORK_TO_CREATE_A_FRIEND
return failure
def _tagFriendDone():
userJob.workDone += WORK_TO_TAG_A_FRIEND
def makeUser(user, thisIndex=None, totalToAdd=None):
newName = user['screen_name']
if thisIndex is not None:
log.msg('Making user %r, friend %d/%d of %r.' %
(newName, thisIndex, totalToAdd, screenname))
else:
log.msg('Making user %r.' % newName)
d = cache.oidUidScreennameCache.objectByUid(user['id'], newName)
d.addCallbacks(_madeUserDone, _madeUserErr, callbackArgs=(user,))
return d
def _ignore404uid(fail, uid):
fail.trap(error.Error)
if int(fail.value.status) == http.NOT_FOUND:
log.msg('Twitter uid %d is no longer found (404). Ignoring.' % uid)
cache.userCache.removeUid(uid)
cache.oidUidScreennameCache.removeUid(uid)
else:
log.msg('Failure fetching Twitter uid %d:' % uid)
log.err(fail)
def makeCreateUserJobs(friendsToAdd):
nToAdd = len(friendsToAdd)
for i, friendUid in enumerate(friendsToAdd):
if userJob.canceled():
log.msg('Detected cancelation of screenname %r.' % screenname)
raise StopIteration
d = cache.userCache.userByUid(friendUid)
d.addCallbacks(makeUser, _ignore404uid,
callbackArgs=(i + 1, nToAdd),
errbackArgs=(friendUid,))
yield d
@defer.inlineCallbacks
def addFriend(friendName, thisIndex, totalToAdd):
log.msg('About to mark user %r as a friend %d/%d of %r.' %
(friendName, thisIndex, totalToAdd, screenname))
d = cache.oidUidScreennameCache.objectIdByScreenname(friendName)
d.addErrback(log.err)
objectId = yield d
log.msg('Marking user %r as a friend %d/%d of %r' %
(friendName, thisIndex, totalToAdd, screenname))
if objectId is not None:
o = Object(objectId)
yield o.set(endpoint, friendTag, None)
log.msg('Marked user %r as a friend %d/%d of %r' %
(friendName, thisIndex, totalToAdd, screenname))
_tagFriendDone()
def makeTagFriendsJobs():
nFriendUids = len(friendUids)
for i, friendUid in enumerate(friendUids):
if userJob.canceled():
log.msg('Detected cancelation of screenname %r.' % screenname)
raise StopIteration
d = cache.userCache.screennameByUid(friendUid)
d.addCallbacks(addFriend, _ignore404uid,
callbackArgs=(i + 1, nFriendUids),
errbackArgs=(friendUid,))
yield d
# Get screename's id and add them as a Twitter user.
user = yield cache.userCache.userByScreenname(screenname)
userObject = yield makeUser(user)
log.msg('User object for %r is %r' % (screenname, userObject))
# Add the amount of work will it be to tag all friends.
userJob.workToDo += (len(friendUids) * WORK_TO_TAG_A_FRIEND)
# Figure out the work will it be to create whatever friends are needed.
friendsToAdd = [fid for fid in friendUids
if not cache.oidUidScreennameCache.knownUid(fid)]
nFriendsToAdd = len(friendsToAdd)
log.msg('Must create %d new user objects as friends of %r.' %
(nFriendsToAdd, screenname))
if nFriendsToAdd and not userJob.canceled():
userJob.workToDo += (nFriendsToAdd * WORK_TO_CREATE_A_FRIEND)
start = time.time()
# Create Fluidinfo objects for all the friends that we don't yet know
# about.
jobs = makeCreateUserJobs(friendsToAdd)
deferreds = []
coop = task.Cooperator()
for i in xrange(MAX_SIMULTANEOUS_REQUESTS):
d = coop.coiterate(jobs)
d.addErrback(log.err)
deferreds.append(d)
yield defer.DeferredList(deferreds)
if not userJob.canceled():
elapsed = time.time() - start
log.msg('Created %d new friend (of %r) objects in %.2f seconds. '
'Mean %.4f' % (nFriendsToAdd, screenname, elapsed,
float(elapsed / nFriendsToAdd)))
if friendUids and not userJob.canceled():
# Tag all friends.
start = time.time()
jobs = makeTagFriendsJobs()
deferreds = []
coop = task.Cooperator()
for i in xrange(MAX_SIMULTANEOUS_REQUESTS):
d = coop.coiterate(jobs)
d.addErrback(log.err)
deferreds.append(d)
log.msg('About to yield friend tagging DL for %r' % screenname)
yield defer.DeferredList(deferreds)
log.msg('Friend tagging DL finished for %r' % screenname)
if not userJob.canceled():
elapsed = time.time() - start
nFriendsUids = len(friendUids)
log.msg('Tagged %d objects as being a friend of %r in %.2f '
'seconds. Mean = %.4f' % (nFriendsUids, screenname,
elapsed, float(elapsed / nFriendsUids)))
if userJob.canceled():
log.msg('Canceled addUserByScreenname for %r.' % screenname)
raise Canceled(screenname)
else:
# Add the updated tag to the user's object.
log.msg('Adding updated tag to user object for %r' % screenname)
yield userObject.set(endpoint, updatedTag, int(time.time()))
log.msg('Successfully added screenname %r.' % (screenname,))
userJob.workDone = userJob.workToDo
