def bfs_get_successors ( client, node, accumulator ):
tid = str(node[0])
depth = node[1]
description = None
if tid not in accumulator:
description = None
try:
description = throttle_fn( client, get_description, tid )
except twython.TwythonRateLimitError as trle:
debug.log( "Error rate limit.")
raise trle
except twython.TwythonError as te:
debug.log( "Error: %s" % str(te) )
# likely a private user so just mark that
description = {"private_user": True, "following_ids": [], "leaf": True}
accumulator[tid] = description
else:
description = accumulator[tid]
frontier = []
if depth != 0 and 'following_ids' not in description:
following = throttle_fn( client, get_following, tid)
utils.debug("loaded edges for %s" % tid)
sing = set(following)
description['following_ids'] = following
description['leaf'] = False
frontier = [ (x, depth - 1) for x in sing ]
elif depth == 0 and 'leaf' not in description:
description['leaf'] = True
return frontier
def download_and_insert( api_client, cursor, user, cohort_id):
data = None
templ = None
try:
description = twitterMine.throttle_fn(api_client, twitterMine.get_description, user)
data = to_user_tuple(description, cohort_id)
templ = INSERT_USER_TEMPL
except twython.TwythonAuthError as e:
utils.debug("not authed. %s is probably protected" % user)
data = user
templ = INSERT_USER_BLOCKED_TEMPL
except twython.TwythonError as e:
utils.debug("Can't load user -- check if deleted %s" % user)
return
try:
cursor.execute(templ, data)
#utils.debug("Insert: %s" % str(data))
except Exception as e:
utils.debug("Error: couldn't insert %s" % str(user))
utils.debug("Cause: %s\n" % str(e))
