def bfs_get_successors ( client, node, accumulator ): tid = str(node[0]) depth = node[1] description = None if tid not in accumulator: description = None try: description = throttle_fn( client, get_description, tid ) except twython.TwythonRateLimitError as trle: debug.log( "Error rate limit.") raise trle except twython.TwythonError as te: debug.log( "Error: %s" % str(te) ) # likely a private user so just mark that description = {"private_user": True, "following_ids": [], "leaf": True} accumulator[tid] = description else: description = accumulator[tid] frontier = [] if depth != 0 and 'following_ids' not in description: following = throttle_fn( client, get_following, tid) utils.debug("loaded edges for %s" % tid) sing = set(following) description['following_ids'] = following description['leaf'] = False frontier = [ (x, depth - 1) for x in sing ] elif depth == 0 and 'leaf' not in description: description['leaf'] = True return frontier
def download_and_insert( api_client, cursor, user, cohort_id): data = None templ = None try: description = twitterMine.throttle_fn(api_client, twitterMine.get_description, user) data = to_user_tuple(description, cohort_id) templ = INSERT_USER_TEMPL except twython.TwythonAuthError as e: utils.debug("not authed. %s is probably protected" % user) data = user templ = INSERT_USER_BLOCKED_TEMPL except twython.TwythonError as e: utils.debug("Can't load user -- check if deleted %s" % user) return try: cursor.execute(templ, data) #utils.debug("Insert: %s" % str(data)) except Exception as e: utils.debug("Error: couldn't insert %s" % str(user)) utils.debug("Cause: %s\n" % str(e))