def add_users(usernames=["SSDummy_Janet", "ssdummy_henry", "ssdummy_hoot", "ssdummy_faye" ,"ssdummy_burt", "ssdummy_gustavo", "ssdummy_amanda" ,"ssdummy_duke", "ssdummy_ian", "ssdummy_ellen" , "ssdummy_chrissy"], clear=False): '''Used by analyse team to test their accuracy! Removes all users from the database and then adds the ones in the usernames list. TODO: test it''' sh = StorageHandler(SOLR_SERVER) if clear: print "CLEARING DATABASE!" sh.delete_all() users_left_to_add = len(usernames) for username in usernames: retry = True while retry: try: print "Adding: " + username + " Left to add: " +str(users_left_to_add) users_left_to_add = users_left_to_add - 1 addalyse.addalyse(sh, username) retry = False except addalyse.AddalyseRateLimitExceededError: sys.stderr.write("Rate limit exceeded, waiting " + str(CONFIG.RATE_LIMIT_EXCEEDED_TIME) + " seconds.\n") time.sleep(CONFIG.RATE_LIMIT_EXCEEDED_TIME) retry = True except addalyse.AddalyseError as err: sys.stderr.write("Got error from addalyse: " + str(err) + "\n") retry = False except Exception: sys.stderr.write("Unhandled exception\n") traceback.print_exc() retry = False print "Done adding test users!"
def load_existing_users(): """Loads users from the storage handler, to gain information of which users to ignore. @return: Set containing twitter usernames.""" # TODO: Implement a real solution, calling the storage handler. if not NO_ANALYSE: sh = StorageHandler(SOLR_SERVER) mset = set([a for (a,) in sh.get_user_fields("*", "id")]) print mset return mset
def __init__(self, api_key_path): self.api_key_path = api_key_path self.visionHandler = VisionHandler(api_key_path) self.translationHandler = TranslationHandler(api_key_path) self.scraper = Scraper() self.storageHandler = StorageHandler(api_key_path) self.overlayHandler = OverlayHandler()
def gather_data_loop(request_per_hour=3600, users_to_add=21, no_analyse=False): """Gathers data about twitter IDs, and sends the data to the storage handler.""" global CONFIG # TODO: Change for real implementation! sleep_time = 3600 / request_per_hour th = TwitterHelp() if not NO_ANALYSE: sh = StorageHandler(SOLR_SERVER) added_users = 0 # Creates a set for all the users that will be added successfully users_added = set() while added_users < users_to_add: # The set of users which will be added. try: set_to_add = th.get_public_tweeters() except twitter.TwitterError as err: if err.message[0:19] == "Rate limit exceeded": # TODO: optimal version of this would query the twitter api for how long to wait exactly! sys.stderr.write( "Rate limit exceeded while trying to get public timeline, trying again in " + str(CONFIG.get_rate_limit_exceeded_time()) + " seconds.\n" ) time.sleep(CONFIG.get_rate_limit_exceeded_time()) else: sys.stderr.write( "Got TwitterError while trying to get public timeline " + str(err) + ". Retrying soon.\n" ) traceback.print_exc() time.sleep(100) continue # retry the loop if not NO_ANALYSE: print "These will be added:" for s in set_to_add: print s for user in set_to_add: if NO_ANALYSE: tweets = th.get_all_statuses(user) print "#####_NEW_USER_#####" print user for t in tweets: try: text = t.GetText() print "#####_NEW_TWEET_#####" print text print "#####_END_OF_TWEET_#####" except UnicodeEncodeError: continue time.sleep(sleep_time) else: if not sh.contains(user): retry = True # A retry variable for an inner "goto" while retry: time.sleep(sleep_time) try: if addalyse.addalyse(SOLR_SERVER, user): users_added.add(user) added_users += 1 retry = False except addalyse.AddalyseRateLimitExceededError as err: # Halt for 1 hour if the rate limit is exceeded sys.stderr.write( "RateLimitExceeded, trying again in " + str(CONFIG.get_rate_limit_exceeded_time()) + " seconds.\n" ) time.sleep(CONFIG.get_rate_limit_exceeded_time()) retry = True except addalyse.AddalyseError as err: # we use polymorphism here, WEE sys.stderr.write("Addalyse threw an error: " + str(err) + "\n") retry = False except Exception: # ignore errors non-silently (we print tracebacks!) # TODO: use the logger for this? sys.stderr.write("Unhandled exception\n") traceback.print_exc() retry = False # For debugging purposes, displays all users found in this session. if not NO_ANALYSE: for key in users_added: print key + " was added"
def redo_all_users(): print "Redoing all users in Solr like the mythological baws." sh = StorageHandler(SOLR_SERVER) add_users(map(lambda (a,): a, sh.get_user_fields('*', 'id')))