def store_tweets(mongo_manager, tweets): k = 0 while True: try: t = tweets.next() except RateLimitException as r: logger.info("Twitter RateLimit. Breaking (moving to next term).") break except TweepError as e: error_dict = parse_tweepy_error(e) if error_dict["code"] == USER_DNE_ERROR: logger.info("User does not exist. Breaking (moving to next user).") break else: logger.info("Tweepy error: {0}, {1}. Breaking.".format(error_dict["code"], error_dict["message"])) break except StopIteration: logger.info("All tweets requested have been fetched.") break except Exception as e: logger.warning("Unknown error, stopping: {0}".format(e)) break mongo_manager.store_tweet(t._json) k += 1 logger.info(".. {0} tweets fetched.".format(k))
def store_tweets(mongo_manager, tweets): k = 0 while True: try: t = tweets.next() except RateLimitException as r: logger.info("Twitter RateLimit. Breaking (moving to next term).") break except TweepError as e: error_dict = parse_tweepy_error(e) if error_dict["code"] == USER_DNE_ERROR: logger.info( "User does not exist. Breaking (moving to next user).") break else: logger.info("Tweepy error: {0}, {1}. Breaking.".format( error_dict["code"], error_dict["message"])) break except StopIteration: logger.info("All tweets requested have been fetched.") break except Exception as e: logger.warning("Unknown error, stopping: {0}".format(e)) break mongo_manager.store_tweet(t._json) k += 1 logger.info(".. {0} tweets fetched.".format(k))
def _call_with_throttling_per_method(self, method_name, *args, **kwargs): api_struct = self._pick_api_with_shortest_waiting_time_for_method(method_name) now = datetime.now() throttle_time = api_struct[1].get(method_name, datetime.min) time_since_throttle = (now - throttle_time).seconds to_wait = self.time_to_wait - time_since_throttle + 1 if to_wait > 0: logging.debug("<{1}>: Rate limits exhausted, waiting {0} seconds".format(to_wait, now.strftime('%H:%M:%S'))) time.sleep(to_wait) try: return api_struct[0].__getattribute__(method_name)(*args, **kwargs) except TweepError as e: error_dict = parse_tweepy_error(e) if error_dict["code"] == RATE_LIMIT_ERROR: api_struct[1][method_name] = now logging.debug("Received rate limit: {0}".format(error_dict["message"])) return self._call_with_throttling_per_method(method_name, *args, **kwargs) elif error_dict["code"] == OVER_CAP_ERROR: api_struct[1][method_name] = now logging.debug("Received over cap.: {0}".format(error_dict["message"])) return self._call_with_throttling_per_method(method_name, *args, **kwargs) else: raise(e)