def store_tweets(mongo_manager, tweets):
k = 0
while True:
try:
t = tweets.next()
except RateLimitException as r:
logger.info("Twitter RateLimit. Breaking (moving to next term).")
break
except TweepError as e:
error_dict = parse_tweepy_error(e)
if error_dict["code"] == USER_DNE_ERROR:
logger.info("User does not exist. Breaking (moving to next user).")
break
else:
logger.info("Tweepy error: {0}, {1}. Breaking.".format(error_dict["code"],
error_dict["message"]))
break
except StopIteration:
logger.info("All tweets requested have been fetched.")
break
except Exception as e:
logger.warning("Unknown error, stopping: {0}".format(e))
break
mongo_manager.store_tweet(t._json)
k += 1
logger.info(".. {0} tweets fetched.".format(k))
def store_tweets(mongo_manager, tweets):
k = 0
while True:
try:
t = tweets.next()
except RateLimitException as r:
logger.info("Twitter RateLimit. Breaking (moving to next term).")
break
except TweepError as e:
error_dict = parse_tweepy_error(e)
if error_dict["code"] == USER_DNE_ERROR:
logger.info(
"User does not exist. Breaking (moving to next user).")
break
else:
logger.info("Tweepy error: {0}, {1}. Breaking.".format(
error_dict["code"], error_dict["message"]))
break
except StopIteration:
logger.info("All tweets requested have been fetched.")
break
except Exception as e:
logger.warning("Unknown error, stopping: {0}".format(e))
break
mongo_manager.store_tweet(t._json)
k += 1
logger.info(".. {0} tweets fetched.".format(k))
def _call_with_throttling_per_method(self, method_name, *args, **kwargs):
api_struct = self._pick_api_with_shortest_waiting_time_for_method(method_name)
now = datetime.now()
throttle_time = api_struct[1].get(method_name, datetime.min)
time_since_throttle = (now - throttle_time).seconds
to_wait = self.time_to_wait - time_since_throttle + 1
if to_wait > 0:
logging.debug("<{1}>: Rate limits exhausted, waiting {0} seconds".format(to_wait, now.strftime('%H:%M:%S')))
time.sleep(to_wait)
try:
return api_struct[0].__getattribute__(method_name)(*args, **kwargs)
except TweepError as e:
error_dict = parse_tweepy_error(e)
if error_dict["code"] == RATE_LIMIT_ERROR:
api_struct[1][method_name] = now
logging.debug("Received rate limit: {0}".format(error_dict["message"]))
return self._call_with_throttling_per_method(method_name, *args, **kwargs)
elif error_dict["code"] == OVER_CAP_ERROR:
api_struct[1][method_name] = now
logging.debug("Received over cap.: {0}".format(error_dict["message"]))
return self._call_with_throttling_per_method(method_name, *args, **kwargs)
else:
raise(e)
