def handle_response(self, query, url, response, recurse=True):
logger.info("url: %s => response: %s", url, response.code)
if response.code != 200:
if response.code == 429:
raise EnhanceYourCalmException("fetch error : %s (%s)" % (response.code, response.body))
data = json.loads(response.body)
results = data["results"]
results.sort(key=itemgetter("id"))
logger.info("got %d tweets (last tweet id: %s)", len(results), len(results) and results[-1]["id"] or None)
for result in results:
tweet = Tweet.parse(result)
self.store.append(query, tweet)
if recurse and data.has_key("next_page"):
new_url = self.url_base + data["next_page"]
self.handle_response(query, new_url, response)
def listen_stream(self):
with tweetstream.FilterStream(self.user_password[0],
self.user_password[1],
track=self._queries) as stream:
for tweet_data in stream:
tweet = Tweet.parse(tweet_data)
logger.info('got candidate tweet: %s', tweet.hashtags)
if self.is_interesting_tweet(tweet):
self.ts_store.append('*',tweet)
logger.info('got interesting tweet: %s', tweet)
if self.should_stop or not self.is_alive():
logger.info('stop command detected')
break
if self.should_stop or not self.is_alive():
raise StopIteration('should stop now !')
def retrieve_tweet(self, tweet_id):
tweet_key = self._tweet_key(tweet_id)
data = self._redis.get(tweet_key)
return Tweet.deserialize(data).todict()
