def handle_response(self, query, url, response, recurse=True): logger.info("url: %s => response: %s", url, response.code) if response.code != 200: if response.code == 429: raise EnhanceYourCalmException("fetch error : %s (%s)" % (response.code, response.body)) data = json.loads(response.body) results = data["results"] results.sort(key=itemgetter("id")) logger.info("got %d tweets (last tweet id: %s)", len(results), len(results) and results[-1]["id"] or None) for result in results: tweet = Tweet.parse(result) self.store.append(query, tweet) if recurse and data.has_key("next_page"): new_url = self.url_base + data["next_page"] self.handle_response(query, new_url, response)
def listen_stream(self): with tweetstream.FilterStream(self.user_password[0], self.user_password[1], track=self._queries) as stream: for tweet_data in stream: tweet = Tweet.parse(tweet_data) logger.info('got candidate tweet: %s', tweet.hashtags) if self.is_interesting_tweet(tweet): self.ts_store.append('*',tweet) logger.info('got interesting tweet: %s', tweet) if self.should_stop or not self.is_alive(): logger.info('stop command detected') break if self.should_stop or not self.is_alive(): raise StopIteration('should stop now !')
def retrieve_tweet(self, tweet_id): tweet_key = self._tweet_key(tweet_id) data = self._redis.get(tweet_key) return Tweet.deserialize(data).todict()