def search(self, term, newsid): print "search('%s','%s') called." % (term, newsid) self.redis.incr("news:%s:crawled_tweets" % newsid) # 1 day for _ in range(1, 2): for page in xrange(1, 16): try: results = self.twitter.search(q=term, page=("%d" % page)) for tweet_data in results["results"]: tweet_data["news_id"] = newsid tweet = Tweet(tweet_data) tweet.save() except: print "Exception: search('%s','%s')" % (term, newsid) print "-" * 60 traceback.print_exc(file=sys.stdout) continue time.sleep(120)
def on_status(self, status): print(status.text) tweet = Tweet(content = status.text) tweet.save()