def __init__(self, api, mongoClient): self.api = api super(tweepy.StreamListener, self).__init__() # Dev self.db = mongoClient.SilverEye self.tweets_counter = 0 self.tweets_counter_pp = 0 self.tweets_counter_ciudadanos = 0 self.tweets_counter_democracia_llibertat = 0 self.tweets_counter_ehbildu = 0 self.tweets_counter_erc = 0 self.tweets_counter_podemos = 0 self.tweets_counter_psoe = 0 self.tweets_counter_unio = 0 self.tweets_counter_upyd = 0 self.last_time = None self.silverEye = SilverEye('0.0.0.0', 27017)
class CustomStreamListener(tweepy.StreamListener): def __init__(self, api, mongoClient): self.api = api super(tweepy.StreamListener, self).__init__() # Dev self.db = mongoClient.SilverEye self.tweets_counter = 0 self.tweets_counter_pp = 0 self.tweets_counter_ciudadanos = 0 self.tweets_counter_democracia_llibertat = 0 self.tweets_counter_ehbildu = 0 self.tweets_counter_erc = 0 self.tweets_counter_podemos = 0 self.tweets_counter_psoe = 0 self.tweets_counter_unio = 0 self.tweets_counter_upyd = 0 self.last_time = None self.silverEye = SilverEye('0.0.0.0', 27017) def on_data(self, data): tweet = json.loads(data) # This code ignores limit notices # https://dev.twitter.com/streaming/overview/messages-types#limit_notices if tweet.get('limit'): logging.debug('Limit notice received: ' + str(tweet['limit']['track'])) logging.debug(datetime.datetime.now()) self.db.twitterLimitNotice.insert(tweet) return True user = tweet['user'] self.tweets_counter += 1 if self.last_time == None: self.last_time = datetime.datetime.now().hour if self.last_time != datetime.datetime.now().hour: self.last_time = datetime.datetime.now().hour logging.info('Tweets: ' + str(self.tweets_counter)) self.db.twitterCounter.insert({"Political": "Streaming", "datetime": datetime.datetime.now(), "num": self.tweets_counter}) # print self.tweets_counter self.tweets_counter = 0 print tweet['text'] tag = "NONE" if any(word in tweet['text'] for word in pp): tag = "PP" elif any(word in tweet['text'] for word in ciudadanos): tag = "CIUDADANOS" elif any(word in tweet['text'] for word in psoe): tag = "PSOE" elif any(word in tweet['text'] for word in podemos): tag = "PODEMOS" elif any(word in tweet['text'] for word in erc): tag = "ERC" elif any(word in tweet['text'] for word in democracia_llibertat): tag = "DEMOCRACIA LLIBERTAT" elif any(word in tweet['text'] for word in upyd): tag = "UPYD" elif any(word in tweet['text'] for word in unio): tag = "UNIO" elif any(word in tweet['text'] for word in ehbildu): tag = "EHBILDU" try: tweet["Political"] = tag self.db.twitterPolitical.update(tweet, tweet, upsert=True) self.db.twitterUser.update({"screen_name": tweet['user']['screen_name']}, user, upsert=True) self.silverEye.analyze_and_save_user_tweet(tweet, self.db.twitterPoliticalAnalyzed, self.db.twitterUser) except Exception as e: # Oh well, reconnect and keep trucking logging.error("On save to db:") logging.error(datetime.datetime.now()) logging.error(e.__class__) logging.error(e.message) logging.error("------------------") return True return True def on_error(self, status): if status == 420: logging.error('RATE LIMITED') logging.error(datetime.datetime.now()) logging.error("------------------") else: logging.error(status) logging.error(datetime.datetime.now()) logging.error("------------------") return True def on_timeout(self): logging.error('CustomStreamListener on_timeout') logging.error(datetime.datetime.now()) logging.error("------------------")