def __init__(self, api, mongoClient):
self.api = api
super(tweepy.StreamListener, self).__init__()
# Dev
self.db = mongoClient.SilverEye
self.tweets_counter = 0
self.tweets_counter_pp = 0
self.tweets_counter_ciudadanos = 0
self.tweets_counter_democracia_llibertat = 0
self.tweets_counter_ehbildu = 0
self.tweets_counter_erc = 0
self.tweets_counter_podemos = 0
self.tweets_counter_psoe = 0
self.tweets_counter_unio = 0
self.tweets_counter_upyd = 0
self.last_time = None
self.silverEye = SilverEye('0.0.0.0', 27017)
class CustomStreamListener(tweepy.StreamListener):
def __init__(self, api, mongoClient):
self.api = api
super(tweepy.StreamListener, self).__init__()
# Dev
self.db = mongoClient.SilverEye
self.tweets_counter = 0
self.tweets_counter_pp = 0
self.tweets_counter_ciudadanos = 0
self.tweets_counter_democracia_llibertat = 0
self.tweets_counter_ehbildu = 0
self.tweets_counter_erc = 0
self.tweets_counter_podemos = 0
self.tweets_counter_psoe = 0
self.tweets_counter_unio = 0
self.tweets_counter_upyd = 0
self.last_time = None
self.silverEye = SilverEye('0.0.0.0', 27017)
def on_data(self, data):
tweet = json.loads(data)
# This code ignores limit notices
# https://dev.twitter.com/streaming/overview/messages-types#limit_notices
if tweet.get('limit'):
logging.debug('Limit notice received: ' + str(tweet['limit']['track']))
logging.debug(datetime.datetime.now())
self.db.twitterLimitNotice.insert(tweet)
return True
user = tweet['user']
self.tweets_counter += 1
if self.last_time == None:
self.last_time = datetime.datetime.now().hour
if self.last_time != datetime.datetime.now().hour:
self.last_time = datetime.datetime.now().hour
logging.info('Tweets: ' + str(self.tweets_counter))
self.db.twitterCounter.insert({"Political": "Streaming", "datetime": datetime.datetime.now(),
"num": self.tweets_counter})
# print self.tweets_counter
self.tweets_counter = 0
print tweet['text']
tag = "NONE"
if any(word in tweet['text'] for word in pp):
tag = "PP"
elif any(word in tweet['text'] for word in ciudadanos):
tag = "CIUDADANOS"
elif any(word in tweet['text'] for word in psoe):
tag = "PSOE"
elif any(word in tweet['text'] for word in podemos):
tag = "PODEMOS"
elif any(word in tweet['text'] for word in erc):
tag = "ERC"
elif any(word in tweet['text'] for word in democracia_llibertat):
tag = "DEMOCRACIA LLIBERTAT"
elif any(word in tweet['text'] for word in upyd):
tag = "UPYD"
elif any(word in tweet['text'] for word in unio):
tag = "UNIO"
elif any(word in tweet['text'] for word in ehbildu):
tag = "EHBILDU"
try:
tweet["Political"] = tag
self.db.twitterPolitical.update(tweet, tweet, upsert=True)
self.db.twitterUser.update({"screen_name": tweet['user']['screen_name']}, user, upsert=True)
self.silverEye.analyze_and_save_user_tweet(tweet, self.db.twitterPoliticalAnalyzed, self.db.twitterUser)
except Exception as e:
# Oh well, reconnect and keep trucking
logging.error("On save to db:")
logging.error(datetime.datetime.now())
logging.error(e.__class__)
logging.error(e.message)
logging.error("------------------")
return True
return True
def on_error(self, status):
if status == 420:
logging.error('RATE LIMITED')
logging.error(datetime.datetime.now())
logging.error("------------------")
else:
logging.error(status)
logging.error(datetime.datetime.now())
logging.error("------------------")
return True
def on_timeout(self):
logging.error('CustomStreamListener on_timeout')
logging.error(datetime.datetime.now())
logging.error("------------------")
