def fetcher(self):
"""
Tweets loader
:return:
"""
for page in self.fetcherInstance.get_tweets():
for tweet in page:
try:
if not Connection.Instance().db.twitter.find_one({'user': tweet.user.screen_name,
'source': 'collector',
'data.id': tweet.id}):
Connection.Instance().db.twitter.insert_one({
'source': 'collector',
'data': {
'created_at': tweet.created_at,
'favorite_count': tweet.favorite_count,
'geo': tweet.geo,
'id': tweet.id,
'source': tweet.source,
'in_reply_to_screen_name': tweet.in_reply_to_screen_name,
'in_reply_to_status_id': tweet.in_reply_to_status_id,
'in_reply_to_user_id': tweet.in_reply_to_user_id,
'retweet_count': tweet.retweet_count,
'retweeted': tweet.retweeted,
'text': tweet.text,
'entities': tweet.entities
},
'user': tweet.user.screen_name,
'created': what_time_is_it()
})
except Exception as genericException:
logging.error("MongoDB Insert Error in collector: %s" % genericException)
import multiprocessing
ProcessManager.terminate_process(multiprocessing.current_process().pid, True)
def on_status(self, tweet):
"""
If new tweet is received, this function will be called.
For each tweet will create a new Mongo Insert
:param tweet:
:return:
"""
try:
Connection.Instance().db.twitter.insert_one({
'source':
'collector',
'data': {
'created_at': tweet.created_at,
'favorite_count': tweet.favorite_count,
'geo': tweet.geo,
'id': tweet.id,
'source': tweet.source,
'in_reply_to_screen_name': tweet.in_reply_to_screen_name,
'in_reply_to_status_id': tweet.in_reply_to_status_id,
'in_reply_to_user_id': tweet.in_reply_to_user_id,
'retweet_count': tweet.retweet_count,
'retweeted': tweet.retweeted,
'text': tweet.text,
'entities': tweet.entities
},
'user':
self.user,
'created':
what_time_is_it()
})
except Exception as e:
logging.error("MongoDB Insert Error in listener: " + e)
def terminate_process(pid, new_status):
"""
Update the "terminated" of a singe process
:param pid: Process ID
:param new_status: True / False
:return:
"""
ProcessManager.update_process(pid, {
'terminated': new_status,
'last_update': what_time_is_it()
})
def update_process_list(self, new_process, ptype):
"""
Add new process to process List.
:param new_process: Process object
:param ptype: Process type
:return:
"""
what_time_is_now = what_time_is_it()
self.insert_process({
"name": new_process.name,
"ptype": ptype,
"pid": new_process.pid,
"is_alive": new_process.is_alive(),
"created": what_time_is_now,
"terminated": False,
"last_update": what_time_is_now
})
def fetcher(self):
"""
Followers loader
:return:
"""
for page in self.fetcherInstance.get_followers():
for follower in page:
try:
if not Connection.Instance().db.twitter.find_one(
{
'source': 'follower',
'user': self.user,
'data.userid': follower.id
}):
Connection.Instance().db.twitter.insert_one({
'source':
'follower',
'data': {
'userid': follower.id,
'description': follower.description,
'favourites_count': follower.favourites_count,
'followers_count': follower.followers_count,
'friends_count': follower.friends_count,
'lang': follower.lang,
'location': follower.location,
'name': follower.name,
'screen_name': follower.screen_name,
'geo_enabled': follower.geo_enabled,
'url': follower.url,
'time_zone': follower.time_zone,
'statuses_count': follower.statuses_count
},
'user':
self.user,
'created':
what_time_is_it()
})
except Exception as e:
logging.error("MongoDB Insert Error in get followers: " +
e)
import multiprocessing
ProcessManager.terminate_process(multiprocessing.current_process().pid,
True)
def on_status(self, tweet):
"""
If new tweet is received, this function will be called.
For each tweet will create a new Mongo Insert
:param tweet:
:return:
"""
try:
Connection.Instance().db.twitter.insert_one({
'source':
'listener',
'data':
tweet._json,
'keywords':
self.keywords,
'created':
what_time_is_it()
})
except Exception as e:
logging.error("MongoDB Insert Error in listener: " + e)
def refresh_status(self):
"""
Refresh process status.
If a process is dead or in zombie status a flag 'is_alive' will be setted to False
:return:
"""
what_time_is_now = what_time_is_it()
for process in Connection.Instance().db.manager.find(
{"is_alive": True}):
if (not psutil.pid_exists(process['pid'])) or \
(psutil.pid_exists(process['pid']) and psutil.Process(process['pid']).status() ==
psutil.STATUS_ZOMBIE):
self.update_process(process['pid'], {
'is_alive': False,
'last_update': what_time_is_now
})
elif psutil.pid_exists(process['pid']):
self.update_process(process['pid'],
{'last_update': what_time_is_now})
logging.info('Refresh Done!')