def get_friends(user, page=1):
"""
Return List of friends of a specific user
:param user: User Screen Name
:param page: Page Number
:return:
"""
total_result = Connection.Instance().db.twitter.find({
"user":
user,
"source":
"friends"
}).count()
try:
page = int(page) if int(page) > 0 else 1
except:
page = 1
n_result = 10
result = list(Connection.Instance().db.twitter.find({
"source": "friends",
"user": user
}).skip((page - 1) * n_result).limit(n_result))
page_size = len(result)
return jsonify(
total_result=total_result,
status=200,
page_size=page_size,
page=page,
result=[data['data'] for data in result],
next="/api/v1/twitter/friends/%s/%s" %
(user, str(page + 1)) if page_size == n_result else None,
before="/api/v1/twitter/friends/%s/%s" %
(user, str(page - 1)) if page > 1 else None), 200
def fetcher(self):
"""
Tweets loader
:return:
"""
for page in self.fetcherInstance.get_tweets():
for tweet in page:
try:
if not Connection.Instance().db.twitter.find_one({'user': tweet.user.screen_name,
'source': 'collector',
'data.id': tweet.id}):
Connection.Instance().db.twitter.insert_one({
'source': 'collector',
'data': {
'created_at': tweet.created_at,
'favorite_count': tweet.favorite_count,
'geo': tweet.geo,
'id': tweet.id,
'source': tweet.source,
'in_reply_to_screen_name': tweet.in_reply_to_screen_name,
'in_reply_to_status_id': tweet.in_reply_to_status_id,
'in_reply_to_user_id': tweet.in_reply_to_user_id,
'retweet_count': tweet.retweet_count,
'retweeted': tweet.retweeted,
'text': tweet.text,
'entities': tweet.entities
},
'user': tweet.user.screen_name,
'created': what_time_is_it()
})
except Exception as genericException:
logging.error("MongoDB Insert Error in collector: %s" % genericException)
import multiprocessing
ProcessManager.terminate_process(multiprocessing.current_process().pid, True)
def on_status(self, tweet):
"""
If new tweet is received, this function will be called.
For each tweet will create a new Mongo Insert
:param tweet:
:return:
"""
try:
Connection.Instance().db.twitter.insert_one({
'source':
'collector',
'data': {
'created_at': tweet.created_at,
'favorite_count': tweet.favorite_count,
'geo': tweet.geo,
'id': tweet.id,
'source': tweet.source,
'in_reply_to_screen_name': tweet.in_reply_to_screen_name,
'in_reply_to_status_id': tweet.in_reply_to_status_id,
'in_reply_to_user_id': tweet.in_reply_to_user_id,
'retweet_count': tweet.retweet_count,
'retweeted': tweet.retweeted,
'text': tweet.text,
'entities': tweet.entities
},
'user':
self.user,
'created':
what_time_is_it()
})
except Exception as e:
logging.error("MongoDB Insert Error in listener: " + e)
def get_single_process(pid):
"""
Get a Single Process given the PID
:param pid: Process ID
:return:
"""
return Connection.Instance().db.manager.find_one({'pid': pid})
def insert_process(process):
"""
Insert new process
:param process: Process object
:return:
"""
return Connection.Instance().db.manager.insert_one(process)
def get_all_processes_with_condition(condition):
"""
Return the list of the processes
:return:
"""
return list(Connection.Instance().db.manager.find(condition).sort([
('last_update', -1)
]))
def update_process(pid, newobj):
"""
Update a single process
:param pid: PID
:param newobj: New Object to Set
:return:
"""
return Connection.Instance().db.manager.update({'pid': pid},
{'$set': newobj})
def get_keywords():
"""
Return the list of all available keywords
:return: Json
"""
return jsonify(status=200,
keywords=Connection.Instance().db.twitter.find({
"source":
"listener"
}).distinct("keywords")), 200
def fetcher(self):
"""
Followers loader
:return:
"""
for page in self.fetcherInstance.get_followers():
for follower in page:
try:
if not Connection.Instance().db.twitter.find_one(
{
'source': 'follower',
'user': self.user,
'data.userid': follower.id
}):
Connection.Instance().db.twitter.insert_one({
'source':
'follower',
'data': {
'userid': follower.id,
'description': follower.description,
'favourites_count': follower.favourites_count,
'followers_count': follower.followers_count,
'friends_count': follower.friends_count,
'lang': follower.lang,
'location': follower.location,
'name': follower.name,
'screen_name': follower.screen_name,
'geo_enabled': follower.geo_enabled,
'url': follower.url,
'time_zone': follower.time_zone,
'statuses_count': follower.statuses_count
},
'user':
self.user,
'created':
what_time_is_it()
})
except Exception as e:
logging.error("MongoDB Insert Error in get followers: " +
e)
import multiprocessing
ProcessManager.terminate_process(multiprocessing.current_process().pid,
True)
def get_users():
"""
Return list of users fetched
:return: Json
"""
return jsonify(status=200,
users=Connection.Instance().db.twitter.find({
"user": {
"$ne": None
}
}).distinct("user")), 200
def search(keyword, page=1):
"""
Get all tweets from a given keyword with pagination support.
:param keyword: Keyword to search
:param page: Page Number
:return:
"""
try:
page = int(page) if int(page) > 0 else 1
except:
page = 1
n_result = 10
total_result = Connection.Instance().db.twitter.find({
"keywords":
keyword,
"source":
"listener"
}).count()
result = list(Connection.Instance().db.twitter.find(
{
"keywords": keyword,
"source": "listener"
}, {
"_id": False,
"created": False,
"keywords": False,
"source": False
}).skip((page - 1) * n_result).limit(n_result))
page_size = len(result)
return jsonify(total_result=total_result,
status=200,
page_size=page_size,
page=page,
result=[data['data'] for data in result],
next="/api/v1/twitter/search/python/%s" %
str(page + 1) if page_size == n_result else None,
before="/api/v1/twitter/search/python/%s" %
str(page - 1) if page > 1 else None), 200
def on_status(self, tweet):
"""
If new tweet is received, this function will be called.
For each tweet will create a new Mongo Insert
:param tweet:
:return:
"""
try:
Connection.Instance().db.twitter.insert_one({
'source':
'listener',
'data':
tweet._json,
'keywords':
self.keywords,
'created':
what_time_is_it()
})
except Exception as e:
logging.error("MongoDB Insert Error in listener: " + e)
def refresh_status(self):
"""
Refresh process status.
If a process is dead or in zombie status a flag 'is_alive' will be setted to False
:return:
"""
what_time_is_now = what_time_is_it()
for process in Connection.Instance().db.manager.find(
{"is_alive": True}):
if (not psutil.pid_exists(process['pid'])) or \
(psutil.pid_exists(process['pid']) and psutil.Process(process['pid']).status() ==
psutil.STATUS_ZOMBIE):
self.update_process(process['pid'], {
'is_alive': False,
'last_update': what_time_is_now
})
elif psutil.pid_exists(process['pid']):
self.update_process(process['pid'],
{'last_update': what_time_is_now})
logging.info('Refresh Done!')