class Generic_Twitter_Crawler(object):
def __init__(self, db):
# AbstractController.__init__(self, db)
self._db = db
self._twitter_rest_api = Twitter_Rest_Api(db)
self._config_parser = getConfig()
self._domain = unicode(self._config_parser.get("DEFAULT", "domain"))
def retrieve_and_save_data_from_twitter_by_terms(self, terms):
posts, total_twitter_users = self.get_posts_and_authors_by_terms(terms)
self._db.addPosts(posts)
self._add_users_to_db(total_twitter_users)
def retrive_and_save_data_from_twitter_by_post_id(self, post_id, label):
post, user = self.get_post_and_author_by_post_id(post_id)
try:
converted_post = self._db.create_post_from_tweet_data(
post, self._domain)
converted_post.post_type = label
self._db.addPost(converted_post)
self._add_users_to_db([user])
self._db.commit()
except TwitterError as e:
exception_response = e[0][0]
logging.info("e.massage =" + exception_response["message"])
code = exception_response["code"]
logging.info("e.code =" + str(exception_response["code"]))
if code == 88:
sec = self._twitter_rest_api.get_sleep_time_for_twitter_status_id(
)
logging.info("Seconds to wait from catched crush is: " +
str(sec))
if sec != 0:
commons.count_down_time(sec)
self._num_of_twitter_status_id_requests = 0
return self.retrive_and_save_data_from_twitter_by_post_id(
post_id, label)
def get_posts_and_authors_by_terms(self, terms):
term_posts_dictionary = self.get_posts_by_terms(terms)
total_twitter_users = []
posts = []
for term in term_posts_dictionary:
posts += [
self._db.create_post_from_tweet_data(tweet, self._domain)
for tweet in term_posts_dictionary[term]
]
total_twitter_users += [
post.user for post in term_posts_dictionary[term]
]
return posts, total_twitter_users
def get_post_and_author_by_post_id(self, post_id):
post = self._twitter_rest_api.get_post_by_post_id(post_id)
user = post.user
return post, user
def _add_users_to_db(self, total_twitter_users):
author_type = None
insertion_type = None
self._twitter_rest_api.save_authors_and_connections(
total_twitter_users, author_type, insertion_type)
def get_posts_by_terms(self, terms):
return self._twitter_rest_api.get_posts_by_terms(terms)
class Generic_Twitter_Crawler(object):
def __init__(self, db):
# AbstractController.__init__(self, db)
self._db = db
self._twitter_rest_api = Twitter_Rest_Api(db)
self._config_parser = getConfig()
self._domain = unicode(self._config_parser.get("DEFAULT", "domain"))
self._users_to_add = []
self._post_to_add = []
def retrieve_and_save_data_from_twitter_by_terms(self, keywords, terms,
topics):
posts, total_twitter_users, connections = self.get_posts_and_authors_by_terms(
keywords)
self._db.addPosts(posts)
self._add_users_to_db(total_twitter_users)
self._db.addPosts(terms)
self._db.addPosts(topics)
self._db.addPosts(connections)
def commit_db(self):
self._db.addPosts(self._post_to_add)
self._add_users_to_db(self._users_to_add)
self._db.commit()
self._users_to_add = []
self._post_to_add = []
def retrieve_and_save_data_from_twitter_by_post_id(self, post_id, label):
post, user = self.get_post_and_author_by_post_id(post_id)
try:
converted_post = self._db.create_post_from_tweet_data(
post, self._domain)
converted_post.post_type = label
self._users_to_add.append(user)
self._post_to_add.append(converted_post)
except TwitterError as e:
exception_response = e[0][0]
logging.info("e.massage =" + exception_response["message"])
code = exception_response["code"]
logging.info("e.code =" + str(exception_response["code"]))
if code == 88:
sec = self._twitter_rest_api.get_sleep_time_for_twitter_status_id(
)
logging.info("Seconds to wait from catched crush is: " +
str(sec))
if sec != 0:
commons.count_down_time(sec)
self._num_of_twitter_status_id_requests = 0
return self.retrieve_and_save_data_from_twitter_by_post_id(
post_id, label)
def get_posts_and_authors_by_terms(self, keywords):
term_tweets_dict = self.get_posts_by_terms(keywords)
total_twitter_users = []
total_posts = []
connections = []
for term, tweets in term_tweets_dict.iteritems():
posts = []
for tweet in tweets:
post = self._db.create_post_from_tweet_data(
tweet, self._domain)
term_post_connection, term_author_connection = self._create_connections(
term, post)
connections.append(term_post_connection)
connections.append(term_author_connection)
posts.append(post)
total_posts += posts
#posts += [self._db.create_post_from_tweet_data(tweet, self._domain) for tweet in term_tweets_dict[term]]
total_twitter_users += [
post.user for post in term_tweets_dict[term]
]
return total_posts, total_twitter_users, connections
def get_post_and_author_by_post_id(self, post_id):
post = self._twitter_rest_api.get_post_by_post_id(post_id)
user = post.user
return post, user
def _add_users_to_db(self, total_twitter_users):
author_type = None
insertion_type = None
self._twitter_rest_api.save_authors_and_connections(
total_twitter_users, author_type, insertion_type)
def get_posts_by_terms(self, terms):
return self._twitter_rest_api.get_posts_by_terms(terms)
def _create_connections(self, term, post):
term_post_connection = AuthorConnection()
term_post_connection.source_author_guid = term
term_post_connection.destination_author_guid = post.post_id
term_post_connection.connection_type = u"term-post"
term_author_connection = AuthorConnection()
term_author_connection.source_author_guid = term
term_author_connection.destination_author_guid = post.author_guid
term_author_connection.connection_type = u"term-author"
return term_post_connection, term_author_connection
class MissingDataComplementor(Method_Executor):
def __init__(self, db):
Method_Executor.__init__(self, db)
self._actions = self._config_parser.eval(self.__class__.__name__,
"actions")
self._minimal_num_of_posts = self._config_parser.eval(
self.__class__.__name__, "minimal_num_of_posts")
self._limit_friend_follower_number = self._config_parser.eval(
self.__class__.__name__, "limit_friend_follower_number")
self._maximal_tweets_count_in_timeline = self._config_parser.eval(
self.__class__.__name__, "maximal_tweets_count_in_timeline")
self._found_twitter_users = []
self._social_network_crawler = Twitter_Rest_Api(db)
self._suspended_authors = []
self._max_users_without_saving = self._config_parser.eval(
self.__class__.__name__, "max_users_without_saving")
self._posts = []
self._authors = []
self._post_citatsions = []
def setUp(self):
pass
def fill_author_guid_to_posts(self):
posts = self._db.get_posts()
num_of_posts = len(posts)
for i, post in enumerate(posts):
msg = "\rPosts to fill: [{0}/{1}]".format(i, num_of_posts)
print(msg, end="")
post.author_guid = compute_author_guid_by_author_name(post.author)
self._db.addPosts(posts)
self._db.insert_or_update_authors_from_posts(self._domain, {}, {})
def fill_data_for_followers(self):
self._fill_data_for_author_connection_type(
Author_Connection_Type.FOLLOWER)
logging.info("---Finished crawl_followers_by_author_ids")
def fill_data_for_friends(self):
self._fill_data_for_author_connection_type(
Author_Connection_Type.FRIEND)
logging.info("---Finished crawl_friends_by_author_ids")
def _fill_data_for_author_connection_type(self, connection_type):
# TEST
self._db.get_authors_by_domain("Microblog")
# TEST
cursor = self._db.get_followers_or_friends_candidats(
connection_type, self._domain, self._limit_friend_follower_number)
followers_or_friends_candidats = self._db.result_iter(cursor)
followers_or_friends_candidats = [
author_id[0] for author_id in followers_or_friends_candidats
]
print("---crawl_followers_by_author_ids---")
author_type = None
are_user_ids = True
insertion_type = DB_Insertion_Type.MISSING_DATA_COMPLEMENTOR
crawl_users_by_author_ids_func_name = "crawl_users_by_author_ids"
getattr(self._social_network_crawler,
crawl_users_by_author_ids_func_name)(
followers_or_friends_candidats, connection_type,
author_type, are_user_ids, insertion_type)
self._db.convert_temp_author_connections_to_author_connections(
self._domain)
def crawl_followers_by_author_ids(self, author_ids):
print("---crawl_followers_by_author_ids---")
author_type = None
are_user_ids = True
inseration_type = DB_Insertion_Type.MISSING_DATA_COMPLEMENTOR
self._social_network_crawler.crawl_followers_by_twitter_author_ids(
author_ids, author_type, are_user_ids, inseration_type)
def crawl_friends_by_author_ids(self, author_ids):
print("---crawl_friends_by_author_ids---")
author_type = None
are_user_ids = True
inseration_type = DB_Insertion_Type.MISSING_DATA_COMPLEMENTOR
self._social_network_crawler.crawl_friends_by_twitter_author_ids(
author_ids, author_type, are_user_ids, inseration_type)
def create_author_screen_names(self):
screen_names = self._db.get_screen_names_for_twitter_authors_by_posts()
return screen_names
def fill_data_for_sources(self):
print(
"---complete_missing_information_for_authors_by_screen_names ---")
logging.info(
"---complete_missing_information_for_authors_by_screen_names ---")
# twitter_author_screen_names = self.create_author_screen_names()
twitter_author_screen_names = self._db.get_missing_data_twitter_screen_names(
)
# twitter_author_screen_names = (twitter_author.name for twitter_author in twitter_authors)
# twitter_author_screen_names = list(twitter_author_screen_names)
author_type = None
are_user_ids = False
inseration_type = DB_Insertion_Type.MISSING_DATA_COMPLEMENTOR
# retrieve_full_data_for_missing_users
total_twitter_users = self._social_network_crawler.handle_get_users_request(
twitter_author_screen_names, are_user_ids, author_type,
inseration_type)
self._social_network_crawler.save_authors_and_connections(
total_twitter_users, author_type, inseration_type)
print(
"---complete_missing_information_for_authors_by_screen_names was completed!!!!---"
)
logging.info(
"---complete_missing_information_for_authors_by_screen_names was completed!!!!---"
)
return total_twitter_users
def complete_missing_information_for_authors_by_ids(self):
print("---complete_missing_information_for_authors_by_ids ---")
logging.info("---complete_missing_information_for_authors_by_ids ---")
# twitter_author_screen_names = self.create_author_screen_names()
twitter_author_screen_names = self._db.get_missing_data_twitter_screen_names(
)
# twitter_author_screen_names = (twitter_author.name for twitter_author in twitter_authors)
# twitter_author_screen_names = list(twitter_author_screen_names)
author_type = None
are_user_ids = False
inseration_type = DB_Insertion_Type.MISSING_DATA_COMPLEMENTOR
# retrieve_full_data_for_missing_users
total_twitter_users = self._social_network_crawler.handle_get_users_request(
twitter_author_screen_names, are_user_ids, author_type,
inseration_type)
# return self._found_twitter_users
print(
"---complete_missing_information_for_authors was completed!!!!---")
logging.info(
"---complete_missing_information_for_authors was completed!!!!---")
return total_twitter_users
def mark_suspended_or_not_existed_authors(self):
suspended_authors = self._db.get_authors_for_mark_as_suspended_or_not_existed(
)
for suspended_author in suspended_authors:
suspended_author.is_suspended_or_not_exists = self._window_start
self._db.set_inseration_date(
suspended_author, DB_Insertion_Type.MISSING_DATA_COMPLEMENTOR)
self._social_network_crawler.save_authors(suspended_authors)
def mark_suspended_from_twitter(self):
self._suspended_authors = []
suspected_authors = self._db.get_not_suspended_authors(self._domain)
suspected_authors_names = [author.name for author in suspected_authors]
chunks = split_into_equal_chunks(
suspected_authors_names, self._social_network_crawler.
_maximal_user_ids_allowed_in_single_get_user_request)
total_chunks = list(chunks)
chunks = split_into_equal_chunks(
suspected_authors_names, self._social_network_crawler.
_maximal_user_ids_allowed_in_single_get_user_request)
i = 1
for chunk_of_names in chunks:
msg = "\rChunck of author to Twitter: [{0}/{1}]".format(
i, len(total_chunks))
print(msg, end="")
i += 1
set_of_send_author_names = set(chunk_of_names)
set_of_received_author_names = set(
self._social_network_crawler.
get_active_users_names_by_screen_names(chunk_of_names))
author_names_of_suspendend_or_not_exists = set_of_send_author_names - set_of_received_author_names
self._update_suspended_authors_by_screen_names(
author_names_of_suspendend_or_not_exists)
self._db.add_authors(self._suspended_authors)
def _update_suspended_authors_by_screen_names(
self, author_names_of_suspendend_or_not_exists):
for author_name in author_names_of_suspendend_or_not_exists:
user_guid = compute_author_guid_by_author_name(
author_name).replace("-", "")
suspended_author = self._db.get_author_by_author_guid(user_guid)
suspended_author.is_suspended_or_not_exists = self._window_start
suspended_author.author_type = Author_Type.BAD_ACTOR
self._db.set_inseration_date(
suspended_author, DB_Insertion_Type.MISSING_DATA_COMPLEMENTOR)
self._suspended_authors.append(suspended_author)
num_of_suspended_authors = len(self._suspended_authors)
if num_of_suspended_authors == self._max_users_without_saving:
self._db.add_authors(self._suspended_authors)
self._suspended_authors = []
def fill_tweet_retweet_connection(self):
'''
Fetches the original tweets being retweeted by our posts.
Updates the followig tables:
* Post_Citations table with tweet-retweet connection
* Posts table with missing tweets
* Authors with the authors of the missing tweets
'''
retweets_with_no_tweet_citation = self._db.get_retweets_with_no_tweet_citation(
)
logging.info(
"Updating tweet-retweet connection of {0} retweets".format(
len(retweets_with_no_tweet_citation)))
self._posts = []
self._authors = []
self._post_citatsions = []
i = 1
for post_guid, post_url in retweets_with_no_tweet_citation.iteritems():
# logging.info("Analyzing retweet: {0} - {1}".format(post_guid, post_url))
msg = "\r Analyzing retweet: {0} - {1} [{2}".format(
post_guid, post_url, i) + "/" + str(
len(retweets_with_no_tweet_citation)) + '] '
print(msg, end="")
i += 1
tweet_data = self.extract_retweet_data(retweet_guid=post_guid,
retweet_url=post_url)
if tweet_data is not None:
if not self._db.isPostExist(tweet_data.tweet_url):
post = Post(guid=tweet_data.tweet_guid,
post_id=tweet_data.tweet_guid,
url=tweet_data.tweet_url,
date=str_to_date(tweet_data.tweet_date),
title=tweet_data.tweet_content,
content=tweet_data.tweet_content,
post_osn_id=tweet_data.tweet_twitter_id,
retweet_count=tweet_data.tweet_retweet_count,
favorite_count=tweet_data.tweet_favorite_count,
author=tweet_data.tweet_author_name,
author_guid=tweet_data.tweet_author_guid,
domain=self._domain,
original_tweet_importer_insertion_date=unicode(
get_current_time_as_string()))
self._posts.append(post)
if not self._db.is_author_exists(tweet_data.tweet_author_guid,
self._domain):
author = Author(
name=tweet_data.tweet_author_name,
domain=self._domain,
author_guid=tweet_data.tweet_author_guid,
original_tweet_importer_insertion_date=unicode(
get_current_time_as_string()))
self._authors.append(author)
if not self._db.is_post_citation_exist(tweet_data.retweet_guid,
tweet_data.tweet_guid):
post_citation = Post_citation(
post_id_from=tweet_data.retweet_guid,
post_id_to=tweet_data.tweet_guid,
url_from=tweet_data.retweet_url,
url_to=tweet_data.tweet_url)
self._post_citatsions.append(post_citation)
self.update_tables_with_tweet_retweet_data(self._posts, self._authors,
self._post_citatsions)
def extract_retweet_data(self, retweet_guid, retweet_url):
'''
:param retweet_guid: the guid of the retweet
:param retweet_url: the url of the retweet
:return: a RetweetData holding the data of the retweet
'''
try:
retweet_id = self.extract_tweet_id(retweet_url)
if retweet_id is None:
return None
retweet_status = self._social_network_crawler.get_status_by_twitter_status_id(
retweet_id)
tweet_status_dict = retweet_status.AsDict()
if 'retweeted_status' in tweet_status_dict:
tweet_status_dict = tweet_status_dict['retweeted_status']
tweet_post_twitter_id = unicode(str(tweet_status_dict['id']))
tweet_author_name = unicode(
tweet_status_dict['user']['screen_name'])
tweet_url = unicode(
generate_tweet_url(tweet_post_twitter_id,
tweet_author_name))
tweet_creation_time = unicode(tweet_status_dict['created_at'])
tweet_str_publication_date = unicode(
extract_tweet_publiction_date(tweet_creation_time))
tweet_guid = unicode(
compute_post_guid(
post_url=tweet_url,
author_name=tweet_author_name,
str_publication_date=tweet_str_publication_date))
tweet_author_guid = unicode(
compute_author_guid_by_author_name(tweet_author_name))
tweet_author_guid = unicode(tweet_author_guid.replace("-", ""))
tweet_content = unicode(tweet_status_dict['text'])
tweet_retweet_count = unicode(
tweet_status_dict['retweet_count'])
tweet_favorite_count = unicode(
tweet_status_dict['favorite_count'])
retweet_data = RetweetData(
retweet_guid=retweet_guid,
retweet_url=retweet_url,
tweet_guid=tweet_guid,
tweet_url=tweet_url,
tweet_author_name=tweet_author_name,
tweet_author_guid=tweet_author_guid,
tweet_date=tweet_str_publication_date,
tweet_content=tweet_content,
tweet_twitter_id=tweet_post_twitter_id,
tweet_retweet_count=tweet_retweet_count,
tweet_favorite_count=tweet_favorite_count)
return retweet_data
else:
return None
except TwitterError as e:
exception_response = e[0][0]
logging.info("e.massage =" + exception_response["message"])
code = exception_response["code"]
logging.info("e.code =" + str(exception_response["code"]))
self.update_tables_with_tweet_retweet_data(self._posts,
self._authors,
self._post_citatsions)
self._posts = []
self._authors = []
self._post_citatsions = []
if code == 88:
sec = self._social_network_crawler.get_sleep_time_for_twitter_status_id(
)
logging.info("Seconds to wait from catched crush is: " +
str(sec))
if sec != 0:
count_down_time(sec)
self._num_of_twitter_status_id_requests = 0
return self._social_network_crawler.get_status(retweet_id)
except Exception as e:
logging.error(
"Cannot fetch data for retweet: {0}. Error message: {1}".
format(retweet_url, e.message))
return None
def extract_tweet_id(self, post_url):
post_url = str(post_url)
pattern = re.compile("http(.*)://twitter.com/(.*)/statuses/(.*)")
extracted_info = pattern.findall(post_url)
if extracted_info == []:
pattern = re.compile("http(.*)://twitter.com/(.*)/status/(.*)")
extracted_info = pattern.findall(post_url)
if len(extracted_info[0]) < 2:
return None
else:
return extracted_info[0][2]
def update_tables_with_tweet_retweet_data(self, posts, authors,
post_citatsions):
self._db.addPosts(posts)
self._db.add_authors(authors)
self._db.addReferences(post_citatsions)
def fill_authors_time_line(self):
'''
Fetches the posts for the authors that are given under authors_twitter_ids_for_timeline_filling in the config file +
update the db
'''
self._db.create_authors_index()
self._db.create_posts_index()
author_screen_names_number_of_posts = self._db.get_author_screen_names_and_number_of_posts(
self._minimal_num_of_posts)
author_screen_names_number_of_posts_dict = self._create_author_screen_name_number_of_posts_dictionary(
author_screen_names_number_of_posts)
index = 1
for author_name in author_screen_names_number_of_posts_dict:
print("Get timeline for {0} : {1}/{2}".format(
author_name, str(index),
str(len(author_screen_names_number_of_posts_dict))))
index += 1
posts = []
logging.info("Fetching timeline for author: " + str(author_name))
posts_counter = 0
try:
posts_needed_from_osn = self._minimal_num_of_posts - author_screen_names_number_of_posts_dict[
author_name]
timeline = self._social_network_crawler.get_timeline_by_author_name(
author_name, posts_needed_from_osn)
# logging.info("Retrived timeline lenght: " + str(len(timeline)))
if timeline is not None:
for post in timeline:
tweet_post_twitter_id = str(post.id)
tweet_url = generate_tweet_url(tweet_post_twitter_id,
author_name)
tweet_creation_time = post.created_at
tweet_str_publication_date = extract_tweet_publiction_date(
tweet_creation_time)
tweet_guid = compute_post_guid(
post_url=tweet_url,
author_name=author_name,
str_publication_date=tweet_str_publication_date)
if self._db.contains_post(tweet_url):
continue
posts_counter = posts_counter + 1
tweet_author_guid = compute_author_guid_by_author_name(
author_name)
post = self._db.create_post_from_tweet_data(
post, self._domain)
posts.append(post)
except Exception as e:
logging.error(
"Cannot fetch data for author: {0}. Error message: {1}".
format(author_name, e.message))
logging.info("Number of posts inserted for author {0}: {1}".format(
author_name, posts_counter))
self._db.addPosts(posts)
def assign_manually_labeled_authors(self):
self._db.assign_manually_labeled_authors()
def delete_acquired_authors(self):
self._db.delete_acquired_authors()
self._db.delete_posts_with_missing_authors()
def delete_manually_labeled_authors(self):
self._db.delete_manually_labeled_authors()
self._db.delete_posts_with_missing_authors()
def assign_acquired_and_crowd_turfer_profiles(self):
self._db.assign_crowdturfer_profiles()
self._db.assign_acquired_profiles()
def _create_author_screen_name_number_of_posts_dictionary(
self, author_screen_names_number_of_posts):
author_screen_names_number_of_posts_dict = {}
for record in author_screen_names_number_of_posts:
author_screen_name = record[0]
num_of_posts = record[1]
author_screen_names_number_of_posts_dict[
author_screen_name] = num_of_posts
logging.info("Number of users to retrieve timelines: " +
str(len(author_screen_names_number_of_posts_dict)))
return author_screen_names_number_of_posts_dict