class Generic_Twitter_Crawler(object): def __init__(self, db): # AbstractController.__init__(self, db) self._db = db self._twitter_rest_api = Twitter_Rest_Api(db) self._config_parser = getConfig() self._domain = unicode(self._config_parser.get("DEFAULT", "domain")) self._users_to_add = [] self._post_to_add = [] def retrieve_and_save_data_from_twitter_by_terms(self, keywords, terms, topics): posts, total_twitter_users, connections = self.get_posts_and_authors_by_terms( keywords) self._db.addPosts(posts) self._add_users_to_db(total_twitter_users) self._db.addPosts(terms) self._db.addPosts(topics) self._db.addPosts(connections) def commit_db(self): self._db.addPosts(self._post_to_add) self._add_users_to_db(self._users_to_add) self._db.commit() self._users_to_add = [] self._post_to_add = [] def retrieve_and_save_data_from_twitter_by_post_id(self, post_id, label): post, user = self.get_post_and_author_by_post_id(post_id) try: converted_post = self._db.create_post_from_tweet_data( post, self._domain) converted_post.post_type = label self._users_to_add.append(user) self._post_to_add.append(converted_post) except TwitterError as e: exception_response = e[0][0] logging.info("e.massage =" + exception_response["message"]) code = exception_response["code"] logging.info("e.code =" + str(exception_response["code"])) if code == 88: sec = self._twitter_rest_api.get_sleep_time_for_twitter_status_id( ) logging.info("Seconds to wait from catched crush is: " + str(sec)) if sec != 0: commons.count_down_time(sec) self._num_of_twitter_status_id_requests = 0 return self.retrieve_and_save_data_from_twitter_by_post_id( post_id, label) def get_posts_and_authors_by_terms(self, keywords): term_tweets_dict = self.get_posts_by_terms(keywords) total_twitter_users = [] total_posts = [] connections = [] for term, tweets in term_tweets_dict.iteritems(): posts = [] for tweet in tweets: post = self._db.create_post_from_tweet_data( tweet, self._domain) term_post_connection, term_author_connection = self._create_connections( term, post) connections.append(term_post_connection) connections.append(term_author_connection) posts.append(post) total_posts += posts #posts += [self._db.create_post_from_tweet_data(tweet, self._domain) for tweet in term_tweets_dict[term]] total_twitter_users += [ post.user for post in term_tweets_dict[term] ] return total_posts, total_twitter_users, connections def get_post_and_author_by_post_id(self, post_id): post = self._twitter_rest_api.get_post_by_post_id(post_id) user = post.user return post, user def _add_users_to_db(self, total_twitter_users): author_type = None insertion_type = None self._twitter_rest_api.save_authors_and_connections( total_twitter_users, author_type, insertion_type) def get_posts_by_terms(self, terms): return self._twitter_rest_api.get_posts_by_terms(terms) def _create_connections(self, term, post): term_post_connection = AuthorConnection() term_post_connection.source_author_guid = term term_post_connection.destination_author_guid = post.post_id term_post_connection.connection_type = u"term-post" term_author_connection = AuthorConnection() term_author_connection.source_author_guid = term term_author_connection.destination_author_guid = post.author_guid term_author_connection.connection_type = u"term-author" return term_post_connection, term_author_connection
class Generic_Twitter_Crawler(object): def __init__(self, db): # AbstractController.__init__(self, db) self._db = db self._twitter_rest_api = Twitter_Rest_Api(db) self._config_parser = getConfig() self._domain = unicode(self._config_parser.get("DEFAULT", "domain")) def retrieve_and_save_data_from_twitter_by_terms(self, terms): posts, total_twitter_users = self.get_posts_and_authors_by_terms(terms) self._db.addPosts(posts) self._add_users_to_db(total_twitter_users) def retrive_and_save_data_from_twitter_by_post_id(self, post_id, label): post, user = self.get_post_and_author_by_post_id(post_id) try: converted_post = self._db.create_post_from_tweet_data( post, self._domain) converted_post.post_type = label self._db.addPost(converted_post) self._add_users_to_db([user]) self._db.commit() except TwitterError as e: exception_response = e[0][0] logging.info("e.massage =" + exception_response["message"]) code = exception_response["code"] logging.info("e.code =" + str(exception_response["code"])) if code == 88: sec = self._twitter_rest_api.get_sleep_time_for_twitter_status_id( ) logging.info("Seconds to wait from catched crush is: " + str(sec)) if sec != 0: commons.count_down_time(sec) self._num_of_twitter_status_id_requests = 0 return self.retrive_and_save_data_from_twitter_by_post_id( post_id, label) def get_posts_and_authors_by_terms(self, terms): term_posts_dictionary = self.get_posts_by_terms(terms) total_twitter_users = [] posts = [] for term in term_posts_dictionary: posts += [ self._db.create_post_from_tweet_data(tweet, self._domain) for tweet in term_posts_dictionary[term] ] total_twitter_users += [ post.user for post in term_posts_dictionary[term] ] return posts, total_twitter_users def get_post_and_author_by_post_id(self, post_id): post = self._twitter_rest_api.get_post_by_post_id(post_id) user = post.user return post, user def _add_users_to_db(self, total_twitter_users): author_type = None insertion_type = None self._twitter_rest_api.save_authors_and_connections( total_twitter_users, author_type, insertion_type) def get_posts_by_terms(self, terms): return self._twitter_rest_api.get_posts_by_terms(terms)