def run_store_profile_info(self, names, query_id): sql = MySqlUtils() non_processed_handles = names # list responses_iterator = self.yield_profile_requests(non_processed_handles) for responses in responses_iterator: for response in responses: if response and response.status_code == 200: twitter_user_dictionary = self.get_profile_dictionary( response, query_id) try: sql.dict_to_sql(twitter_user_dictionary, 'user') except Exception as e: print(e) print(twitter_user_dictionary)
def got_query(query, start_date, end_date, max_num): # Wrapper for get old tweets #:param start_date: #:param end_date: #:return: print("{} {} {} {}".format(query, start_date, end_date, max_num)) tweetCriteria = gotmanager.TweetCriteria().setQuerySearch(query).setSince( start_date).setUntil(end_date).setMaxTweets(max_num) tweets = gotmanager.TweetManager.getTweets(tweetCriteria) sql = MySqlUtils() for tweet in tweets: tweet.query = query # print(tweet.__dict__) sql.dict_to_sql(get_cleaned_tweet_dictionary(tweet), 'tweet')
def got_username(user_handle=None, start_date=None, end_date=None, max_num=None, include_retweets=None, include_replies=False): if start_date is None: start_date = "2017-01-01" if end_date is None: end_date = "2018-01-01" #usernames=joblib.load('/home/analytics/analytics/data/username.txt') print(start_date) for name in user_handle: print(name) print( '----------------------------------------------------------------') tweetCriteria = gotmanager.TweetCriteria() tweetCriteria.setUsername('@' + name.strip('@')).setSince( start_date).setUntil(end_date).setMaxTweets(max_num) tweetCriteria.include_retweets = include_retweets tweetCriteria.include_replies = include_replies tweets = gotmanager.TweetManager.getTweets(tweetCriteria) print(len(tweets)) sql = MySqlUtils() tweet_dicts = [] for tweet in tweets: #print (tweet) print(tweet.__dict__) tweet_dict = get_cleaned_tweet_dictionary(tweet) try: tweet_dicts.append(tweet_dict) sql.dict_to_sql(tweet_dict, 'tweet') except UnicodeEncodeError as e: print(tweet_dict) raise e data = pd.DataFrame(tweet_dicts)