def run_store_profile_info(self, names, query_id):
sql = MySqlUtils()
non_processed_handles = names # list
responses_iterator = self.yield_profile_requests(non_processed_handles)
for responses in responses_iterator:
for response in responses:
if response and response.status_code == 200:
twitter_user_dictionary = self.get_profile_dictionary(
response, query_id)
try:
sql.dict_to_sql(twitter_user_dictionary, 'user')
except Exception as e:
print(e)
print(twitter_user_dictionary)
def got_query(query, start_date, end_date, max_num):
# Wrapper for get old tweets
#:param start_date:
#:param end_date:
#:return:
print("{} {} {} {}".format(query, start_date, end_date, max_num))
tweetCriteria = gotmanager.TweetCriteria().setQuerySearch(query).setSince(
start_date).setUntil(end_date).setMaxTweets(max_num)
tweets = gotmanager.TweetManager.getTweets(tweetCriteria)
sql = MySqlUtils()
for tweet in tweets:
tweet.query = query
# print(tweet.__dict__)
sql.dict_to_sql(get_cleaned_tweet_dictionary(tweet), 'tweet')
def got_username(user_handle=None,
start_date=None,
end_date=None,
max_num=None,
include_retweets=None,
include_replies=False):
if start_date is None:
start_date = "2017-01-01"
if end_date is None:
end_date = "2018-01-01"
#usernames=joblib.load('/home/analytics/analytics/data/username.txt')
print(start_date)
for name in user_handle:
print(name)
print(
'----------------------------------------------------------------')
tweetCriteria = gotmanager.TweetCriteria()
tweetCriteria.setUsername('@' + name.strip('@')).setSince(
start_date).setUntil(end_date).setMaxTweets(max_num)
tweetCriteria.include_retweets = include_retweets
tweetCriteria.include_replies = include_replies
tweets = gotmanager.TweetManager.getTweets(tweetCriteria)
print(len(tweets))
sql = MySqlUtils()
tweet_dicts = []
for tweet in tweets:
#print (tweet)
print(tweet.__dict__)
tweet_dict = get_cleaned_tweet_dictionary(tweet)
try:
tweet_dicts.append(tweet_dict)
sql.dict_to_sql(tweet_dict, 'tweet')
except UnicodeEncodeError as e:
print(tweet_dict)
raise e
data = pd.DataFrame(tweet_dicts)