def create_hashtags_table(connection, tweet, table_name, schema=None, key='id', flavor='mysql'):
try:
hashtags_df = pd.DataFrame()
hashtags_df['text'] = [i['text'] for i in tweet['entities']['hashtags']]
hashtags_df['tweet_id'] = [tweet['id']]
print hashtags_df.head()
auth_utils.upsert_data(connection, hashtags_df, table_name, schema, key, flavor)
except:
pass
def create_tweets_table(connection, tweet, table_name, schema=None, key='id', flavor='mysql'):
tweet_df = pd.DataFrame()
tweet_df['id'] = [tweet['id']]
tweet_df['text'] = [tweet['text'].encode('utf8mb4')]
tweet_df['user'] = [tweet['user']['id']]
tweet_df['favorite_count'] = [tweet['favorite_count']]
tweet_df['retweet_count'] = [tweet['retweet_count']]
tweet_df['created_at'] = [tweet['created_at']]
tweet_df['location'] = [tweet['coordinates']]
#tweet_df['quoted_tweet'] = [tweet.quoted_status_id]
tweet_df['in_response_to'] = [tweet['in_reply_to_screen_name']]
tweet_df['language'] = [tweet['lang']]
print tweet_df.head()
auth_utils.upsert_data(connection, tweet_df, table_name, schema, key, flavor)
def create_users_table(connection, tweet, table_name, schema=None, key='id', flavor='mysql'):
user_df = pd.DataFrame()
user_df['id'] = [tweet['user']['id']]
user_df['screen_name'] = [tweet['user']['screen_name']]
user_df['name'] = [tweet['user']['name']]
user_df['tweet_count'] = [tweet['user']['statuses_count']]
user_df['followers_count'] = [tweet['user']['followers_count']]
user_df['following_count'] = [tweet['user']['friends_count']]
user_df['favorite_count'] = [tweet['user']['favourites_count']]
user_df['location'] = [tweet['user']['location']]
user_df['time_zone'] = [tweet['user']['time_zone']]
user_df['member_since'] = [tweet['user']['created_at']]
print user_df.head()
auth_utils.upsert_data(connection, user_df, table_name, schema, key, flavor)
