log_behavior = ['LogFriends','LogProfilePhotos','LogCoverPhotos','LogPosts','LogPostTextLengthMedian', 'LogSharedNewsSum','LogUploadVideo','LogUploadPhoto','LogInteraction', 'GenderCode']
pi = ['Arts_and_Entertainment', 'Business_and_Services', 'Community_and_Organizations', 'Education_y', 'Health_and_Beauty', 'Home_Foods_and_Drinks', 'Law_Politic_and_Government',
'News_and_Media', 'Pets_And_Animals', 'Recreation_and_Sports', 'Regional', 'Religion_and_Spirituality', 'Restaurants_and_Bars', 'Technology_Computers_and_Internet', 'Transportation_Travel_and_Tourism']
# obtain NoPosts, SharedNewsSum, UploadVideoSum
df_1 = ld.aggr_feature_user(ds_file)
aggr_df = pd.merge(aggr_df, df_1, how='inner', left_on='UserID', right_on='UserID')
# obtain about
df_2 = ld.about_dataset(ds_file)
aggr_df = pd.merge(aggr_df, df_2, how='inner', left_on='UserID', right_on='UserID')
# UserID, NoProfilePhotos, NoCoverPhotos, NoUploadedPhotos, NoPhotos
df_3 = ld.photo_dataset(photo_file)
aggr_df = pd.merge(df_3, aggr_df, how='inner', left_on='UserID', right_on='UserID')
# NumOfFriends
df_4 = ld.friendsnum_dataset(friendsnum_file)
aggr_df = pd.merge(df_4, aggr_df, how='inner', left_on='UserID', right_on='UserID')
# separate golden standard column name
target_dummies = pd.get_dummies(aggr_df['ActiveInterests'])
aggr_df = aggr_df.join(target_dummies)
target = sorted( list( set( target_dummies.columns.tolist() ) - set( ['Random'] ) ) )
# columns = aggr_df.columns.tolist()
zone_columns = [ x for x in dict_aggr_mean ]
x_df = robust_modified(aggr_df[zone_columns])
def dataset(type, polarity_zone, subjectivity_zone):
if type == 'old':
# Load the text dataset, drop null non-text, assign UserId
all_df = ld.all_post()
all_df = all_df[['postId', 'LikesCount', 'SharesCount', 'CommentsCount', 'PostTextLength', 'PostTextPolarity', 'PostTextSubjectivity', 'PostTime']].dropna()
all_df['UserId'] = all_df['postId'].str.split('_').str.get(0).astype(int)
all_df['zone'] = 'a'
all_df = threshold_zones(all_df, polarity_zone, subjectivity_zone)
zone_dummies = pd.get_dummies(all_df['zone'])
column_zone_dummies = zone_dummies.columns.tolist()
ent_ = [x + '_ratio' for x in column_zone_dummies]
ent_dummies = pd.get_dummies(all_df['entropy_all'])
all_df = all_df.join(ent_dummies)
all_df = all_df.join(zone_dummies)
dict_aggr = {x:np.sum for x in column_zone_dummies}
dict_aggr.update({x: np.mean for x in ent_})
aggr_df = all_df.groupby(['UserId'], sort=True).agg(dict_aggr).reset_index()
# add day part
day_part_df = ld.func_day_part(all_df)
aggr_df = pd.merge(aggr_df, day_part_df, how='inner', left_on='UserId', right_on='UserId')
# Load golden standard file
gs_df = pd.read_csv('data/userlevel_all_features_1007.csv', header=0)
# Merge zone file and golden standard
aggr_df = pd.merge(aggr_df, gs_df, how='inner', left_on='UserId', right_on='UserNum')
# aggr_df = aggr_df[['UserId', 'ActiveInterests']+column_zone_dummies+ent_]
# harusnya digabung dulu, baru activeinterests dibikin dummies
elif type == 'new':
# Load the new dataset, drop null non-text, assign UserId
ds_file_array = ['data/english_foodgroup_new.json', 'data/english_TEDtranslate_new.json',
'data/english_traveladdiction_new.json']
photo_file = ['data/album_english_foodgroups.json', 'data/album_english_TEDtranslate.json',
'data/album_english_traveladdiction.json']
friendsnum_file = ['data/english_foodgroups_friendsnum.json', 'data/english_TEDtranslate_friendsnum.json',
'data/english_traveladdiction_friendsnum.json']
all_df = ld.new_dataset(ds_file_array)
# print all_df.dtypes
all_df = all_df[['UserID', 'LikesCount', 'SharesCount', 'CommentsCount', 'PostTextLength', 'PostTextPolarity','PostTextSubjectivity', 'ActiveInterests', 'PostTime']].dropna()
all_df['UserId'] = all_df['UserID']
all_df['zone'] = 'a'
all_df = threshold_zones(all_df, polarity_zone, subjectivity_zone)
zone_dummies = pd.get_dummies(all_df['zone'])
column_zone_dummies = zone_dummies.columns.tolist()
ent_ = [x + '_ratio' for x in column_zone_dummies]
ent_dummies = pd.get_dummies(all_df['entropy_all'])
all_df = all_df.join(ent_dummies)
all_df = all_df.join(zone_dummies)
dict_aggr = {x: np.sum for x in column_zone_dummies}
dict_aggr.update({'ActiveInterests': np.min})
dict_aggr.update({x: np.mean for x in ent_})
aggr_df = all_df.groupby(['UserId'], sort=True).agg(dict_aggr).reset_index()
# obtain NoPosts, SharedNewsSum, UploadVideoSum
df_1 = ld.aggr_feature_user(ds_file_array)
aggr_df = pd.merge(aggr_df, df_1, how='inner', left_on='UserId', right_on='UserID')
# obtain about
df_2 = ld.about_dataset(ds_file_array)
aggr_df = pd.merge(aggr_df, df_2, how='inner', left_on='UserId', right_on='UserID')
# UserID, NoProfilePhotos, NoCoverPhotos, NoUploadedPhotos, NoPhotos
df_3 = ld.photo_dataset(photo_file)
aggr_df = pd.merge(aggr_df, df_3, how='inner', left_on='UserId', right_on='UserID')
# NumOfFriends
df_4 = ld.friendsnum_dataset(friendsnum_file)
aggr_df = pd.merge(aggr_df, df_4, how='inner', left_on='UserId', right_on='UserID')
# day_part
df_5 = ld.func_day_part(all_df)
aggr_df = pd.merge(aggr_df, df_5, how='inner', left_on='UserId', right_on='UserId')
aggr_df.drop(['userId', 'UserID_y', 'UserID_x'], axis=1, inplace=True)
# print 'data baru kolom', aggr_df.dtypes
aggr_df['frequent_day_part'] = aggr_df['frequent_day_part'].map(
{'Early_Morning': 0, 'Morning': 1, 'Afternoon': 2, 'Evening': 3})
# aggr_df = all_df.groupby(['UserId'], sort=True).agg(dict_aggr).reset_index()
# add entropy features
aggr_df = entropy_features_(aggr_df, ent_)
return aggr_df, column_zone_dummies #, target