# features 'Followers' behavior = ['Friends','NoPhotos','NoProfilePhotos','NoCoverPhotos','NoPosts','PostTextLengthMedian','SharedNewsSum','UploadVideoSum','UploadPhotoSum','Interaction', 'GenderCode'] log_behavior = ['LogFriends','LogProfilePhotos','LogCoverPhotos','LogPosts','LogPostTextLengthMedian', 'LogSharedNewsSum','LogUploadVideo','LogUploadPhoto','LogInteraction', 'GenderCode'] pi = ['Arts_and_Entertainment', 'Business_and_Services', 'Community_and_Organizations', 'Education_y', 'Health_and_Beauty', 'Home_Foods_and_Drinks', 'Law_Politic_and_Government', 'News_and_Media', 'Pets_And_Animals', 'Recreation_and_Sports', 'Regional', 'Religion_and_Spirituality', 'Restaurants_and_Bars', 'Technology_Computers_and_Internet', 'Transportation_Travel_and_Tourism'] # obtain NoPosts, SharedNewsSum, UploadVideoSum df_1 = ld.aggr_feature_user(ds_file) aggr_df = pd.merge(aggr_df, df_1, how='inner', left_on='UserID', right_on='UserID') # obtain about df_2 = ld.about_dataset(ds_file) aggr_df = pd.merge(aggr_df, df_2, how='inner', left_on='UserID', right_on='UserID') # UserID, NoProfilePhotos, NoCoverPhotos, NoUploadedPhotos, NoPhotos df_3 = ld.photo_dataset(photo_file) aggr_df = pd.merge(df_3, aggr_df, how='inner', left_on='UserID', right_on='UserID') # NumOfFriends df_4 = ld.friendsnum_dataset(friendsnum_file) aggr_df = pd.merge(df_4, aggr_df, how='inner', left_on='UserID', right_on='UserID') # separate golden standard column name target_dummies = pd.get_dummies(aggr_df['ActiveInterests']) aggr_df = aggr_df.join(target_dummies) target = sorted( list( set( target_dummies.columns.tolist() ) - set( ['Random'] ) ) )
def dataset(type, polarity_zone, subjectivity_zone): if type == 'old': # Load the text dataset, drop null non-text, assign UserId all_df = ld.all_post() all_df = all_df[['postId', 'LikesCount', 'SharesCount', 'CommentsCount', 'PostTextLength', 'PostTextPolarity', 'PostTextSubjectivity', 'PostTime']].dropna() all_df['UserId'] = all_df['postId'].str.split('_').str.get(0).astype(int) all_df['zone'] = 'a' all_df = threshold_zones(all_df, polarity_zone, subjectivity_zone) zone_dummies = pd.get_dummies(all_df['zone']) column_zone_dummies = zone_dummies.columns.tolist() ent_ = [x + '_ratio' for x in column_zone_dummies] ent_dummies = pd.get_dummies(all_df['entropy_all']) all_df = all_df.join(ent_dummies) all_df = all_df.join(zone_dummies) dict_aggr = {x:np.sum for x in column_zone_dummies} dict_aggr.update({x: np.mean for x in ent_}) aggr_df = all_df.groupby(['UserId'], sort=True).agg(dict_aggr).reset_index() # add day part day_part_df = ld.func_day_part(all_df) aggr_df = pd.merge(aggr_df, day_part_df, how='inner', left_on='UserId', right_on='UserId') # Load golden standard file gs_df = pd.read_csv('data/userlevel_all_features_1007.csv', header=0) # Merge zone file and golden standard aggr_df = pd.merge(aggr_df, gs_df, how='inner', left_on='UserId', right_on='UserNum') # aggr_df = aggr_df[['UserId', 'ActiveInterests']+column_zone_dummies+ent_] # harusnya digabung dulu, baru activeinterests dibikin dummies elif type == 'new': # Load the new dataset, drop null non-text, assign UserId ds_file_array = ['data/english_foodgroup_new.json', 'data/english_TEDtranslate_new.json', 'data/english_traveladdiction_new.json'] photo_file = ['data/album_english_foodgroups.json', 'data/album_english_TEDtranslate.json', 'data/album_english_traveladdiction.json'] friendsnum_file = ['data/english_foodgroups_friendsnum.json', 'data/english_TEDtranslate_friendsnum.json', 'data/english_traveladdiction_friendsnum.json'] all_df = ld.new_dataset(ds_file_array) # print all_df.dtypes all_df = all_df[['UserID', 'LikesCount', 'SharesCount', 'CommentsCount', 'PostTextLength', 'PostTextPolarity','PostTextSubjectivity', 'ActiveInterests', 'PostTime']].dropna() all_df['UserId'] = all_df['UserID'] all_df['zone'] = 'a' all_df = threshold_zones(all_df, polarity_zone, subjectivity_zone) zone_dummies = pd.get_dummies(all_df['zone']) column_zone_dummies = zone_dummies.columns.tolist() ent_ = [x + '_ratio' for x in column_zone_dummies] ent_dummies = pd.get_dummies(all_df['entropy_all']) all_df = all_df.join(ent_dummies) all_df = all_df.join(zone_dummies) dict_aggr = {x: np.sum for x in column_zone_dummies} dict_aggr.update({'ActiveInterests': np.min}) dict_aggr.update({x: np.mean for x in ent_}) aggr_df = all_df.groupby(['UserId'], sort=True).agg(dict_aggr).reset_index() # obtain NoPosts, SharedNewsSum, UploadVideoSum df_1 = ld.aggr_feature_user(ds_file_array) aggr_df = pd.merge(aggr_df, df_1, how='inner', left_on='UserId', right_on='UserID') # obtain about df_2 = ld.about_dataset(ds_file_array) aggr_df = pd.merge(aggr_df, df_2, how='inner', left_on='UserId', right_on='UserID') # UserID, NoProfilePhotos, NoCoverPhotos, NoUploadedPhotos, NoPhotos df_3 = ld.photo_dataset(photo_file) aggr_df = pd.merge(aggr_df, df_3, how='inner', left_on='UserId', right_on='UserID') # NumOfFriends df_4 = ld.friendsnum_dataset(friendsnum_file) aggr_df = pd.merge(aggr_df, df_4, how='inner', left_on='UserId', right_on='UserID') # day_part df_5 = ld.func_day_part(all_df) aggr_df = pd.merge(aggr_df, df_5, how='inner', left_on='UserId', right_on='UserId') aggr_df.drop(['userId', 'UserID_y', 'UserID_x'], axis=1, inplace=True) # print 'data baru kolom', aggr_df.dtypes aggr_df['frequent_day_part'] = aggr_df['frequent_day_part'].map( {'Early_Morning': 0, 'Morning': 1, 'Afternoon': 2, 'Evening': 3}) # aggr_df = all_df.groupby(['UserId'], sort=True).agg(dict_aggr).reset_index() # add entropy features aggr_df = entropy_features_(aggr_df, ent_) return aggr_df, column_zone_dummies #, target