def create_feature(self): if is_test_or_val_set(self.dataset_id): train_dataset_id = get_train_set_id_from_test_or_val_set(self.dataset_id) engager_id_feature = MappedFeatureEngagerId(train_dataset_id) engagement_feature = TweetFeatureEngagementIsReply(train_dataset_id) engager_id_df = engager_id_feature.load_or_create() engagement_df = engagement_feature.load_or_create() # Load the media column dataframe = pd.concat([ engager_id_df, engagement_df, ], axis=1 ) dataframe = dataframe[dataframe[engagement_feature.feature_name]] dataframe = pd.DataFrame({self.feature_name: dataframe.groupby(engager_id_feature.feature_name).size()}) dictionary = dataframe.to_dict()[self.feature_name] test_engager_id_feature = MappedFeatureEngagerId(self.dataset_id) test_engager_id_df = test_engager_id_feature.load_or_create() engagement_count_df = pd.DataFrame( test_engager_id_df[engager_id_feature.feature_name].map(lambda x: dictionary.get(x, 0))) self.save_feature(engagement_count_df) else: engager_id_feature = MappedFeatureEngagerId(self.dataset_id) engagement_feature = TweetFeatureEngagementIsReply(self.dataset_id) engager_id_df = engager_id_feature.load_or_create() engagement_df = engagement_feature.load_or_create() # Load the media column dataframe = pd.concat([ engager_id_df, engagement_df, ], axis=1 ) dataframe = dataframe[dataframe[engagement_feature.feature_name]] dataframe = pd.DataFrame({self.feature_name: dataframe.groupby(engager_id_feature.feature_name).size()}) dictionary = dataframe.to_dict()[self.feature_name] engagement_count_df = pd.DataFrame( engager_id_df[engager_id_feature.feature_name].map(lambda x: dictionary.get(x, 0))) self.save_feature(engagement_count_df)
def create_feature(self): # Check if the dataset id is train or test if is_test_or_val_set(self.dataset_id): train_dataset_id = get_train_set_id_from_test_or_val_set( self.dataset_id) test_dataset_id = self.dataset_id else: train_dataset_id = self.dataset_id test_dataset_id = get_test_or_val_set_id_from_train( train_dataset_id) # Load features creation_timestamps_feature = RawFeatureTweetTimestamp( train_dataset_id) engagers_feature = MappedFeatureEngagerId(train_dataset_id) # Save the column name eng_col = engagers_feature.feature_name dataframe = pd.concat([ creation_timestamps_feature.load_or_create(), engagers_feature.load_or_create(), ], axis=1) dataframe.sort_values(creation_timestamps_feature.feature_name, inplace=True) engager_counter_array = np.zeros( dataframe[engagers_feature.feature_name].max() + 1, dtype=int) result = pd.DataFrame([ find_and_increase(engager_id, engager_counter_array) for engager_id in dataframe[eng_col] ], index=dataframe.index) if not EngagerFeatureNumberOfPreviousEngagement( train_dataset_id).has_feature(): result.sort_index(inplace=True) EngagerFeatureNumberOfPreviousEngagement( train_dataset_id).save_feature(result) if not EngagerFeatureNumberOfPreviousEngagement( test_dataset_id).has_feature(): # Load features creation_timestamps_feature = RawFeatureTweetTimestamp( test_dataset_id) engagers_feature = MappedFeatureEngagerId(test_dataset_id) dataframe = pd.concat([ creation_timestamps_feature.load_or_create(), engagers_feature.load_or_create(), ], axis=1) if dataframe[engagers_feature.feature_name].max( ) + 1 > engager_counter_array.size: engager_counter_array = np.pad( engager_counter_array, pad_width=(0, dataframe[engagers_feature.feature_name].max() + 1 - engager_counter_array.size), mode='constant', constant_values=0) result = pd.DataFrame( dataframe[eng_col].map(lambda x: engager_counter_array[x]), index=dataframe.index) EngagerFeatureNumberOfPreviousEngagement( test_dataset_id).save_feature(result)
def create_feature(self): # Check if the dataset id is train or test if is_test_or_val_set(self.dataset_id): train_dataset_id = get_train_set_id_from_test_or_val_set( self.dataset_id) test_dataset_id = self.dataset_id else: train_dataset_id = self.dataset_id test_dataset_id = get_test_or_val_set_id_from_train( train_dataset_id) # Load features creation_timestamps_feature = RawFeatureTweetTimestamp( train_dataset_id) engagers_feature = MappedFeatureEngagerId(train_dataset_id) creators_feature = MappedFeatureCreatorId(train_dataset_id) language_feature = MappedFeatureTweetLanguage(train_dataset_id) engagement_feature = TweetFeatureEngagementIsNegative(train_dataset_id) dataframe = pd.concat([ creation_timestamps_feature.load_or_create(), engagers_feature.load_or_create(), engagement_feature.load_or_create(), creators_feature.load_or_create(), language_feature.load_or_create() ], axis=1) dataframe.sort_values(creation_timestamps_feature.feature_name, inplace=True) # KEY: a tuple (creator, engager) # VALUE: the number of time the engager has engaged with the creator # If key does not exists -> 0 times. engagement_dict = {} result = pd.DataFrame([ find_and_increase_engager(eng_id, cre_id, lang, engagement_dict) if engagement else engagement_dict.get((eng_id, lang), 0) for eng_id, cre_id, lang, engagement in zip( dataframe[engagers_feature.feature_name], dataframe[ creators_feature.feature_name], dataframe[ language_feature.feature_name], dataframe[ engagement_feature.feature_name]) ], index=dataframe.index) if not EngagerFeatureNumberOfPreviousNegativeEngagementWithLanguage( train_dataset_id).has_feature(): result.sort_index(inplace=True) EngagerFeatureNumberOfPreviousNegativeEngagementWithLanguage( train_dataset_id).save_feature(result) if not EngagerFeatureNumberOfPreviousNegativeEngagementWithLanguage( test_dataset_id).has_feature(): # Load features creation_timestamps_feature = RawFeatureTweetTimestamp( test_dataset_id) engagers_feature = MappedFeatureEngagerId(test_dataset_id) language_feature = MappedFeatureTweetLanguage(test_dataset_id) creators_feature = MappedFeatureCreatorId(test_dataset_id) dataframe = pd.concat([ creation_timestamps_feature.load_or_create(), engagers_feature.load_or_create(), creators_feature.load_or_create(), language_feature.load_or_create() ], axis=1) dataframe.sort_values(creation_timestamps_feature.feature_name, inplace=True) result = pd.DataFrame([ find_and_increase_creator(eng_id, cre_id, lang, engagement_dict) for eng_id, cre_id, lang in zip( dataframe[engagers_feature.feature_name], dataframe[ creators_feature.feature_name], dataframe[ language_feature.feature_name]) ], index=dataframe.index) result.sort_index(inplace=True) EngagerFeatureNumberOfPreviousNegativeEngagementWithLanguage( test_dataset_id).save_feature(result)
def create_feature(self): # Check if the dataset id is train or test if is_test_or_val_set(self.dataset_id): train_dataset_id = get_train_set_id_from_test_or_val_set( self.dataset_id) test_dataset_id = self.dataset_id else: train_dataset_id = self.dataset_id test_dataset_id = get_test_or_val_set_id_from_train( train_dataset_id) # Load features creation_timestamps_feature = RawFeatureTweetTimestamp( train_dataset_id) creators_feature = MappedFeatureCreatorId(train_dataset_id) engagers_feature = MappedFeatureEngagerId(train_dataset_id) language_feature = MappedFeatureGroupedTweetLanguage(train_dataset_id) engagement_feature = TweetFeatureEngagementIsPositive(train_dataset_id) dataframe = pd.concat([ creation_timestamps_feature.load_or_create(), creators_feature.load_or_create(), engagers_feature.load_or_create(), language_feature.load_or_create(), engagement_feature.load_or_create() ], axis=1) dataframe.sort_values(creation_timestamps_feature.feature_name, inplace=True) engager_counter_array = np.zeros( (data.DataStats.get_max_user_id() + 1, 70), dtype=np.uint16) result = pd.DataFrame([ find_and_increase_engager(engager_id, creator_id, language, engagement, engager_counter_array) for engager_id, creator_id, language, engagement in zip( dataframe[engagers_feature.feature_name], dataframe[ creators_feature.feature_name], dataframe[ language_feature.feature_name], dataframe[ engagement_feature.feature_name]) ], index=dataframe.index) if not EngagerMainGroupedLanguage(train_dataset_id).has_feature(): result.sort_index(inplace=True) EngagerMainGroupedLanguage(train_dataset_id).save_feature(result) if not EngagerMainGroupedLanguage(test_dataset_id).has_feature(): # Load features creation_timestamps_feature = RawFeatureTweetTimestamp( test_dataset_id) creators_feature = MappedFeatureCreatorId(test_dataset_id) engagers_feature = MappedFeatureEngagerId(test_dataset_id) language_feature = MappedFeatureGroupedTweetLanguage( test_dataset_id) dataframe = pd.concat([ creation_timestamps_feature.load_or_create(), creators_feature.load_or_create(), engagers_feature.load_or_create(), language_feature.load_or_create() ], axis=1) dataframe.sort_values(creation_timestamps_feature.feature_name, inplace=True) result = pd.DataFrame([ find_and_increase_engager(engager_id, creator_id, language, False, engager_counter_array) for engager_id, creator_id, language in zip( dataframe[engagers_feature.feature_name], dataframe[ creators_feature.feature_name], dataframe[ language_feature.feature_name]) ], index=dataframe.index) result.sort_index(inplace=True) EngagerMainGroupedLanguage(test_dataset_id).save_feature(result)
def create_feature(self): # Check if the dataset id is train or test if is_test_or_val_set(self.dataset_id): train_dataset_id = get_train_set_id_from_test_or_val_set( self.dataset_id) test_dataset_id = self.dataset_id else: train_dataset_id = self.dataset_id test_dataset_id = get_test_or_val_set_id_from_train( train_dataset_id) start_time = time.time() # Load features creation_timestamps_feature = RawFeatureTweetTimestamp( train_dataset_id) creator_id_feature = MappedFeatureCreatorId(train_dataset_id) engager_id_feature = MappedFeatureEngagerId(train_dataset_id) engagement_feature = self._get_engagement_feature(train_dataset_id) # save column names creator_id_col = creator_id_feature.feature_name engager_id_col = engager_id_feature.feature_name engagement_col = engagement_feature.feature_name dataframe = pd.concat([ creation_timestamps_feature.load_or_create(), creator_id_feature.load_or_create(), engager_id_feature.load_or_create(), engagement_feature.load_or_create() ], axis=1) dataframe.sort_values(creation_timestamps_feature.feature_name, inplace=True) max_id = max(dataframe[creator_id_col].max(), dataframe[engager_id_col].max()) counter_array = np.zeros(max_id + 1, dtype=int) result = pd.DataFrame([ find_and_increase(creator_id=creator_id, engager_id=engager_id, counter_array=counter_array) if engagement else counter_array[creator_id] for creator_id, engager_id, engagement in zip( dataframe[creator_id_col], dataframe[engager_id_col], dataframe[engagement_col]) ], index=dataframe.index) self._save_train_result_if_not_present(result, train_dataset_id) if not self._exists_test_feature(test_dataset_id): # Load features # Load features creation_timestamps_feature = RawFeatureTweetTimestamp( test_dataset_id) creator_id_feature = MappedFeatureCreatorId(test_dataset_id) engager_id_feature = MappedFeatureEngagerId(test_dataset_id) # save column names creator_id_col = creator_id_feature.feature_name engager_id_col = engager_id_feature.feature_name dataframe = pd.concat([ creator_id_feature.load_or_create(), engager_id_feature.load_or_create(), creation_timestamps_feature.load_or_create(), ], axis=1) dataframe.sort_values(creation_timestamps_feature.feature_name, inplace=True) max_id = max(dataframe[creator_id_col].max(), dataframe[engager_id_col].max()) if max_id + 1 > counter_array.size: counter_array = np.pad(counter_array, pad_width=(0, max_id + 1 - counter_array.size), mode='constant', constant_values=0) result = pd.DataFrame( dataframe[creator_id_col].map(lambda x: counter_array[x]), index=dataframe.index) result.sort_index(inplace=True) print("time:") print(time.time() - start_time) self._save_test_result(result, test_dataset_id)