def main(): base_directory = os.path.abspath(os.curdir) data_directory = os.path.join( os.path.abspath(os.path.join(base_directory, os.pardir)), 'results') exp_directory = 'one_player' is_backward = False sample_participants = [True, 60] if is_backward: backward_appen = 'backward' else: backward_appen = 'grid' # dict for the directories file names. dir_name: [file name, if not_agent_data(2 players exp), number_of_trials, # known_unknown_exp] directory_files_dict = { 'agent_mturk': ['all_data_after_4_1_and_data_to_use_before.xlsx', False, 50, False], 'one_player': ['first_results.xlsx', False, 50, False], 'agent_no_payment': ['agent_no_payment.xlsx', False, 50, False], 'all': ['results_payments_status.csv', True, 50, False], 'second_agent_exp': ['mturk_ag_second_exp.xlsx', False, 60, True], 'all_agents': ['concat_data_27_02.xlsx', False, 50, False], } not_agent_data = directory_files_dict[exp_directory][1] num_trials = directory_files_dict[exp_directory][2] known_unknown_exp = directory_files_dict[exp_directory][3] appendix = 'group' if not_agent_data else 'player' # create folder for experiment exp_directory_path = utils.set_folder(exp_directory, 'classifier_results') experiment_path = utils.set_folder( datetime.now().strftime(backward_appen + '_%d_%m_%Y_%H_%M'), exp_directory_path) file_name = datetime.now().strftime('LogFile_temporary_role_effect.log') # set up logging to file - see previous section for more details logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%d/%m/%Y %I:%M:%S', filename=os.path.join(experiment_path, file_name), filemode='w') # define a Handler which writes INFO messages or higher to the sys.stderr console = logging.StreamHandler() console.setLevel(logging.INFO) # set a format which is simpler for console use formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(console) ################################################################################################## folds = 5 # if folds = 1, run train-test, # else - run cross-validation with relevant number of folds """The base features are the all features we will use that are in the original data, and the add_feature_list are the features we want to add each time. Use receiver_timeout and lottery_result only on time t'<t and only in temporal analysis""" orign_data_features = [ appendix + '_lottery_result', appendix + '_receiver_timeout', appendix + '_x_lottery', appendix + '_y_lottery', appendix + '_p_lottery', 'subsession_round_number', 'player_age', 'player_is_student', 'payed', 'participant_code' ] columns_not_features = ['participant_code' ] # 'player_expert_type', 'player_id_in_group'] add_feature_list = [ # [], # [appendix + '_sender_answer'], # [appendix + '_receiver_choice'], [appendix + '_receiver_choice', appendix + '_sender_answer'] ] all_add_features_lists = list( set(itertools.chain.from_iterable(add_feature_list))) """The label we want to predict""" label_dict = { appendix + '_receiver_choice': ['DM chose SQ', 'DM chose gamble'], } # data personal_features = [ 'player_age', 'player_is_student', 'payed', 'male', 'female' ] # two_players_features if not_agent_data: orign_data_features += [appendix + '_sender_timeout'] label_dict[appendix + '_sender_answer'] = ['regression results'] # columns_not_features += ['pair_id'] """The size of window to take as the features of sample at time t, 0- for no temporal classifier""" window_size_list = [5, 1, 10] model_dict = {appendix + '_receiver_choice': # classification models: [LearnMajorityLastTrials(Perceptron()), LearnMajorityLastTrials(XGBClassifier()), XGBClassifier(max_depth=5), MajorityLastTrials(), RandomForestClassifier(), MajorityPerProblem()], appendix + '_sender_answer': # regression models [RandomForestRegressor(), Perceptron(), SGDRegressor(), PassiveAggressiveRegressor(), SVR(), XGBRegressor()] } # model_dict = {appendix + '_receiver_choice': # classification models: # [MajorityRule()], # appendix + '_sender_answer': # regression models # [MajorityRule()] # } data_file_path = os.path.join(data_directory, exp_directory, directory_files_dict[exp_directory][0]) ################################################################################################## # load data and create features create_data_obj = create_data.CreateData() create_data_obj.load_data(data_file_path, not_agent_data=not_agent_data) create_data_obj.create_features_label( list(label_dict.keys()), orign_data_features, all_add_features_lists, appendix, sample_participants=sample_participants) # evaluate_backward_search if is_backward: backward_add_features_list = [ feature for feature in create_data_obj.base_features + all_add_features_lists ] backward_add_features_list.append(None) candidates_features = copy.deepcopy(backward_add_features_list) candidates_features.remove('participant_code') execute_evaluate.evaluate_backward_search( data=create_data_obj.features_labels, base_features=backward_add_features_list, candidates_features=candidates_features, window_size_list=window_size_list, label_dict=label_dict, num_folds=folds, model_dict=model_dict, classifier_results_dir=experiment_path, appendix=appendix, personal_features=personal_features) # evaluate_grid_search else: # base_features = ['participant_code', 'subsession_round_number'] # add_feature_list = [feature for feature in create_data_obj.base_features + all_add_features_lists if # feature not in base_features] execute_evaluate.evaluate_grid_search( data=create_data_obj.features_labels, base_features=create_data_obj.base_features, add_feature_list=add_feature_list, window_size_list=window_size_list, label_dict=label_dict, num_folds=folds, model_dict=model_dict, classifier_results_dir=experiment_path, appendix=appendix, personal_features=personal_features) logging.info('Done!')
def train_valid_base_text_model(model_name): """ :param model_name: the full model name to use :return: """ token_indexer = {"tokens": ELMoTokenCharactersIndexer()} def tokenizer(x: str): return [ w.text for w in SpacyWordSplitter(language='en_core_web_sm', pos_tags=False).split_words(x) ] reader = TextExpDataSetReader(token_indexers=token_indexer, tokenizer=tokenizer, add_numeric_data=False) train_instances = reader.read(train_data_file_path) validation_instances = reader.read(validation_data_file_path) vocab = Vocabulary() # TODO: change this if necessary # batch_size should be: 10 or 9 depends on the input # and not shuffle so all the data of the same pair will be in the same batch iterator = BasicIterator( batch_size=batch_size) # , instances_per_epoch=10) # sorting_keys=[('sequence_review', 'list_num_tokens')]) iterator.index_with(vocab) options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/' \ 'elmo_2x1024_128_2048cnn_1xhighway_options.json' weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/' \ 'elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5' # TODO: check the output of this # elmo_embedder = Elmo(options_file, weight_file, num_output_representations=2) # word_embeddings = elmo_embedder elmo_embedder = ElmoTokenEmbedder(options_file, weight_file) word_embeddings = BasicTextFieldEmbedder({"tokens": elmo_embedder}) review_attention_layer = models.AttentionSoftMaxLayer( BilinearMatrixAttention(word_embeddings.get_output_dim(), word_embeddings.get_output_dim())) seq_attention_layer = models.AttentionSoftMaxLayer( DotProductMatrixAttention()) feed_forward = FeedForward(input_dim=batch_size, num_layers=2, hidden_dims=[batch_size, 1], activations=ReLU(), dropout=[0.2, 0.0]) fc_review_rep = FeedForward(input_dim=124, num_layers=1, hidden_dims=[10], activations=ReLU()) criterion = nn.MSELoss() metrics_dict = { 'mean_absolute_error': MeanAbsoluteError(), } model = models.BasicTextModel( word_embedding=word_embeddings, review_representation_layer=review_attention_layer, seq_representation_layer=seq_attention_layer, vocab=vocab, criterion=criterion, metrics_dict=metrics_dict, classifier_feedforward=feed_forward, fc_review_rep=fc_review_rep) optimizer = optim.Adam(model.parameters(), lr=0.1) num_epochs = 2 run_log_directory = utils.set_folder( datetime.now().strftime( f'{model_name}_{num_epochs}_epochs_%d_%m_%Y_%H_%M_%S'), 'logs') if not os.path.exists(run_log_directory): os.makedirs(run_log_directory) trainer = Trainer( model=model, optimizer=optimizer, iterator=iterator, train_dataset=train_instances, validation_dataset=validation_instances, num_epochs=num_epochs, shuffle=False, serialization_dir=run_log_directory, patience=10, histogram_interval=10, ) model_dict = trainer.train() print(f'{model_name}: evaluation measures are:') for key, value in model_dict.items(): print(f'{key}: {value}')
def execute_single_experiment(self, experiment_id: int, train_x: pd.DataFrame, train_y: pd.Series, test_x: pd.DataFrame, test_y: pd.Series, test_folds: pd.DataFrame=None): """ This function get train and test data and labels and execute the model :param experiment_id: running number :param train_x: train features :param train_y: train labels :param test_x: test features :param test_folds: test fold numbers (option - for cross validation) :param test_y: test labels :return: """ trained_models_directory_path = utils.set_folder('trained_models', father_folder_path=self.classifier_results_dir) logging.info('Starting model fit and save model') self.model_per_fold[experiment_id] = copy.deepcopy(self.model).fit(train_x, train_y) joblib.dump(self.model_per_fold[experiment_id], os.path.join( trained_models_directory_path, type(self.model).__name__ + str(experiment_id) + '.pkl')) logging.info('Starting model predict') # if hasattr(self.model_per_fold[experiment_id].__class__, 'predict_proba'): # predictions = self.model_per_fold[experiment_id].predict_proba(test_x) # else: predictions = self.model_per_fold[experiment_id].predict(test_x) if type(predictions) == Exception: return predictions test_y.name = 'label' train_y.name = 'label' if predictions.dtype == float: # regression- create bins to measure the F-score # for prediction keep_mask = predictions < 0.33 bin_prediction = np.where(predictions < 0.67, 1, 2) bin_prediction[keep_mask] = 0 bin_prediction = pd.Series(bin_prediction, name='bin_predictions', index=test_y.index) # for test_y keep_mask = test_y < 0.33 bin_test_y = np.where(test_y < 0.67, 1, 2) bin_test_y[keep_mask] = 0 bin_test_y = pd.Series(bin_test_y, name='bin_label', index=test_y.index) else: bin_prediction, bin_test_y = pd.Series(name='bin_prediction'), pd.Series(name='bin_label') if test_folds is not None: self.predictions_per_fold[experiment_id] =\ test_folds.join(test_y).join(pd.Series(predictions, name='predictions', index=test_y.index)).join( bin_test_y).join(bin_prediction) else: # for train_test self.predictions_per_fold[experiment_id] = \ pd.concat([test_y, pd.Series(predictions, name='predictions', index=test_y.index)], axis=1) # add correct column: self.predictions_per_fold[experiment_id]['correct'] = \ np.where(self.predictions_per_fold[experiment_id].predictions == self.predictions_per_fold[experiment_id].label, 1, 0) self.predictions_per_fold[experiment_id]['bin_correct'] = \ np.where(self.predictions_per_fold[experiment_id].bin_predictions == self.predictions_per_fold[experiment_id].bin_label, 1, 0) # create confusion matrix for classification if predictions.dtype == 'int': cnf_matrix = confusion_matrix(test_y, predictions) self.total_conf_mtx += cnf_matrix # else: # cnf_matrix = confusion_matrix(bin_test_y, bin_prediction) # self.total_conf_mtx += cnf_matrix return
def train_valid_transformer_text_decision_fix_text_features_model( model_name: str, all_data_file_name: str, features_max_size: int, func_batch_size: int = 9, predict_seq: bool = True, predict_avg_total_payoff: bool = True): """ This function train and validate model that use fix texts features only. It use LSTM model to predict the label of each round in the saifa given a saifa. The labels of this model is 0 for hotel and 1 for stay at home :param model_name: the full model name :param all_data_file_name: the name of the data file to use :param features_max_size: the max number of features per round :param func_batch_size: the batch size to use :param predict_seq: if we want to predict the seq :param predict_avg_total_payoff: if we want to predict the final average total payoff of the saifa :return: """ all_data_file_inner_path = os.path.join(data_directory, all_data_file_name) # split data to 5 folds num_folds = 5 num_epochs = 100 all_validation_accuracy = list() all_train_accuracy = list() HIDDEN_DIM = 100 run_log_directory = utils.set_folder( datetime.now().strftime( f'{model_name}_{num_epochs}_epochs_{num_folds}_folds_{HIDDEN_DIM}_hidden_dim_' f'%d_%m_%Y_%H_%M_%S'), 'logs') print(f'run_log_directory is: {run_log_directory}') all_seq_predictions = pd.DataFrame() all_reg_predictions = pd.DataFrame() if 'csv' in all_data_file_inner_path: data_df = pd.read_csv(all_data_file_inner_path) elif 'xlsx' in all_data_file_inner_path: data_df = pd.read_excel(all_data_file_inner_path) elif 'pkl' in all_data_file_inner_path: data_df = joblib.load(all_data_file_inner_path) else: print('Data format is not csv or csv or pkl') return folds = get_folds_per_participant(data=data_df, k_folds=num_folds, col_to_group='pair_id', col_to_group_in_df=True) if num_folds == 2: # train-test --> the fold to test is 1. num_folds = [1] for fold in range(num_folds): run_log_directory_fold = utils.set_folder(f'fold_{fold}', run_log_directory) train_pair_ids = folds.loc[folds.fold_number != fold].pair_id.tolist() test_pair_ids = folds.loc[folds.fold_number == fold].pair_id.tolist() # load train data train_reader = TransformerDatasetReader( pair_ids=train_pair_ids, features_max_size=features_max_size) test_reader = TransformerDatasetReader( pair_ids=test_pair_ids, features_max_size=features_max_size) train_instances = train_reader.read(all_data_file_inner_path) validation_instances = test_reader.read(all_data_file_inner_path) vocab = Vocabulary.from_instances(train_instances + validation_instances) # hotel_label_0 = True if vocab._index_to_token['labels'][0] == 'hotel' else False metrics_dict_seq = { 'Accuracy': CategoricalAccuracy(), # BooleanAccuracy(), # 'auc': Auc(), 'F1measure_hotel_label': F1Measure(positive_label=vocab._token_to_index['labels']['hotel']), 'F1measure_home_label': F1Measure( positive_label=vocab._token_to_index['labels']['stay_home']), } metrics_dict_reg = { 'mean_absolute_error': MeanAbsoluteError(), } # TODO: change this if necessary # batch_size should be: 10 or 9 depends on the input # and not shuffle so all the data of the same pair will be in the same batch iterator = BasicIterator( batch_size=func_batch_size) # , instances_per_epoch=10) iterator.index_with(vocab) model = models.TransformerFixTextFeaturesDecisionResultModel( vocab=vocab, metrics_dict_seq=metrics_dict_seq, metrics_dict_reg=metrics_dict_reg, predict_avg_total_payoff=predict_avg_total_payoff, linear_dim=None, batch_size=func_batch_size, input_dim=train_reader.input_dim, feedforward_hidden_dim=int(0.5 * train_reader.input_dim), ) print(model) if torch.cuda.is_available(): cuda_device = 0 model = model.cuda(cuda_device) else: cuda_device = -1 optimizer = optim.SGD(model.parameters(), lr=0.1) validation_metric = '+Accuracy' if predict_seq else '-loss' trainer = Trainer( model=model, optimizer=optimizer, iterator=iterator, train_dataset=train_instances, validation_dataset=validation_instances, num_epochs=num_epochs, shuffle=False, serialization_dir=run_log_directory_fold, patience=10, histogram_interval=10, cuda_device=cuda_device, validation_metric=validation_metric, ) model_dict = trainer.train() print(f'{model_name}: evaluation measures for fold {fold} are:') for key, value in model_dict.items(): print(f'{key}: {value}') if predict_seq: fold_seq_predictions = pd.DataFrame.from_dict( model.seq_predictions, orient='index') fold_seq_predictions = fold_seq_predictions.assign(fold=fold) fold_seq_predictions['final_prediction'] =\ fold_seq_predictions[f'predictions_{model_dict["training_epochs"]+1}'] fold_seq_predictions['final_total_payoff_prediction'] =\ fold_seq_predictions[f'total_payoff_prediction_{model_dict["training_epochs"]+1}'] all_seq_predictions = pd.concat( [all_seq_predictions, fold_seq_predictions], sort=True) calc_print_measures(fold_seq_predictions) all_validation_accuracy.append(model_dict['validation_Accuracy']) all_train_accuracy.append(model_dict['training_Accuracy']) if predict_avg_total_payoff: fold_reg_predictions = model.reg_predictions fold_reg_predictions = fold_reg_predictions.assign(fold=fold) fold_reg_predictions['final_total_payoff_prediction'] =\ fold_reg_predictions[f'prediction_{model_dict["training_epochs"]+1}'] all_reg_predictions = pd.concat( [all_reg_predictions, fold_reg_predictions], sort=True) # save the model predictions all_seq_predictions.to_csv( os.path.join(run_log_directory, 'seq_predictions.csv')) all_reg_predictions.to_csv( os.path.join(run_log_directory, 'reg_predictions.csv')) print(f'All folds measures:') calc_print_measures(all_seq_predictions) print( f'Train accuracy per round: {sum(all_train_accuracy)/len(all_train_accuracy)}, ' f'Validation accuracy per round: {sum(all_validation_accuracy)/len(all_validation_accuracy)}' ) results = calculate_measures_for_continues_labels( all_seq_predictions, 'final_total_payoff_prediction', 'total_payoff_label', label_options=[ 'total future payoff < 1/3', '1/3 < total future payoff < 2/3', 'total future payoff > 2/3' ]) results.to_csv(os.path.join(run_log_directory, 'results.csv'))
def train_valid_base_text_decision_fix_text_features_model( model_name: str, single_round_label: bool, use_only_prev_round: bool, train_data_file_name: str, validation_data_file_name: str, no_history: bool = False, func_batch_size: int = 9, numbers_columns: list = None, add_numeric_data: bool = True): """ This function train and validate model that use fix texts features only. :param: model_name: the full model name :param single_round_label: the label to use: single round of total payoff :param use_only_prev_round: if to use all the history or only the previous round :param train_data_file_name: the name of the train_data to use :param validation_data_file_name: the name of the validation_data to use :param no_history: if we don't want to use any history data :param func_batch_size: the batch size to use :param model_name: the name of the model we run :param numbers_columns: the names of the columns to use for the numeric data :param add_numeric_data: if we want to add numbers data :return: """ reader = TextExpDataSetReader(add_numeric_data=add_numeric_data, use_only_prev_round=use_only_prev_round, single_round_label=single_round_label, three_losses=True, fix_text_features=True, no_history=no_history, numbers_columns_name=numbers_columns) train_data_file_inner_path = os.path.join(data_directory, train_data_file_name) validation_data_file_inner_path = os.path.join(data_directory, validation_data_file_name) train_instances = reader.read(train_data_file_inner_path) validation_instances = reader.read(validation_data_file_inner_path) vocab = Vocabulary() # TODO: change this if necessary # batch_size should be: 10 or 9 depends on the input # and not shuffle so all the data of the same pair will be in the same batch iterator = BasicIterator( batch_size=func_batch_size) # , instances_per_epoch=10) # sorting_keys=[('sequence_review', 'list_num_tokens')]) iterator.index_with(vocab) # the shape of the flatten data rep if 'bert' in train_data_file_name: # fix features are BERT vector text_feedtorward = FeedForward(input_dim=reader.max_tokens_len, num_layers=2, hidden_dims=[300, 50], activations=ReLU(), dropout=[0.0, 0.0]) reader.max_tokens_len = 50 else: text_feedtorward = None feed_forward_input_dim = reader.max_seq_len * (reader.max_tokens_len + reader.number_length) feed_forward_classification = FeedForward(input_dim=feed_forward_input_dim, num_layers=1, hidden_dims=[2], activations=LeakyReLU(), dropout=[0.3]) criterion_classification = nn.BCEWithLogitsLoss() metrics_dict = { 'Accuracy': CategoricalAccuracy() # BooleanAccuracy(), # 'auc': Auc(), # 'F1measure': F1Measure(positive_label=1), } model = models.BasicFixTextFeaturesDecisionModel( vocab=vocab, classifier_feedforward_classification=feed_forward_classification, criterion_classification=criterion_classification, metrics_dict=metrics_dict, max_tokens_len=reader.max_tokens_len, text_feedforward=text_feedtorward, regularizer=RegularizerApplicator([("", L1Regularizer())]), ) optimizer = optim.Adam(model.parameters(), lr=0.1) num_epochs = 100 run_log_directory = utils.set_folder( datetime.now().strftime( f'{model_name}_{num_epochs}_epochs_%d_%m_%Y_%H_%M_%S'), 'logs') trainer = Trainer( model=model, optimizer=optimizer, iterator=iterator, train_dataset=train_instances, validation_dataset=validation_instances, num_epochs=num_epochs, shuffle=False, serialization_dir=run_log_directory, patience=10, histogram_interval=10, ) model_dict = trainer.train() print(f'{model_name}: evaluation measures are:') for key, value in model_dict.items(): if 'accuracy' in key: value = value * 100 print(f'{key}: {value}') # save the model predictions model.predictions.to_csv(os.path.join(run_log_directory, 'predictions.csv'))
def train_valid_base_text_decision_results_ep_model( model_name: str, single_round_label: bool, use_only_prev_round: bool, train_data_file_name: str, validation_data_file_name: str, no_history: bool = False): """ This function train and validate model that use texts and numbers. :param: model_name: the full model name :param single_round_label: the label to use: single round of total payoff :param use_only_prev_round: if to use all the history or only the previous round :param train_data_file_name: the name of the train_data to use :param validation_data_file_name: the name of the validation_data to use :param no_history: if we don't want to use any history data :return: """ token_indexer = {"tokens": ELMoTokenCharactersIndexer()} def tokenizer(x: str): return [ w.text for w in SpacyWordSplitter(language='en_core_web_sm', pos_tags=False).split_words(x) ] reader = TextExpDataSetReader(token_indexers=token_indexer, tokenizer=tokenizer, add_numeric_data=True, use_only_prev_round=use_only_prev_round, single_round_label=single_round_label, three_losses=True, no_history=no_history) train_data_file_inner_path = os.path.join(data_directory, train_data_file_name) validation_data_file_inner_path = os.path.join(data_directory, validation_data_file_name) train_instances = reader.read(train_data_file_inner_path) validation_instances = reader.read(validation_data_file_inner_path) vocab = Vocabulary() # TODO: change this if necessary # batch_size should be: 10 or 9 depends on the input # and not shuffle so all the data of the same pair will be in the same batch iterator = BasicIterator(batch_size=9) # , instances_per_epoch=10) # sorting_keys=[('sequence_review', 'list_num_tokens')]) iterator.index_with(vocab) options_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/' \ 'elmo_2x1024_128_2048cnn_1xhighway_options.json' weight_file = 'https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/' \ 'elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5' # TODO: check the output of this # elmo_embedder = Elmo(options_file, weight_file, num_output_representations=2) # word_embeddings = elmo_embedder elmo_embedder = ElmoTokenEmbedder(options_file, weight_file) word_embeddings = BasicTextFieldEmbedder({"tokens": elmo_embedder}) review_attention_layer =\ models.AttentionSoftMaxLayer(BilinearMatrixAttention(word_embeddings.get_output_dim(), word_embeddings.get_output_dim())) seq_attention_layer = models.AttentionSoftMaxLayer( DotProductMatrixAttention()) fc_review_rep_output_dim = reader.max_tokens_len fc_review_rep = FeedForward(input_dim=reader.max_tokens_len, num_layers=1, hidden_dims=[fc_review_rep_output_dim], activations=ReLU()) # seq_attention_layer = FeedForward(input_dim=) # numbers_lstm: Seq2VecEncoder = PytorchSeq2VecWrapper(nn.LSTM(2, 10, bidirectional=True, batch_first=True)) # the shape of the flatten data rep feed_forward_input_dim = reader.max_seq_len * (fc_review_rep_output_dim + reader.number_length) feed_forward_classification = FeedForward(input_dim=feed_forward_input_dim, num_layers=1, hidden_dims=[2], activations=ReLU(), dropout=[0.0]) feed_forward_regression = FeedForward(input_dim=feed_forward_input_dim, num_layers=1, hidden_dims=[1], activations=ReLU(), dropout=[0.0]) criterion_classification = nn.BCEWithLogitsLoss() criterion_regression = nn.MSELoss() metrics_dict = { "accuracy": CategoricalAccuracy(), # 'auc': Auc(), # 'F1measure': F1Measure(positive_label=1), } model = models.BasicTextDecisionResultModel( word_embedding=word_embeddings, review_representation_layer=review_attention_layer, seq_representation_layer=seq_attention_layer, vocab=vocab, classifier_feedforward_classification=feed_forward_classification, classifier_feedforward_regression=feed_forward_regression, fc_review_rep=fc_review_rep, criterion_classification=criterion_classification, criterion_regression=criterion_regression, metrics_dict=metrics_dict, add_numbers=True, max_tokens_len=reader.max_tokens_len, ) optimizer = optim.Adam(model.parameters(), lr=0.1) num_epochs = 2 run_log_directory = utils.set_folder( datetime.now().strftime( f'{model_name}_{num_epochs}_epochs_%d_%m_%Y_%H_%M_%S'), 'logs') trainer = Trainer( model=model, optimizer=optimizer, iterator=iterator, train_dataset=train_instances, validation_dataset=validation_instances, num_epochs=num_epochs, shuffle=False, serialization_dir=run_log_directory, patience=10, histogram_interval=10, ) model_dict = trainer.train() print(f'{model_name}: evaluation measures are:') for key, value in model_dict.items(): if 'accuracy' in key: value = value * 100 print(f'{key}: {value}') # save the model predictions model.predictions.to_csv(os.path.join(run_log_directory, 'predictions.csv'))
def train_test_simple_features_model( model_name: str, features_data_file_path: str, backward_search: bool = False, inner_data_directory: str = data_directory, label: str = 'label'): """ This function train and test some simple ML models that use text manual features to predict decisions :param features_data_file_path: hte path to the features file :param model_name: the full model name :param backward_search: use backward_search to find the best features :param inner_data_directory: the data directory to use :param label: the label to predict :return: """ experiment_path = utils.set_folder( datetime.now().strftime(f'{model_name}_%d_%m_%Y_%H_%M'), 'logs') file_name = datetime.now().strftime('LogFile.log') # set up logging to file - see previous section for more details logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', datefmt='%d/%m/%Y %I:%M:%S', filename=os.path.join(experiment_path, file_name), filemode='w') # define a Handler which writes INFO messages or higher to the sys.stderr console = logging.StreamHandler() console.setLevel(logging.INFO) # set a format which is simpler for console use formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(console) ################################################################################################## print( f'Start running train_test_simple_features_model on file: {features_data_file_path}' ) logging.info( f'Start running train_test_simple_features_model on file {features_data_file_path}' ) folds = 5 # if folds = 1, run train-test, else - run cross-validation with relevant number of folds # load the features data if 'pkl' in features_data_file_path: data = joblib.load( os.path.join(inner_data_directory, features_data_file_path)) else: data = pd.read_csv( os.path.join(inner_data_directory, features_data_file_path)) data.index = data.sample_id # The label we want to predict label_dict = { 'label': ['DM chose stay home', 'DM chose hotel'], 'future_total_payoff': ['DM chose stay home', 'DM chose hotel'], } data = data.drop(['k_size', 'sample_id'], axis=1) features = [ item for item in data.columns.tolist() if item not in [label, 'pair_id'] ] inner_batch_size = 9 if 'history' in features_data_file_path or 'prev' in features_data_file_path else 10 print(f'batch size is: {inner_batch_size}') candidates_features = copy.deepcopy(features) model_dict = {'label': # classification models: [[ EnsembleClassifier(XGBClassifier(max_depth=10), SVC(), LogisticRegression()), PredictLastDecision(), DummyClassifier(strategy='stratified'), DummyClassifier(strategy='most_frequent'), XGBClassifier(max_depth=10), DecisionTreeClassifier(), AdaBoostClassifier(n_estimators=100), LinearKerasModel(input_dim=len(features), batch_size=inner_batch_size), SVC(), LogisticRegression(), Perceptron(), RandomForestClassifier(), SVC(kernel='linear'), SGDClassifier(), PassiveAggressiveClassifier(), # ConvolutionalKerasModel(num_features=len([feature for feature in features if '_1' in feature]), # input_len=len(features)) ], 'classification'], 'future_total_payoff': # regression models [[ RandomForestRegressor(), SGDRegressor(), PassiveAggressiveRegressor(), SVR(), SVR(kernel='linear'), EnsembleClassifier(XGBRegressor(max_depth=10), SVR(), RandomForestRegressor()), DecisionTreeRegressor(), XGBRegressor(max_depth=10), DummyRegressor(strategy='median'), DummyRegressor(strategy='mean')], 'regression'], } model_dict_to_use = {label: model_dict[label][0]} label_dict_to_use = {label: label_dict[label]} if backward_search: execute_evaluate.evaluate_backward_search( data=data, base_features=features, window_size_list=[0], label_dict=label_dict_to_use, num_folds=folds, model_dict=model_dict_to_use, classifier_results_dir=experiment_path, appendix='', personal_features=[], model_type=model_dict[label][1], col_to_group='pair_id', candidates_features=candidates_features) else: execute_evaluate.evaluate_grid_search( data=data, base_features=features, add_feature_list=[''], window_size_list=[0], label_dict=label_dict_to_use, num_folds=folds, model_dict=model_dict_to_use, classifier_results_dir=experiment_path, appendix='', personal_features=[], model_type=model_dict[label][1], col_to_group='pair_id') logging.info('Done!')
'predict_future': False, } dir_name_component = [ f'use_forward_backward_fix_history_' if model_param['use_forward_backward_fix_history'] else '', 'use_viterbi_fix_history_' if model_param['use_viterbi_fix_history'] else '', f'squared_sigma_{model_param["squared_sigma"]}_', 'predict_only_last_' if model_param['predict_only_last'] else '', 'predict_future_' if model_param['predict_future'] else '', ] base_file_name = ''.join(dir_name_component) model_directory = utils.set_folder( datetime.now().strftime(f'CRF_cv_{base_file_name}%d_%m_%Y_%H_%M'), 'logs') log_file_name = os.path.join(model_directory, datetime.now().strftime('LogFile.log')) logging.basicConfig( filename=log_file_name, level=logging.DEBUG, format='%(asctime)s: %(levelname)s %(message)s', datefmt='%H:%M:%S', ) main(test=True, cv=True, test_chunking=False, use_forward_backward_fix_history=model_param[