def summary_result(result): logger = get_logger(name='summary_result', file_name=log_dir + '/summary_result', verbose=True) logger.info('\tTraining set\t\t\t\t\t\t\tTest set\t\t\t\t\t\t') logger.info( 'TPR\tTNR\tAUC\tTPR\tTNR\tAUC\tR1\tR2\tR3\tR4\tR5\tP1\tP2\tP3\tP4\tP5') for train_result, test_result, lift_result, param in result: response_matrix = np.array( lift_result[0])[:, 0:5] # take only first 5 decile profit_matrix = np.array( lift_result[1])[:, 0:5] # take only first 5 decile # train_result = np.array() all_metric = np.hstack( [train_result, test_result, response_matrix, profit_matrix]) all_metric = np.vstack([ all_metric, np.mean(all_metric, axis=0), np.std(all_metric, axis=0) ]) # param = experiment_list[i] logger.info(param) for x in all_metric: s = '' for y in x: s = s + str(y) + '\t' logger.info('%s' % s)
def run_experiment_parallel(): experiment_list = [] experiment_list.append( {'h': [60, 60], 'training_epochs': 100, 'pretraining_epochs': 20, 'finetune_lr': 0.01, 'beta': 1, 'cl': [0.1, 0.1], 'cost_vec': [1, 3], 'batch_size': 1, 'pretrain_lr': 0.001}) experiment_list.append( {'h': [100, 100], 'training_epochs': 100, 'pretraining_epochs': 20, 'finetune_lr': 0.01, 'beta': 1, 'cl': [0.1, 0.1], 'cost_vec': [1, 3], 'batch_size': 1, 'pretrain_lr': 0.001}) experiment_list.append( {'h': [120, 120], 'training_epochs': 100, 'pretraining_epochs': 20, 'finetune_lr': 0.01, 'beta': 1, 'cl': [0.1, 0.1], 'cost_vec': [1, 3], 'batch_size': 1, 'pretrain_lr': 0.001}) experiment_list.append( {'h': [60], 'training_epochs': 100, 'pretraining_epochs': 20, 'finetune_lr': 0.01, 'beta': 1, 'cl': [0.1, 0.1], 'cost_vec': [1, 3], 'batch_size': 1, 'pretrain_lr': 0.001}) experiment_list.append( {'h': [80], 'training_epochs': 100, 'pretraining_epochs': 20, 'finetune_lr': 0.01, 'beta': 1, 'cl': [0.1, 0.1], 'cost_vec': [1, 3], 'batch_size': 1, 'pretrain_lr': 0.001}) experiment_list.append( {'h': [100], 'training_epochs': 100, 'pretraining_epochs': 20, 'finetune_lr': 0.01, 'beta': 1, 'cl': [0.1, 0.1], 'cost_vec': [1, 3], 'batch_size': 1, 'pretrain_lr': 0.001}) experiment_list.append( {'h': [120], 'training_epochs': 100, 'pretraining_epochs': 20, 'finetune_lr': 0.01, 'beta': 1, 'cl': [0.1, 0.1], 'cost_vec': [1, 3], 'batch_size': 1, 'pretrain_lr': 0.001}) experiment_list.append( {'h': [130], 'training_epochs': 100, 'pretraining_epochs': 20, 'finetune_lr': 0.01, 'beta': 1, 'cl': [0.1, 0.1], 'cost_vec': [1, 3], 'batch_size': 1, 'pretrain_lr': 0.001}) logger = get_logger(name='experiment_settings', file_name=log_dir + '/experiment_setting.txt', verbose=True) for e in experiment_list: logger.info('experiment_list.append(%s)', e) p = multiprocessing.Pool() func = partial(run_helper, log_dir) result = (p.map(func, experiment_list)) summary_result(result)
def csdnn_helper(self, p_dict): # (train_result, test_result,model_name,y_pred_score_test) logger = get_logger(name=p_dict['nn'], file_name=p_dict['nn'], verbose=True) train_result, test_result, model_path, y_pred_score_test = csdnn_classifier( log_dir=self.log_dir, logger=logger, p_dict=p_dict) return (train_result, test_result, p_dict), y_pred_score_test
def run_helper(log_dir, p_dict,ens_param=None): # contain dict # If main model existed nn = log_dir + '/' +p_dict['exp_name']+'_'+str(random.random()) + '.txt' main_file_list.append(nn) logger = get_logger(name=nn, file_name= nn, verbose=True) train_result, test_result = test_SdA(p_dict=p_dict, logger=logger,ens_param=ens_param) return train_result, test_result, p_dict
def run_helper(log_dir,p_dict): # contain dict nn = str(random.random()) + '.txt' logger = get_logger(name=nn, file_name=log_dir + '/' + nn, verbose=True) train_result, test_result, test_lift = test_SdA(finetune_lr=p_dict['finetune_lr'], pretraining_epochs=p_dict['pretraining_epochs'], pretrain_lr=p_dict['pretrain_lr'], training_epochs=p_dict['training_epochs'], batch_size=p_dict['batch_size'], h=p_dict['h'], cl=p_dict['cl'], cost_vec=p_dict['cost_vec'],beta=p_dict['beta'],logger=logger) return train_result, test_result, test_lift, p_dict
s = s + str(y) + '\t' logger.info('%s' % s) if __name__ == '__main__': datetim = time.strftime("%d_%m_%Y_%H_%M_%S") log_dir = 'log/' + data_dir + '/' + datetim if not os.path.exists(log_dir): os.makedirs(log_dir) start_time = timeit.default_timer() run_experiment_parallel() #run_experiment() end_time = (timeit.default_timer() - start_time) / 60 #logger.info('Time took:%f mn', end_time) logger = get_logger(name='summary_result', file_name=log_dir + '/summary_result', verbose=True) logger.info('Time took:%f mn', end_time) # combine test file to summary.txt time.sleep(5) file_list = [f for f in os.listdir(log_dir)] with open(log_dir + '/' + datetim + '.txt', 'w') as outfile: for fname in file_list: with open(log_dir + '/' + fname) as infile: for line in infile: outfile.write(line) print(file_list)
if __name__ == '__main__': main_file_list = [] datetim = time.strftime("%d_%m_%Y_%H_%M_%S") log_dir = 'log/' + data_dir + '/' + datetim if not os.path.exists(log_dir): os.makedirs(log_dir) if not os.path.exists(log_dir + '/tmp_model'): os.makedirs(log_dir + '/tmp_model') if not os.path.exists(log_dir + '/tmp_data'): os.makedirs(log_dir + '/tmp_data') settings.init(log_dir) start_time = timeit.default_timer() nn = log_dir + '/summary_result.txt' main_summary_logger = get_logger(name=nn, file_name=nn, verbose=False) # run_experiment_parallel() # run_experiment() run_repeat_experiment() end_time = (timeit.default_timer() - start_time) / 60 # logger.info('Time took:%f mn', end_time) logger = get_logger(name='summary_result', file_name=log_dir + '/summary_result.txt', verbose=True) logger.info('Time took:%f mn', end_time) time.sleep(10) main_file_list.append(log_dir + '/summary_result.txt') combine_log(path=log_dir + '/' + datetim + '.txt', file_list=main_file_list)