def talos_model(self, force=False): """ Método para retornar um modelo iterativo do Talos. Se um modelo já existir, carregará os pesos e retornará um objeto de modelo iterativo. Caso não exista um modelo, um será gerado. :param force: se deve ou não forçar a geração de um modelo fresco :return: modelo iterativo do talos """ model_path = self.MODULE_PATH + "/data/talos/fakedata.zip" if Path(model_path).is_file() and force is False: return ta.Restore(model_path) else: t = ta.Scan(x=self.X, y=self.Y, model=self.fake_news_model, grid_downsample=.01, params=self.p, dataset_name='fakenews', experiment_no='1') ta.Deploy(t, str(model_path).replace(".zip", "")) return t
def get_best_Talos(windows): df = create_finance(returns=False, plot_corr=False, Trends=False) for i in windows: data = prepare_data(df, i, 10, 10, returns=False, normalize_cheat=False) data.create_windows() x_train = data.x_train.reshape(len(data.x_train), -1) y_train = data.y_train.reshape(-1, 1) x_valid = data.x_valid.reshape(len(data.x_valid), -1) y_valid = data.y_valid.reshape(-1, 1) file = 'Talos_Results/ANN_WithoutTrends_stock_window_' + str(i) print("Running Talos on window size: {}".format(i)) t = ta.Scan(x=x_train, y=y_train, x_val=x_valid, y_val=y_valid, model=create_model, params=p, experiment_name=file, fraction_limit=0.1, seed=2) dpl = file + '_deploy' with CustomObjectScope({'GlorotUniform': glorot_uniform()}): ta.Deploy(scan_object=t, model_name=dpl, metric='mean_squared_error', asc=True)
steps_per_epoch=train.n / train.batch_size, epochs=EPOCHS, validation_data=valid, validation_steps=valid.n / valid.batch_size, callbacks=[ckpt, reduce_lr, early_stopping, tensorboard]) return out, model if __name__ == '__main__': x, y, x_val, y_val = [np.array([1, 2]) for i in range(4)] h = ta.Scan( x, y, params=p, model=input_model, #grid_downsample=.1, #reduction_method='correlation', x_val=x_val, y_val=y_val, val_split=0, shuffle=False, last_epoch_value=True, print_params=True) # accessing the results data frame print(h.data.head()) # get the highest result ('val_acc' by default) print('Best accuracy: ', r.high()) ta.Deploy(h, 'talos_prelim_model_search')
def test_rest(scan_object): print('\n >>> start testing the rest... \n') import talos import random deploy_filename = 'test' + str(random.randint(1, 20000000000)) print('\n ...Deploy()... \n') talos.Deploy(scan_object, deploy_filename, 'val_acc') print('\n ...Restore()... \n') restored = talos.Restore(deploy_filename + '.zip') x, y = talos.templates.datasets.breast_cancer() x = x[:50] y = y[:50] x_train, y_train, x_val, y_val = talos.utils.val_split(x, y, .2) x = talos.utils.rescale_meanzero(x) callbacks = [ talos.utils.early_stopper(10), talos.utils.ExperimentLogCallback('test', {}) ] metrics = [ talos.utils.metrics.f1score, talos.utils.metrics.fbeta, talos.utils.metrics.mae, talos.utils.metrics.mape, talos.utils.metrics.matthews, talos.utils.metrics.mse, talos.utils.metrics.msle, talos.utils.metrics.precision, talos.utils.metrics.recall, talos.utils.metrics.rmae, talos.utils.metrics.rmse, talos.utils.metrics.rmsle ] from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense print('\n ...callbacks and metrics... \n') model1 = Sequential() model1.add(Dense(10, input_dim=x.shape[1])) model1.add(Dense(1)) model1.compile('adam', 'logcosh', metrics=metrics) model1.fit(x, y, callbacks=callbacks) print('\n ...generator... \n') model2 = Sequential() model2.add(Dense(10, input_dim=x.shape[1])) model2.add(Dense(1)) model2.compile('adam', 'logcosh') model2.fit_generator(talos.utils.generator(x, y, 10), 5) print('\n ...SequenceGenerator... \n') model3 = Sequential() model3.add(Dense(10, input_dim=x.shape[1])) model3.add(Dense(1)) model3.compile('adam', 'logcosh') model3.fit_generator(talos.utils.SequenceGenerator(x, y, 10)) print('\n ...gpu_utils... \n') talos.utils.gpu_utils.force_cpu() talos.utils.gpu_utils.parallel_gpu_jobs() print('\n ...gpu_utils... \n') from talos.utils.test_utils import create_param_space create_param_space(restored.results, 5) print('finished testing the rest \n')
# up to two dimensional kernel density estimator r.plot_kde('val_acc') # a simple histogram r.plot_hist(bins=50) # heatmap correlation r.plot_corr() # a four dimensional bar grid r.plot_bars('batch_size', 'val_acc', 'first_neuron', 'lr') e = ta.Evaluate(h) e.evaluate(x, y, folds=10, average='macro') ta.Deploy(h, 'iris') iris = ta.Restore('iris.zip') # make predictions with the model iris.model.predict(x) # get the meta-data for the experiment print(iris.details) # get the hyperparameter space boundary print(iris.params) # sample of x and y data print(iris.x) print(iris.y) # the results dataframe print(iris.results)
# Scan scan_object = test_scan_object() # Reporting test_reporting_object(scan_object) start_time = str(time.strftime("%s")) p = ta.Predict(scan_object) p.predict(scan_object.x) p.predict_classes(scan_object.x) ta.Autom8(scan_object, scan_object.x, scan_object.y) ta.Evaluate(scan_object) ta.Deploy(scan_object, start_time) ta.Restore(start_time + '.zip') test_random_methods() fit_generator = generator(scan_object.x, scan_object.y, 20) force_cpu() TestCancer().test_scan_cancer_metric_reduction() TestCancer().test_scan_cancer_loss_reduction() TestCancer().test_linear_method() TestCancer().test_reverse_method() TestIris().test_scan_iris_explicit_validation_set() TestIris().test_scan_iris_explicit_validation_set_force_fail() TestIris().test_scan_iris_1() TestIris().test_scan_iris_2()
def main(): expert_path = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/Push_49_ep/' # r_model = Restore('imitation_learning.zip') heat_maps = np.load(expert_path + 'heatmap.npy') states = np.load(expert_path + 'states.npy') actions = np.load(expert_path + 'actions.npy') # p = {'hist_size': [1, 2, 3, 4, 5], # 'input_conv_kernel_size': [(2, 2), (3, 3)], # 'input_conv_filters': [8, 16, 32], # 'conv_dropout': [0.0, 0.1, 0.25], # 'conv_layers': [1, 2, 3], # 'conv_kernel_size' : [(2, 2), (3, 3)], # 'conv_filters' : [8, 16, 32, 64], # 'conv_strides' : [(1, 1),(2, 2)], # 'batch_norm' : [True, False], # 'output_conv_kernel_size' : [(2, 2), (3, 3)], # 'output_conv_filters' :[32, 64], # 'input_layer_size' : [32, 64, 128], # 'hidden_layers' : [1, 2, 3], # 'layer_size' : [32, 64, 128], # 'output_layer_size': [16, 32, 64, 128], # 'bias': [True, False], # 'batch_size': [16, 32, 64], # 'learning_rate': [1e-4, 5e-4, 1e-5, 5e-5], # 'epochs' : [500], # 'optimizer' : ['adam', 'nadam', 'sgd'], # 'activations' : ['relu', 'elu', 'sigmoid'], # 'output_activations' : ['relu', 'elu', 'sigmoid']} hist_size = 1 # # heat_map_hist = [] # # for kk in range(len(heat_maps) - hist_size): # heat_map_hist.append(heat_maps[kk:kk + hist_size, :, :]) # # heat_maps = np.array(heat_map_hist) # actions = actions[hist_size:] heat_maps = heat_maps.reshape(len(heat_maps), hist_size, 100, 6) # p = {'input_conv_kernel_size': [(2, 2), (3, 3)], # 'input_conv_filters': [8, 16, 32], # 'conv_dropout': [0.0, 0.25], # 'conv_layers': [1, 2], # 'conv_kernel_size': [(2, 2), (3, 3)], # 'conv_filters': [8, 16, 32], # 'conv_strides': [(1, 1), (2, 2)], # 'batch_norm': [False], # 'output_conv_kernel_size': [(2, 2), (3, 3), (4, 4)], # 'output_conv_filters': [16, 32, 64], # 'input_layer_size' : [64, 128, 256], # 'hidden_layers': [1, 2, 3], # 'layer_size': [64, 128, 256], # 'output_layer_size': [16, 32, 64], # 'output_bias': [False, True], # 'batch_size': [64], # 'learning_rate': [1e-4, 2.5e-4, 5e-4], # 'epochs': [400], # 'optimizer': ['adam'], # 'activations': ['relu'], # 'output_activations': ['sigmoid']} # p = { 'input_conv_kernel_size': [(2, 2)], 'input_conv_filters': [8], 'conv_dropout': [0.0], 'conv_layers': [1], 'conv_kernel_size': [(3, 3)], 'conv_filters': [32], 'conv_strides': [(1, 1)], 'batch_norm': [False], 'output_conv_kernel_size': [(3, 3)], 'output_conv_filters': [64], 'input_layer_size': [256], 'hidden_layers': [1], 'layer_size': [256], 'output_layer_size': [64], 'output_bias': [False], 'batch_size': [64], 'learning_rate': [5e-4], 'epochs': [400], 'optimizer': ['adam'], 'activations': ['relu'], 'output_activations': ['sigmoid'] } # imitation_learning(heat_maps, actions, params=p) t = talos.Scan(x=heat_maps, y=actions, model=imitation_learning, params=p, experiment_name='BC_HM_Push') talos.Deploy(scan_object=t, model_name='Push_BC_best_vl', metric='val_loss', asc=True) talos.Deploy(scan_object=t, model_name='Push_BC_best_l', metric='loss', asc=True)
experiment_name='CNN_Optimization', round_limit=10, fraction_limit=0.05) cnn_analyze = talos.Analyze(cnn_scan) documentation_file_parameteropt.write( "CNN: Best parameters {}, reached score: {} \n".format( cnn_analyze.best_params('accuracy', ['accuracy', 'loss', 'val_loss']), cnn_analyze.high('accuracy'))) pred_cnn = talos.Predict(cnn_scan).predict(x_t, metric='val_f1score', asc=True) #evaluate the model cnn_evaluation_scores, cnn_cm = evaluation.multilabel_evaluation( d_test_array, label_binarizer.inverse_transform(pred_cnn), "CNN") documentation_file_modelopt.write(cnn_evaluation_scores) #deploy best model model_cnn = talos.Deploy(cnn_scan, "model_cnn_transformerxl", metric='val_accuracy') #build LSTM model and evaluate the model print("LSTM model evaluation") def lstm_optimization(x_train, y_train, x_test, y_test, params): """Randomized search to optimize parameters of Neural Network.""" optimization_model = models.Sequential() optimization_model.add(layers.LSTM(params['units'], return_sequences=True)) optimization_model.add(layers.LSTM(params['units'], return_sequences=False)) optimization_model.add(layers.Dropout(0.5)) optimization_model.add(layers.Dense(params['dense'], activation=relu)) optimization_model.add(layers.Dense(params['dense'], activation=relu))
out = model.fit_generator( train, steps_per_epoch=train.n / train.batch_size, epochs=EPOCHS, validation_data=valid, validation_steps=valid.n / valid.batch_size, callbacks=[ckpt, reduce_lr, early_stopping, tensorboard]) return out, model if __name__ == '__main__': # workaround to feed data in batches instead of loading into memory as talos requires x, y, x_val, y_val = [np.array([1, 2]) for i in range(4)] h = ta.Scan( x, y, params=p, model=input_model, #grid_downsample=.1, #reduction_method='correlation', x_val=x_val, y_val=y_val, val_split=0, shuffle=False, last_epoch_value=True, print_params=True) ta.Deploy(h, 'talos_beauty_inception_resnet_transfer') print(h.data.head()) print(h.details)
params=cnn_params, experiment_name='CNN_Optimization', round_limit=10, fraction_limit=0.05) cnn_analyze = talos.Analyze(cnn_scan) documentation_file_parameteropt.write( "CNN: Best parameters {}, reached score: {} \n".format( cnn_analyze.best_params('accuracy', ['accuracy', 'loss', 'val_loss']), cnn_analyze.high('accuracy'))) pred_cnn = talos.Predict(cnn_scan).predict(x_t, metric='val_f1score', asc=True) #evaluate the model cnn_evaluation_scores, cnn_cm = evaluation.multilabel_evaluation( d_test_array, label_binarizer.inverse_transform(pred_cnn), "CNN") documentation_file_modelopt.write(cnn_evaluation_scores) #deploy best model model_cnn = talos.Deploy(cnn_scan, "model_cnn_scibert", metric='val_accuracy') #build LSTM model and evaluate the model print("LSTM model evaluation") def lstm_optimization(x_train, y_train, x_test, y_test, params): """Randomized search to optimize parameters of Neural Network.""" optimization_model = models.Sequential() optimization_model.add(layers.LSTM(params['units'], return_sequences=True)) optimization_model.add(layers.LSTM(params['units'], return_sequences=False)) optimization_model.add(layers.Dropout(0.5)) optimization_model.add(layers.Dense(params['dense'], activation=relu)) optimization_model.add(layers.Dense(params['dense'], activation=relu)) optimization_model.add(layers.Dense(int(num_classes),
'activation': ['relu'], 'kernel_initializer': ['glorot_uniform'], 'h_nodes': [100, 128], 'kernel_size': [3, 5, 9] } ##probabilistic reduction based on correlation t_nonrandom_downsampling = talos.Scan(x=x_train, y=y_train, model=cnn_model, dataset_name='isic_hpt', experiment_no='2', params=parameters_small, reduction_method='correlation', reduction_interval=10, reduction_window=10, grid_downsample=1, talos_log_name='hpt_nonrandom') try: talos.Deploy(t_nonrandom_downsampling, 't_nonrandom_tuning') except: print(" ") OutputResults(x_train, y_train, 't_nonrandom_tuning/t_nonrandom_tuning_model.json', 't_nonrandom_tuning/t_nonrandom_tuning_model.h5') OutputResults(x_test, y_test, 't_nonrandom_tuning/t_nonrandom_tuning_model.json', 't_nonrandom_tuning/t_nonrandom_tuning_model.h5')
def test_rest(scan_object): print('\n >>> start testing the rest... \n') import talos import random print('\n ...Deploy()... \n') talos.Deploy(scan_object, 'testing_deploy', 'val_acc') print('\n ...Restore()... \n') talos.Restore('testing_deploy' + '.zip') x, y = talos.templates.datasets.breast_cancer() x = x[:50] y = y[:50] callbacks = [ talos.utils.early_stopper(10), talos.utils.ExperimentLogCallback('test', {}) ] metrics = [ talos.utils.metrics.f1score, talos.utils.metrics.fbeta, talos.utils.metrics.mae, talos.utils.metrics.mape, talos.utils.metrics.matthews, talos.utils.metrics.mse, talos.utils.metrics.msle, talos.utils.metrics.precision, talos.utils.metrics.recall, talos.utils.metrics.rmae, talos.utils.metrics.rmse, talos.utils.metrics.rmsle ] from keras.models import Sequential from keras.layers import Dense print('\n ...callbacks and metrics... \n') model1 = Sequential() model1.add(Dense(10, input_dim=x.shape[1])) model1.add(Dense(1)) model1.compile('adam', 'logcosh', metrics=metrics) model1.fit(x, y, callbacks=callbacks) print('\n ...generator... \n') model2 = Sequential() model2.add(Dense(10, input_dim=x.shape[1])) model2.add(Dense(1)) model2.compile('adam', 'logcosh') model2.fit_generator(talos.utils.generator(x, y, 10), 5) print('\n ...SequenceGenerator... \n') model3 = Sequential() model3.add(Dense(10, input_dim=x.shape[1])) model3.add(Dense(1)) model3.compile('adam', 'logcosh') model3.fit_generator(talos.utils.SequenceGenerator(x, y, 10)) print('\n ...gpu_utils... \n') talos.utils.gpu_utils.force_cpu() talos.utils.gpu_utils.parallel_gpu_jobs() print('finised testing the rest \n')
out = model.fit_generator( train, steps_per_epoch=train.n / train.batch_size, epochs=EPOCHS, validation_data=valid, validation_steps=valid.n / valid.batch_size, callbacks=[ckpt, reduce_lr, early_stopping, tensorboard]) return out, model if __name__ == '__main__': # workaround to feed data in batches instead of loading into memory as talos requires x, y, x_val, y_val = [np.array([1, 2]) for i in range(4)] h = ta.Scan( x, y, params=p, model=input_model, #grid_downsample=.1, #reduction_method='correlation', x_val=x_val, y_val=y_val, val_split=0, shuffle=False, last_epoch_value=True, print_params=True) ta.Deploy(h, 'talos_fashion_inception_resnet_transfer') print(h.data.head()) print(h.details)
def main(): expert_path_1 = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/no_clip/all_recs_no_peel/' expert_path_2 = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/no_clip/lift_23_ep/' # expert_path = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/sand_levels_RHM/y_clip/' states_1 = np.load(expert_path_1+'states.npy', allow_pickle=True) states_2 = np.load(expert_path_2+'states.npy', allow_pickle=True) states = np.load(expert_path_2 + 'states.npy', allow_pickle=True) inputs = np.concatenate([np.stack(states[:, 1]), np.array(states[:, 5]).reshape(len(states), 1)], axis=1) labels = np.array(states[:, 4]) # p = {'hist_size': [1, 2, 3, 4, 5], # 'input_conv_kernel_size': [(2, 2), (3, 3)], # 'input_conv_filters': [8, 16, 32], # 'conv_dropout': [0.0, 0.1, 0.25], # 'conv_layers': [1, 2, 3], # 'conv_kernel_size': [(2, 2), (3, 3)], # 'conv_filters': [8, 16, 32, 64], # 'conv_strides': [(1, 1), (2, 2)], # 'batch_norm': [True, False], # 'output_conv_kernel_size': [(2, 2), (3, 3)], # 'output_conv_filters': [32, 64], # 'input_layer_size': [32, 64, 128], # 'hidden_layers': [1, 2, 3], # 'layer_size': [32, 64, 128], # 'output_layer_size': [16, 32, 64, 128], # 'output_bias': [True, False], # 'batch_size': [16, 32, 64], # 'learning_rate': [1e-4, 5e-4, 1e-5, 5e-5], # 'epochs': [500], # 'optimizer': ['adam', 'nadam', 'sgd'], # 'activations': ['relu', 'elu', 'sigmoid'], # 'output_activations': ['relu', 'elu', 'sigmoid']} # p = {'input_conv_kernel_size': [(2, 2), (3, 3)], # 'input_conv_filters': [8, 16, 32], # 'conv_dropout': [0.0, 0.25], # 'conv_layers': [1, 2], # 'conv_kernel_size': [(2, 2), (3, 3)], # 'conv_filters': [8, 16, 32], # 'conv_strides': [(1, 1), (2, 2)], # 'batch_norm': [False], # 'output_conv_kernel_size': [(2, 2), (3, 3), (4, 4)], # 'output_conv_filters': [16, 32, 64], # 'input_layer_size' : [64, 128, 256], # 'hidden_layers': [1, 2, 3], # 'layer_size': [64, 128, 256], # 'output_layer_size': [16, 32, 64], # 'output_bias': [False, True], # 'batch_size': [64], # 'learning_rate': [1e-4, 2.5e-4, 5e-4], # 'epochs': [400], # 'optimizer': ['adam'], # 'activations': ['relu'], # 'output_activations': ['sigmoid']} # fract_lim = 0.00001 p = {'input_conv_kernel_size': [(3, 3)], 'input_conv_filters': [16], 'conv_dropout': [0.25], 'conv_layers': [2], 'conv_kernel_size': [(2, 2)], 'conv_filters': [16], 'conv_strides': [(1, 1)], 'batch_norm': [False], 'output_conv_kernel_size': [(3, 3)], 'output_conv_filters': [16], 'input_layer_size': [256], 'hidden_layers': [1], 'layer_size': [256], 'output_layer_size': [32], 'output_bias': [False], 'batch_size': [64], 'learning_rate': [1e-5], 'epochs': [1000], 'optimizer': ['adam'], 'activations': ['relu'], 'output_activations': ['sigmoid']} fract_lim = 1 learn = True test_name = 'new_test_new_recordings_LP_model_10_pred' if learn: # labels = actions # aug_heat_maps = heat_maps.reshape(heat_maps.shape[0],1,heat_maps.shape[1],heat_maps.shape[2]) t = talos.Scan(x=aug_heat_maps, y=labels, model=supervised_learning, params=p, fraction_limit=fract_lim, experiment_name=test_name) talos.Deploy(scan_object=t, model_name=test_name+'_best_vl', metric='val_loss', asc=True) else: json_file = open('/home/graphics/git/SmartLoader/'+test_name+'_best_vl/'+test_name+'_best_vl_model.json', 'rb') loaded_model_json = json_file.read() json_file.close() model = model_from_json(loaded_model_json) # load weights into new model model.load_weights('/home/graphics/git/SmartLoader/'+test_name+'_best_vl/'+test_name+'_best_vl_model.h5') print(' ------------ now lets evaluate -------------') evaluate = False if evaluate: loss = [] evals = 50 for k in range(evals): index = np.random.randint(len(aug_heat_maps)) states = model.predict(aug_heat_maps[index, :, :, :].reshape(1, 1, aug_heat_maps.shape[2], aug_heat_maps.shape[3])) arm_height = states[0][0]*50+150 plt.imshow(aug_heat_maps[index, :, :, :].squeeze(), aspect=1) plt.text(40, 5, arm_height, fontsize=15) print(labels[index][0] * 50 + 150) plt.show() model.save('/home/graphics/git/SmartLoader/saved_models/'+test_name) print('saved')
lr=lr_normalizer(params['lr'], params['optimizer'])), loss=loss_fx, metrics=metrics, class_weight=class_weight) out = model.fit_generator( imageLoader(sub_train_images, y_train, params['batch_size']), steps_per_epoch=sub_train_images.shape[0] // params['batch_size'], epochs=20, validation_data=imageLoader(sub_val_images, y_val, params['batch_size']), validation_steps=sub_val_images.shape[0] // params['batch_size'], callbacks=[es, mc], verbose=2) #print(f"out:{out.history.keys()}") return out, model #print(f"y_train[0]: {y_train[0]}") # hyperparameter optimization t = ta.Scan(x=sub_train_images, y=y_train, x_val=sub_val_images, y_val=y_val, params=p, model=talos_model, experiment_name='exp_' + exp_id) ta.Deploy(t, exp_id, metric="val_accuracy", asc=False)
fraction_limit=1, #time_limit = '2019-09-25 10:00', seed=Config.random_seed) f = open("Talos_scan_object.pickle", "wb+") pickle.dump(scan_object, f) f.close() #from talos import Evaluate # # create the evaluate object #e = Evaluate(scan_object) # ## perform the evaluation # #tcga_x = np.expand_dims(Data.all_data, axis=2) #tcga_y = Data.integer_encoded #####evaluate change the one hot encode to int encode, and not mentioned in talos doc..... #print('Cross validation:') #scan_object.evaluate_models(x=tcga_x, # y=tcga_y, # folds = 10, # metric='val_acc', # ) out = 'cnn_1d_tumor_type_t_' + Config.treat if os.path.isdir(out): shutil.rmtree(out) ta.Deploy(scan_object, out, metric='val_acc') print("--- %s seconds ---" % (tt.time() - start_time))
def main(): global y_map_clip expert_path = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/lift_ep_12_5/' lift_est_model = load_model('/home/graphics/git/SmartLoader/saved_models/lift_est_model_corrected') heat_maps = np.load(expert_path+'heatmaps.npy') states = np.load(expert_path+'states.npy', allow_pickle=True) actions = np.load(expert_path+'actions.npy') ep_starts = np.load(expert_path+'starts.npy') ep_starts[0] = True # rand_ind = np.random.randint(len(states), size=100) # fictive_maps = np.array([np.ones(heat_maps[0].shape)*0.1]*100) # fictive_states = states[rand_ind] # fictive_states[:, 5] = 165 # fictive_states[:, 4] = 140 # fictive_ep_starts = ep_starts[rand_ind] # heat_maps = np.vstack([heat_maps, fictive_maps]) # states = np.vstack([states, fictive_states]) # ep_starts = np.hstack([ep_starts, fictive_ep_starts]) labels = [] aug_heat_maps = [] ######################################## # organize data to fit mission: ######################################## label_horizon = 5 # how far to the future will the network predict num_of_step_labels = 5 # how many future steps will the network predict x_map_front_clip = 100 # window size in front of blade x_map_front_offset = -60 # offset size in front of blade y_map_clip = 30 # window size on sides of the blade learn = False test_name = 'lift_task_LP_model_5_pred_b_wind' if learn: for k in range(label_horizon+300, len(states)): state_pred = [] if ep_starts[(k - label_horizon+1):(k + 1)].any(): continue # aug_map = map_clipper(heat_maps[k], states[k, 1], x_map_front_clip, # x_map_front_offset, y_map_clip) # t_lift_shovle_pos, t_lift_pitch_state = lift_normalize_states(states[k][1], states[k][5]) # t_lift_state = lift_est_model.predict(np.hstack([t_lift_shovle_pos, t_lift_pitch_state]).reshape(1, 4)) # show_heatmap(aug_map, t_lift_state) for j in range(k-num_of_step_labels+1, k+1): shovle_pos = states[j][1] pitch_state = states[j][[5]] # only pitch values lift_shovle_pos, lift_pitch_state = lift_normalize_states(shovle_pos, pitch_state) lift_state = lift_est_model.predict(np.hstack([lift_shovle_pos, lift_pitch_state]).reshape(1, 4)) shovle_pos, pitch_state = map_normalize_states(shovle_pos, pitch_state) # aug_states = [lift_state, pitch_state] ## for LP model aug_states = shovle_pos[0] ## for x model # aug_states[0:3] -= states[k - label_horizon][1] ## for moving set-point # aug_states = actions[j,1] ## for thrust model state_pred.append(aug_states) aug_map = map_clipper(heat_maps[k-label_horizon], states[k-label_horizon, 1], x_map_front_clip, x_map_front_offset, y_map_clip) state_pred = np.array(state_pred).squeeze() # show_heatmap(heat_maps[k - label_horizon], state_pred[-1][0] * 100 + 150) # show_heatmap(aug_map, state_pred[-1][0]*100+150) if not aug_map.any(): continue labels.append(np.array(state_pred).reshape(np.prod(state_pred.shape))) aug_heat_maps.append(aug_map) # if (k % 20) == 0: # plt.close() # if ep_starts[k+label_horizon]: # plt.close() aug_heat_maps = np.array(aug_heat_maps) aug_heat_maps = aug_heat_maps.reshape([aug_heat_maps.shape[0], 1, aug_heat_maps.shape[1], aug_heat_maps.shape[2]]) labels = np.array(labels) p = {'input_conv_kernel_size': [(3, 3)], 'input_conv_filters': [16], 'conv_dropout': [0.25], 'conv_layers': [2], 'conv_kernel_size': [(2, 2)], 'conv_filters': [16], 'conv_strides': [(1, 1)], 'batch_norm': [False], 'output_conv_kernel_size': [(3, 3)], 'output_conv_filters': [16], 'input_layer_size': [256], 'hidden_layers': [1], 'layer_size': [256], 'output_layer_size': [32], 'output_bias': [False], 'batch_size': [64], 'learning_rate': [5e-5], 'epochs': [200], 'optimizer': ['adam'], 'activations': ['relu'], 'output_activations': ['sigmoid']} fract_lim = 1 # labels = actions # aug_heat_maps = heat_maps.reshape(heat_maps.shape[0],1,heat_maps.shape[1],heat_maps.shape[2]) t = talos.Scan(x=aug_heat_maps, y=labels, model=supervised_learning, params=p, fraction_limit=fract_lim, experiment_name=test_name) talos.Deploy(scan_object=t, model_name=test_name+'_best_vl', metric='val_loss', asc=True) else: json_file = open('/home/graphics/git/SmartLoader/'+test_name+'_best_vl/'+test_name+'_best_vl_model.json', 'rb') loaded_model_json = json_file.read() json_file.close() model = model_from_json(loaded_model_json) # load weights into new model model.load_weights('/home/graphics/git/SmartLoader/'+test_name+'_best_vl/'+test_name+'_best_vl_model.h5') print(' ------------ now lets evaluate -------------') evaluate = False if evaluate: loss = [] evals = 50 for k in range(evals): index = np.random.randint(len(aug_heat_maps)) states = model.predict(aug_heat_maps[index, :, :, :].reshape(1, 1, aug_heat_maps.shape[2], aug_heat_maps.shape[3])) arm_height = states[0][0]*50+150 plt.imshow(aug_heat_maps[index, :, :, :].squeeze(), aspect=1) plt.text(40, 5, arm_height, fontsize=15) print(labels[index][0] * 50 + 150) plt.show() model.save('/home/graphics/git/SmartLoader/saved_models/'+test_name) print('saved')
verbose=1) test_predict = model.predict_generator( generator=rescale_generator.flow(x=test_X, batch_size=predict_batch_size, shuffle=False), steps=test_X.shape[0] / predict_batch_size, verbose=1) # Write train, val, and test predictions to file for i, predictions in enumerate([train_predict, val_predict, test_predict]): split = ["train", "val", "test"][i] with open(experiment_name + '_' + split + 'Predictions.txt', 'w') as predictOutput: for i in range(len(predictions)): _ = predictOutput.write(f"{predictions[i][0]}\n") for i, trues in enumerate([train_y, val_y, test_y]): split = ["train", "val", "test"][i] with open(experiment_name + '_' + split + 'True.txt', 'w') as trueOutput: for i in range(len(trues)): _ = trueOutput.write(f"{trues[i]}\n") # Save best model scan_object.x = np.zeros(500) # necessary for model to restore properly scan_object.y = np.zeros(500) ta.Deploy(scan_object, experiment_name + "_hyperopt_models", metric="val_acc", asc=False)
return out, model #STEP 4. SCANNING THE HYPER PARAMETER SPACE import talos #Random reduction t_random = talos.Scan(x=x_train, y=y_trainHot, model=cnn_model, dataset_name='wbc', experiment_no='2', params=hyperP, grid_downsample=0.5) talos.Deploy(t_random, "talos_random4_tuning", metric='accuracy') #try: # talos.Deploy(t_random, "talos_random3_tuning", metric='val_acc') #except: # print(" ") def OutputResults(x_data, y_data, bestmodel_json_file, bestmodel_weights_h5_file): from keras.models import model_from_json file = open(bestmodel_json_file, 'r') lines = file.read() file.close() model = model_from_json(lines) model.load_weights(bestmodel_weights_h5_file)
def main(): from keras import backend as K K.set_session( K.tf.Session(config=K.tf.ConfigProto(intra_op_parallelism_threads=32, inter_op_parallelism_threads=32))) print("using 32 CPUs or 64 Cores on cluster") if len(sys.argv) != 3: print("incorrect number of arguments - exiting program") sys.exit(0) s = random.getstate() snp_file = str(sys.argv[1]) print("filename for SNPs {}".format(snp_file)) #pheno_file=str(sys.argv[2]) #print("filename for phenotype is {}".format(pheno_file)) path_input = "/home/jgrealey/Simulations/ten_k_samples/test/" #Maybe i want the standardised genotype #path_pheno="/home/jgrealey/Simulations/ten_k_samples/test/" results_dir = "/projects/jgrealey/embedding/ten_k_samples/results/dae/" plots_dir = "/projects/jgrealey/embedding/ten_k_samples/plots/dae/" numsam = 1000 nphenotoload = numsam - 3 # listtoload = range(numsam) numsnps = 300 testcols = np.arange(100) datafile = snp_file model_name = str(sys.argv[2]) #"dae_test_hiddenlay_model" experiment_name = model_name + "_" + snp_file print(experiment_name) print(model_name) #'weight_regulizer':[None]} #print(listtoload) #for x, rows are snps and columns are samples so usecoles #x=pd.read_csv(path_input+snp_file+".csv",index_col=0,usecols=listtoload,nrows=numsnps)#rows are snps#nrows=100)#testing for faster I/O #full laod #cols_read=pd.read_csv(path_input+snp_file+".csv",usecols=[0]) #print(cols_read) #sys.texit(0) #full load x = pd.read_csv( path_input + snp_file + ".csv", index_col=0, dtype='object' ) #,usecols=listtoload,nrows=numsnps)#rows are snps#nrows=100)#testing for faster I/O #testing #TESTING# #x=pd.read_csv(path_input+snp_file+".csv",index_col=0,usecols=listtoload)#,nrows=numsnps)#rows are snps#nrows=100)#testing for faster I/O #ONLY TRANSPOSE IF READING GENOTYPE FILE #STANDARDISED FILE DOESNT NEED TO BE TRANSPOSED #x=x.T #TESTING# #y=pd.read_csv(path_pheno+pheno_file+".csv",index_col=0,header=None)#,dtype='float')#tasting for faster I/O #full load #y=pd.read_csv(path_pheno+pheno_file+".csv",index_col=0,nrows=nphenotoload,header=None)#tasting for faster I/O #y.columns=["phenotype"] #print(x.tail()) #print(y.tail()) index = x.index cols = x.columns #print(index) #print(cols) samples = index[2:-3] testsamples = index[2:] #print(testsamples) #print(y) #xnorm=normalise(x,[2,-3]) #del x #x=xnorm #print(xnorm) genotypes = x.loc[samples] genotypes = genotypes.astype(np.float) #pd.to_numeric(genotypes) #print(genotypes) #noisy_genotypes=mask_function(dataframe=genotypes,noise=0.2,batch_size=100) #print(noisy_genotypes) #genotypes=x.loc[testsamples] # calculate sparsity #sparsity = 1.0 - np.count_nonzero(genotypes) /genotypes.size #print(sparsity)0.7520695419592109 labels = x.loc[index[-1]].astype('int') #labels.index=xnorm.index #print(labels) beta = x.loc[index[-2]].astype('float') maf = x.loc[index[-3]].astype('float') SNP_id = x.loc[index[0]] SNP_pos = x.loc[index[1]] #print(genotypes) #df=genotypes.join(y) #print(df) #train, test = train_test_split(df, test_size=0.60, random_state=42)#40% used in training train, test = train_test_split(genotypes, test_size=0.80, random_state=42) #40% used in training train_samples = train.index #noisy_train=mask_function(dataframe=train,noise=0.3,batch_sizes=100) vali, test = train_test_split(test, test_size=0.9, random_state=42) # vali_samples = vali.index test_samples = test.index #noisy_vali=mask_function(dataframe=vali,noise=0.3,batch_sizes=100) noisy_test = mask_function(dataframe=test, noise=0.3, batch_sizes=100) #print(train.head) #print(noisy_train.head) #print(train_samples) #print(test_samples) #print(train.loc[train_samples]) #print(train.shape)#(4000, 1000 #print(noisy_train.shape) #print(test.shape)#(5400, 1000) #print(vali.shape)#(600, 1000) #sys.exit(0) #print(train[samples]) #print(test) #print(df) num_dum = 100 numSNPs = train.shape[1] print(numSNPs) pfull = { 'lr': (0.07, 1.0, 5), #'lr': (0.1, 10), 'first_neuron': [200, 500, 1000], 'embedding_size': [250, 500, 750], 'batch_size': [50, 100], 'epochs': [200], 'noise': [0.3, 0.2], 'dropout': [0, 0.40], 'hidden_layers': [0, 1, 2], 'activation': [keras.activations.relu, keras.activations.elu], #'x_val_noise':noisy_vali.values, #'x_train_noise':noisy_train.values, 'optimizer': [ keras.optimizers.Adam, keras.optimizers.RMSprop, keras.optimizers.Adadelta ], 'loss': ["mse"], # categorical_crossentropy, logcosh], 'last_activation': [keras.activations.relu, keras.activations.elu] } #, #'weight_regulizer':[None]} ptest = { 'lr': [0.1], #'lr': (0.1, 10), 'first_neuron': [2000], 'embedding_size': [1000], 'batch_size': [100], 'epochs': [200], 'noise': [0.1], 'dropout': [0.20], 'hidden_layers': [1], 'activation': [keras.activations.tanh], #'x_val_noise':noisy_vali.values, #'x_train_noise':noisy_train.values, 'optimizer': [keras.optimizers.Adam], 'loss': [ keras.losses.cosine_proximity ], # keras.losses.cosine_proximity categorical_crossentropy, logcosh], 'last_activation': [keras.activations.relu] } # print("scanning") p = { 'lr': (0.1, 1.0, 3), #'lr': (0.1, 10), 'first_neuron': [300], 'embedding_size': [200], 'batch_size': [50], 'epochs': [200], 'noise': [0.5], 'dropout': [ 0, ], 'hidden_layers': [1, 2], #'x_val_noise':noisy_vali.values, #'x_train_noise':noisy_train.values, 'optimizer': [keras.optimizers.Nadam], 'activation': [keras.activations.relu], 'loss': ["mse"], # categorical_crossentropy, logcosh], 'last_activation': [keras.activations.elu] } #, from talos.utils.gpu_utils import force_cpu # Force CPU use on a GPU system force_cpu() #h = ta.Scan(x=train.values,x_val=vali.values, y=train.values,y_val=vali.values, params=pfull, model=dae_model_hl, #dataset_name=model_name,#'dae_test_hiddenlay_model', #experiment_no='4', #grid_downsample=0.012) h = ta.Scan( x=train.values, x_val=vali.values, y=train.values, y_val=vali.values, params=ptest, model=dae_model_hl, dataset_name=model_name, #'dae_test_hiddenlay_model', experiment_no='5') #3grid_downsample=0.012) #h = ta.Scan(x=train.values,x_val=vali.values, y=train.values,y_val=vali.values, params=pfull, model=dae_model, #dataset_name='dae_fullsize_model', #experiment_no='2', #grid_downsample=0.1) print(h.data.head()) # accessing the results data frame print(h.peak_epochs_df) # accessing epoch entropy values for each round print(h.details ) # accessing summary detailspoch entropy values for each round r = ta.Reporting(h) # returns the saved models (json) #print(h.saved_models) #h.saved_models # returns the saved model weights #print(h.saved_weights) #h.saved_models #h.saved_weights # get the number of rounds in the Scan print("number of rounds {}".format(r.rounds())) print("highest validation accuracy {} ".format(r.high())) # get the highest result ('val_acc' by default) #print(r.high()) # get the highest result for any metric print("highest accuracy {}".format(r.high('acc'))) #print(r.high('acc')) #r.plot_hist() # get the round with the best result print("round with best score {}".format(r.rounds2high())) # get the best paramaters print("best paramaters {}".format(r.best_params())) print("deploying best model") deploy_name = experiment_name + "deploy" dep_model = ta.Deploy(h, deploy_name) # get correlation for hyperparameters against a metric #print(r.correlate('val_acc')) print("loading best model and embedding snps") #deploy_load=experiment_name+"deploy" print(deploy_name) #name of deploy zf = zipfile.ZipFile(deploy_name + ".zip", 'r') #accessing zip archive print("loading Json model") json_file = zf.open(deploy_name + '_model.json', 'r') #opening json file for architecture loaded_model_json = json_file.read() json_file.close() print("loading model weights") h5_file = zf.extract( deploy_name + '_model.h5', path='tmp/' ) #extracting weights (not sure how to do it a different way) loaded_model = keras.models.model_from_json(loaded_model_json) loaded_model.load_weights(h5_file) #model fully loaded #encodings=Model(inputs=test.values,outputs=loaded_model.get_layer('bottleneck').output) #print(encodings) print(loaded_model.layers) num_layers = len(loaded_model.layers) #input_img = Input(shape=(input_dim,)) #encoder_layer1 = loaded_model.layers[0] #if num_layers>=5: # encoder_layer2 = loaded_model.layers[1] inputs = Input(shape=(test.shape[1], )) ncomps = loaded_model.get_layer('bottleneck').output_shape[1] print(ncomps) layer_names = [layer.name for layer in loaded_model.layers] #layer_names=[layer_names[i] for i in layer_names if re.search(layer_names[i],Denese encoder_layers = loaded_model.layers[0:int((float(num_layers) / 2))] #accessing middle layer print(encoder_layers.name for layer in encoder_layers) #funct=[layersfor layers in encoders_layers #ifstatement for number of layers print("number of layers") print(len(encoder_layers)) #output_layers=[] #for j in range(len(encoder_layers)): # if j ==0: # output_layers=encoder_layers[0](inputs) # else: # output_layers=output_layers(output_layers) #print(output_layers) #encoder=Model(inputs=inputs, outputs=output_layers) # if len(encoder_layers) == 5: print("5 layers") encoder = Model(inputs=inputs, outputs=encoder_layers[4](encoder_layers[3]( encoder_layers[2](encoder_layers[1]( encoder_layers[0](inputs)))))) elif len(encoder_layers) == 4: print("4 layers") encoder = Model(inputs=inputs, outputs=encoder_layers[3](encoder_layers[2]( encoder_layers[1](encoder_layers[0](inputs))))) elif len(encoder_layers) == 7: print("7 layers") encoder = Model(inputs=inputs, outputs=encoder_layers[6](encoder_layers[5]( encoder_layers[4](encoder_layers[3]( encoder_layers[2](encoder_layers[1]( encoder_layers[0](inputs)))))))) elif len(encoder_layers) == 6: print("6 layers") encoder = Model(inputs=inputs, outputs=encoder_layers[5](encoder_layers[4]( encoder_layers[3](encoder_layers[2]( encoder_layers[1]( encoder_layers[0](inputs))))))) elif len(encoder_layers) == 3: print("3 layers") encoder = Model(inputs=inputs, outputs=encoder_layers[2](encoder_layers[1]( encoder_layers[0](inputs)))) elif len(encoder_layers) == 2: print("2 layers") encoder = Model(inputs=inputs, outputs=encoder_layers[1](encoder_layers[0](inputs))) encoder.summary() encoded = encoder.predict(test) print("dimensions of test set are") print(test.shape) embed_cols = ['embed_%i' % i for i in range(1, ncomps + 1, 1)] print("dimension of embeddings from test set is") print(encoded.shape) embeddings_test = pd.DataFrame(data=encoded, index=test.index, columns=embed_cols) print(embeddings_test.head) print(embeddings_test.shape) embeddings_test.to_csv( "/projects/jgrealey/embedding/ten_k_samples/results/dae/" + experiment_name + ".csv") print( "embeddings_saved at /projects/jgrealey/embedding/ten_k_samples/results/dae/{}.csv" .format(experiment_name)) '''