def talos_model(self, force=False):
"""
Método para retornar um modelo iterativo do Talos. Se um modelo já existir, carregará os pesos e retornará um objeto
de modelo iterativo. Caso não exista um modelo, um será gerado.
:param force: se deve ou não forçar a geração de um modelo fresco
:return: modelo iterativo do talos
"""
model_path = self.MODULE_PATH + "/data/talos/fakedata.zip"
if Path(model_path).is_file() and force is False:
return ta.Restore(model_path)
else:
t = ta.Scan(x=self.X,
y=self.Y,
model=self.fake_news_model,
grid_downsample=.01,
params=self.p,
dataset_name='fakenews',
experiment_no='1')
ta.Deploy(t, str(model_path).replace(".zip", ""))
return t
def get_best_Talos(windows):
df = create_finance(returns=False, plot_corr=False, Trends=False)
for i in windows:
data = prepare_data(df,
i,
10,
10,
returns=False,
normalize_cheat=False)
data.create_windows()
x_train = data.x_train.reshape(len(data.x_train), -1)
y_train = data.y_train.reshape(-1, 1)
x_valid = data.x_valid.reshape(len(data.x_valid), -1)
y_valid = data.y_valid.reshape(-1, 1)
file = 'Talos_Results/ANN_WithoutTrends_stock_window_' + str(i)
print("Running Talos on window size: {}".format(i))
t = ta.Scan(x=x_train,
y=y_train,
x_val=x_valid,
y_val=y_valid,
model=create_model,
params=p,
experiment_name=file,
fraction_limit=0.1,
seed=2)
dpl = file + '_deploy'
with CustomObjectScope({'GlorotUniform': glorot_uniform()}):
ta.Deploy(scan_object=t,
model_name=dpl,
metric='mean_squared_error',
asc=True)
steps_per_epoch=train.n / train.batch_size,
epochs=EPOCHS,
validation_data=valid,
validation_steps=valid.n / valid.batch_size,
callbacks=[ckpt, reduce_lr, early_stopping, tensorboard])
return out, model
if __name__ == '__main__':
x, y, x_val, y_val = [np.array([1, 2]) for i in range(4)]
h = ta.Scan(
x,
y,
params=p,
model=input_model,
#grid_downsample=.1,
#reduction_method='correlation',
x_val=x_val,
y_val=y_val,
val_split=0,
shuffle=False,
last_epoch_value=True,
print_params=True)
# accessing the results data frame
print(h.data.head())
# get the highest result ('val_acc' by default)
print('Best accuracy: ', r.high())
ta.Deploy(h, 'talos_prelim_model_search')
def test_rest(scan_object):
print('\n >>> start testing the rest... \n')
import talos
import random
deploy_filename = 'test' + str(random.randint(1, 20000000000))
print('\n ...Deploy()... \n')
talos.Deploy(scan_object, deploy_filename, 'val_acc')
print('\n ...Restore()... \n')
restored = talos.Restore(deploy_filename + '.zip')
x, y = talos.templates.datasets.breast_cancer()
x = x[:50]
y = y[:50]
x_train, y_train, x_val, y_val = talos.utils.val_split(x, y, .2)
x = talos.utils.rescale_meanzero(x)
callbacks = [
talos.utils.early_stopper(10),
talos.utils.ExperimentLogCallback('test', {})
]
metrics = [
talos.utils.metrics.f1score, talos.utils.metrics.fbeta,
talos.utils.metrics.mae, talos.utils.metrics.mape,
talos.utils.metrics.matthews, talos.utils.metrics.mse,
talos.utils.metrics.msle, talos.utils.metrics.precision,
talos.utils.metrics.recall, talos.utils.metrics.rmae,
talos.utils.metrics.rmse, talos.utils.metrics.rmsle
]
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
print('\n ...callbacks and metrics... \n')
model1 = Sequential()
model1.add(Dense(10, input_dim=x.shape[1]))
model1.add(Dense(1))
model1.compile('adam', 'logcosh', metrics=metrics)
model1.fit(x, y, callbacks=callbacks)
print('\n ...generator... \n')
model2 = Sequential()
model2.add(Dense(10, input_dim=x.shape[1]))
model2.add(Dense(1))
model2.compile('adam', 'logcosh')
model2.fit_generator(talos.utils.generator(x, y, 10), 5)
print('\n ...SequenceGenerator... \n')
model3 = Sequential()
model3.add(Dense(10, input_dim=x.shape[1]))
model3.add(Dense(1))
model3.compile('adam', 'logcosh')
model3.fit_generator(talos.utils.SequenceGenerator(x, y, 10))
print('\n ...gpu_utils... \n')
talos.utils.gpu_utils.force_cpu()
talos.utils.gpu_utils.parallel_gpu_jobs()
print('\n ...gpu_utils... \n')
from talos.utils.test_utils import create_param_space
create_param_space(restored.results, 5)
print('finished testing the rest \n')
# up to two dimensional kernel density estimator
r.plot_kde('val_acc')
# a simple histogram
r.plot_hist(bins=50)
# heatmap correlation
r.plot_corr()
# a four dimensional bar grid
r.plot_bars('batch_size', 'val_acc', 'first_neuron', 'lr')
e = ta.Evaluate(h)
e.evaluate(x, y, folds=10, average='macro')
ta.Deploy(h, 'iris')
iris = ta.Restore('iris.zip')
# make predictions with the model
iris.model.predict(x)
# get the meta-data for the experiment
print(iris.details)
# get the hyperparameter space boundary
print(iris.params)
# sample of x and y data
print(iris.x)
print(iris.y)
# the results dataframe
print(iris.results)
# Scan
scan_object = test_scan_object()
# Reporting
test_reporting_object(scan_object)
start_time = str(time.strftime("%s"))
p = ta.Predict(scan_object)
p.predict(scan_object.x)
p.predict_classes(scan_object.x)
ta.Autom8(scan_object, scan_object.x, scan_object.y)
ta.Evaluate(scan_object)
ta.Deploy(scan_object, start_time)
ta.Restore(start_time + '.zip')
test_random_methods()
fit_generator = generator(scan_object.x, scan_object.y, 20)
force_cpu()
TestCancer().test_scan_cancer_metric_reduction()
TestCancer().test_scan_cancer_loss_reduction()
TestCancer().test_linear_method()
TestCancer().test_reverse_method()
TestIris().test_scan_iris_explicit_validation_set()
TestIris().test_scan_iris_explicit_validation_set_force_fail()
TestIris().test_scan_iris_1()
TestIris().test_scan_iris_2()
def main():
expert_path = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/Push_49_ep/'
# r_model = Restore('imitation_learning.zip')
heat_maps = np.load(expert_path + 'heatmap.npy')
states = np.load(expert_path + 'states.npy')
actions = np.load(expert_path + 'actions.npy')
# p = {'hist_size': [1, 2, 3, 4, 5],
# 'input_conv_kernel_size': [(2, 2), (3, 3)],
# 'input_conv_filters': [8, 16, 32],
# 'conv_dropout': [0.0, 0.1, 0.25],
# 'conv_layers': [1, 2, 3],
# 'conv_kernel_size' : [(2, 2), (3, 3)],
# 'conv_filters' : [8, 16, 32, 64],
# 'conv_strides' : [(1, 1),(2, 2)],
# 'batch_norm' : [True, False],
# 'output_conv_kernel_size' : [(2, 2), (3, 3)],
# 'output_conv_filters' :[32, 64],
# 'input_layer_size' : [32, 64, 128],
# 'hidden_layers' : [1, 2, 3],
# 'layer_size' : [32, 64, 128],
# 'output_layer_size': [16, 32, 64, 128],
# 'bias': [True, False],
# 'batch_size': [16, 32, 64],
# 'learning_rate': [1e-4, 5e-4, 1e-5, 5e-5],
# 'epochs' : [500],
# 'optimizer' : ['adam', 'nadam', 'sgd'],
# 'activations' : ['relu', 'elu', 'sigmoid'],
# 'output_activations' : ['relu', 'elu', 'sigmoid']}
hist_size = 1
#
# heat_map_hist = []
#
# for kk in range(len(heat_maps) - hist_size):
# heat_map_hist.append(heat_maps[kk:kk + hist_size, :, :])
#
# heat_maps = np.array(heat_map_hist)
# actions = actions[hist_size:]
heat_maps = heat_maps.reshape(len(heat_maps), hist_size, 100, 6)
# p = {'input_conv_kernel_size': [(2, 2), (3, 3)],
# 'input_conv_filters': [8, 16, 32],
# 'conv_dropout': [0.0, 0.25],
# 'conv_layers': [1, 2],
# 'conv_kernel_size': [(2, 2), (3, 3)],
# 'conv_filters': [8, 16, 32],
# 'conv_strides': [(1, 1), (2, 2)],
# 'batch_norm': [False],
# 'output_conv_kernel_size': [(2, 2), (3, 3), (4, 4)],
# 'output_conv_filters': [16, 32, 64],
# 'input_layer_size' : [64, 128, 256],
# 'hidden_layers': [1, 2, 3],
# 'layer_size': [64, 128, 256],
# 'output_layer_size': [16, 32, 64],
# 'output_bias': [False, True],
# 'batch_size': [64],
# 'learning_rate': [1e-4, 2.5e-4, 5e-4],
# 'epochs': [400],
# 'optimizer': ['adam'],
# 'activations': ['relu'],
# 'output_activations': ['sigmoid']}
#
p = {
'input_conv_kernel_size': [(2, 2)],
'input_conv_filters': [8],
'conv_dropout': [0.0],
'conv_layers': [1],
'conv_kernel_size': [(3, 3)],
'conv_filters': [32],
'conv_strides': [(1, 1)],
'batch_norm': [False],
'output_conv_kernel_size': [(3, 3)],
'output_conv_filters': [64],
'input_layer_size': [256],
'hidden_layers': [1],
'layer_size': [256],
'output_layer_size': [64],
'output_bias': [False],
'batch_size': [64],
'learning_rate': [5e-4],
'epochs': [400],
'optimizer': ['adam'],
'activations': ['relu'],
'output_activations': ['sigmoid']
}
# imitation_learning(heat_maps, actions, params=p)
t = talos.Scan(x=heat_maps,
y=actions,
model=imitation_learning,
params=p,
experiment_name='BC_HM_Push')
talos.Deploy(scan_object=t,
model_name='Push_BC_best_vl',
metric='val_loss',
asc=True)
talos.Deploy(scan_object=t,
model_name='Push_BC_best_l',
metric='loss',
asc=True)
experiment_name='CNN_Optimization',
round_limit=10,
fraction_limit=0.05)
cnn_analyze = talos.Analyze(cnn_scan)
documentation_file_parameteropt.write(
"CNN: Best parameters {}, reached score: {} \n".format(
cnn_analyze.best_params('accuracy', ['accuracy', 'loss', 'val_loss']),
cnn_analyze.high('accuracy')))
pred_cnn = talos.Predict(cnn_scan).predict(x_t, metric='val_f1score', asc=True)
#evaluate the model
cnn_evaluation_scores, cnn_cm = evaluation.multilabel_evaluation(
d_test_array, label_binarizer.inverse_transform(pred_cnn), "CNN")
documentation_file_modelopt.write(cnn_evaluation_scores)
#deploy best model
model_cnn = talos.Deploy(cnn_scan,
"model_cnn_transformerxl",
metric='val_accuracy')
#build LSTM model and evaluate the model
print("LSTM model evaluation")
def lstm_optimization(x_train, y_train, x_test, y_test, params):
"""Randomized search to optimize parameters of Neural Network."""
optimization_model = models.Sequential()
optimization_model.add(layers.LSTM(params['units'], return_sequences=True))
optimization_model.add(layers.LSTM(params['units'],
return_sequences=False))
optimization_model.add(layers.Dropout(0.5))
optimization_model.add(layers.Dense(params['dense'], activation=relu))
optimization_model.add(layers.Dense(params['dense'], activation=relu))
out = model.fit_generator(
train,
steps_per_epoch=train.n / train.batch_size,
epochs=EPOCHS,
validation_data=valid,
validation_steps=valid.n / valid.batch_size,
callbacks=[ckpt, reduce_lr, early_stopping, tensorboard])
return out, model
if __name__ == '__main__':
# workaround to feed data in batches instead of loading into memory as talos requires
x, y, x_val, y_val = [np.array([1, 2]) for i in range(4)]
h = ta.Scan(
x,
y,
params=p,
model=input_model,
#grid_downsample=.1,
#reduction_method='correlation',
x_val=x_val,
y_val=y_val,
val_split=0,
shuffle=False,
last_epoch_value=True,
print_params=True)
ta.Deploy(h, 'talos_beauty_inception_resnet_transfer')
print(h.data.head())
print(h.details)
params=cnn_params,
experiment_name='CNN_Optimization',
round_limit=10,
fraction_limit=0.05)
cnn_analyze = talos.Analyze(cnn_scan)
documentation_file_parameteropt.write(
"CNN: Best parameters {}, reached score: {} \n".format(
cnn_analyze.best_params('accuracy', ['accuracy', 'loss', 'val_loss']),
cnn_analyze.high('accuracy')))
pred_cnn = talos.Predict(cnn_scan).predict(x_t, metric='val_f1score', asc=True)
#evaluate the model
cnn_evaluation_scores, cnn_cm = evaluation.multilabel_evaluation(
d_test_array, label_binarizer.inverse_transform(pred_cnn), "CNN")
documentation_file_modelopt.write(cnn_evaluation_scores)
#deploy best model
model_cnn = talos.Deploy(cnn_scan, "model_cnn_scibert", metric='val_accuracy')
#build LSTM model and evaluate the model
print("LSTM model evaluation")
def lstm_optimization(x_train, y_train, x_test, y_test, params):
"""Randomized search to optimize parameters of Neural Network."""
optimization_model = models.Sequential()
optimization_model.add(layers.LSTM(params['units'], return_sequences=True))
optimization_model.add(layers.LSTM(params['units'],
return_sequences=False))
optimization_model.add(layers.Dropout(0.5))
optimization_model.add(layers.Dense(params['dense'], activation=relu))
optimization_model.add(layers.Dense(params['dense'], activation=relu))
optimization_model.add(layers.Dense(int(num_classes),
'activation': ['relu'],
'kernel_initializer': ['glorot_uniform'],
'h_nodes': [100, 128],
'kernel_size': [3, 5, 9]
}
##probabilistic reduction based on correlation
t_nonrandom_downsampling = talos.Scan(x=x_train,
y=y_train,
model=cnn_model,
dataset_name='isic_hpt',
experiment_no='2',
params=parameters_small,
reduction_method='correlation',
reduction_interval=10,
reduction_window=10,
grid_downsample=1,
talos_log_name='hpt_nonrandom')
try:
talos.Deploy(t_nonrandom_downsampling, 't_nonrandom_tuning')
except:
print(" ")
OutputResults(x_train, y_train,
't_nonrandom_tuning/t_nonrandom_tuning_model.json',
't_nonrandom_tuning/t_nonrandom_tuning_model.h5')
OutputResults(x_test, y_test,
't_nonrandom_tuning/t_nonrandom_tuning_model.json',
't_nonrandom_tuning/t_nonrandom_tuning_model.h5')
def test_rest(scan_object):
print('\n >>> start testing the rest... \n')
import talos
import random
print('\n ...Deploy()... \n')
talos.Deploy(scan_object, 'testing_deploy', 'val_acc')
print('\n ...Restore()... \n')
talos.Restore('testing_deploy' + '.zip')
x, y = talos.templates.datasets.breast_cancer()
x = x[:50]
y = y[:50]
callbacks = [
talos.utils.early_stopper(10),
talos.utils.ExperimentLogCallback('test', {})
]
metrics = [
talos.utils.metrics.f1score, talos.utils.metrics.fbeta,
talos.utils.metrics.mae, talos.utils.metrics.mape,
talos.utils.metrics.matthews, talos.utils.metrics.mse,
talos.utils.metrics.msle, talos.utils.metrics.precision,
talos.utils.metrics.recall, talos.utils.metrics.rmae,
talos.utils.metrics.rmse, talos.utils.metrics.rmsle
]
from keras.models import Sequential
from keras.layers import Dense
print('\n ...callbacks and metrics... \n')
model1 = Sequential()
model1.add(Dense(10, input_dim=x.shape[1]))
model1.add(Dense(1))
model1.compile('adam', 'logcosh', metrics=metrics)
model1.fit(x, y, callbacks=callbacks)
print('\n ...generator... \n')
model2 = Sequential()
model2.add(Dense(10, input_dim=x.shape[1]))
model2.add(Dense(1))
model2.compile('adam', 'logcosh')
model2.fit_generator(talos.utils.generator(x, y, 10), 5)
print('\n ...SequenceGenerator... \n')
model3 = Sequential()
model3.add(Dense(10, input_dim=x.shape[1]))
model3.add(Dense(1))
model3.compile('adam', 'logcosh')
model3.fit_generator(talos.utils.SequenceGenerator(x, y, 10))
print('\n ...gpu_utils... \n')
talos.utils.gpu_utils.force_cpu()
talos.utils.gpu_utils.parallel_gpu_jobs()
print('finised testing the rest \n')
out = model.fit_generator(
train,
steps_per_epoch=train.n / train.batch_size,
epochs=EPOCHS,
validation_data=valid,
validation_steps=valid.n / valid.batch_size,
callbacks=[ckpt, reduce_lr, early_stopping, tensorboard])
return out, model
if __name__ == '__main__':
# workaround to feed data in batches instead of loading into memory as talos requires
x, y, x_val, y_val = [np.array([1, 2]) for i in range(4)]
h = ta.Scan(
x,
y,
params=p,
model=input_model,
#grid_downsample=.1,
#reduction_method='correlation',
x_val=x_val,
y_val=y_val,
val_split=0,
shuffle=False,
last_epoch_value=True,
print_params=True)
ta.Deploy(h, 'talos_fashion_inception_resnet_transfer')
print(h.data.head())
print(h.details)
def main():
expert_path_1 = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/no_clip/all_recs_no_peel/'
expert_path_2 = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/no_clip/lift_23_ep/'
# expert_path = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/sand_levels_RHM/y_clip/'
states_1 = np.load(expert_path_1+'states.npy', allow_pickle=True)
states_2 = np.load(expert_path_2+'states.npy', allow_pickle=True)
states = np.load(expert_path_2 + 'states.npy', allow_pickle=True)
inputs = np.concatenate([np.stack(states[:, 1]), np.array(states[:, 5]).reshape(len(states), 1)], axis=1)
labels = np.array(states[:, 4])
# p = {'hist_size': [1, 2, 3, 4, 5],
# 'input_conv_kernel_size': [(2, 2), (3, 3)],
# 'input_conv_filters': [8, 16, 32],
# 'conv_dropout': [0.0, 0.1, 0.25],
# 'conv_layers': [1, 2, 3],
# 'conv_kernel_size': [(2, 2), (3, 3)],
# 'conv_filters': [8, 16, 32, 64],
# 'conv_strides': [(1, 1), (2, 2)],
# 'batch_norm': [True, False],
# 'output_conv_kernel_size': [(2, 2), (3, 3)],
# 'output_conv_filters': [32, 64],
# 'input_layer_size': [32, 64, 128],
# 'hidden_layers': [1, 2, 3],
# 'layer_size': [32, 64, 128],
# 'output_layer_size': [16, 32, 64, 128],
# 'output_bias': [True, False],
# 'batch_size': [16, 32, 64],
# 'learning_rate': [1e-4, 5e-4, 1e-5, 5e-5],
# 'epochs': [500],
# 'optimizer': ['adam', 'nadam', 'sgd'],
# 'activations': ['relu', 'elu', 'sigmoid'],
# 'output_activations': ['relu', 'elu', 'sigmoid']}
# p = {'input_conv_kernel_size': [(2, 2), (3, 3)],
# 'input_conv_filters': [8, 16, 32],
# 'conv_dropout': [0.0, 0.25],
# 'conv_layers': [1, 2],
# 'conv_kernel_size': [(2, 2), (3, 3)],
# 'conv_filters': [8, 16, 32],
# 'conv_strides': [(1, 1), (2, 2)],
# 'batch_norm': [False],
# 'output_conv_kernel_size': [(2, 2), (3, 3), (4, 4)],
# 'output_conv_filters': [16, 32, 64],
# 'input_layer_size' : [64, 128, 256],
# 'hidden_layers': [1, 2, 3],
# 'layer_size': [64, 128, 256],
# 'output_layer_size': [16, 32, 64],
# 'output_bias': [False, True],
# 'batch_size': [64],
# 'learning_rate': [1e-4, 2.5e-4, 5e-4],
# 'epochs': [400],
# 'optimizer': ['adam'],
# 'activations': ['relu'],
# 'output_activations': ['sigmoid']}
# fract_lim = 0.00001
p = {'input_conv_kernel_size': [(3, 3)],
'input_conv_filters': [16],
'conv_dropout': [0.25],
'conv_layers': [2],
'conv_kernel_size': [(2, 2)],
'conv_filters': [16],
'conv_strides': [(1, 1)],
'batch_norm': [False],
'output_conv_kernel_size': [(3, 3)],
'output_conv_filters': [16],
'input_layer_size': [256],
'hidden_layers': [1],
'layer_size': [256],
'output_layer_size': [32],
'output_bias': [False],
'batch_size': [64],
'learning_rate': [1e-5],
'epochs': [1000],
'optimizer': ['adam'],
'activations': ['relu'],
'output_activations': ['sigmoid']}
fract_lim = 1
learn = True
test_name = 'new_test_new_recordings_LP_model_10_pred'
if learn:
# labels = actions
# aug_heat_maps = heat_maps.reshape(heat_maps.shape[0],1,heat_maps.shape[1],heat_maps.shape[2])
t = talos.Scan(x=aug_heat_maps, y=labels, model=supervised_learning, params=p, fraction_limit=fract_lim, experiment_name=test_name)
talos.Deploy(scan_object=t, model_name=test_name+'_best_vl', metric='val_loss', asc=True)
else:
json_file = open('/home/graphics/git/SmartLoader/'+test_name+'_best_vl/'+test_name+'_best_vl_model.json', 'rb')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
# load weights into new model
model.load_weights('/home/graphics/git/SmartLoader/'+test_name+'_best_vl/'+test_name+'_best_vl_model.h5')
print(' ------------ now lets evaluate -------------')
evaluate = False
if evaluate:
loss = []
evals = 50
for k in range(evals):
index = np.random.randint(len(aug_heat_maps))
states = model.predict(aug_heat_maps[index, :, :, :].reshape(1, 1, aug_heat_maps.shape[2], aug_heat_maps.shape[3]))
arm_height = states[0][0]*50+150
plt.imshow(aug_heat_maps[index, :, :, :].squeeze(), aspect=1)
plt.text(40, 5, arm_height, fontsize=15)
print(labels[index][0] * 50 + 150)
plt.show()
model.save('/home/graphics/git/SmartLoader/saved_models/'+test_name)
print('saved')
lr=lr_normalizer(params['lr'], params['optimizer'])),
loss=loss_fx,
metrics=metrics,
class_weight=class_weight)
out = model.fit_generator(
imageLoader(sub_train_images, y_train, params['batch_size']),
steps_per_epoch=sub_train_images.shape[0] // params['batch_size'],
epochs=20,
validation_data=imageLoader(sub_val_images, y_val,
params['batch_size']),
validation_steps=sub_val_images.shape[0] // params['batch_size'],
callbacks=[es, mc],
verbose=2)
#print(f"out:{out.history.keys()}")
return out, model
#print(f"y_train[0]: {y_train[0]}")
# hyperparameter optimization
t = ta.Scan(x=sub_train_images,
y=y_train,
x_val=sub_val_images,
y_val=y_val,
params=p,
model=talos_model,
experiment_name='exp_' + exp_id)
ta.Deploy(t, exp_id, metric="val_accuracy", asc=False)
fraction_limit=1,
#time_limit = '2019-09-25 10:00',
seed=Config.random_seed)
f = open("Talos_scan_object.pickle", "wb+")
pickle.dump(scan_object, f)
f.close()
#from talos import Evaluate
#
# create the evaluate object
#e = Evaluate(scan_object)
#
## perform the evaluation
#
#tcga_x = np.expand_dims(Data.all_data, axis=2)
#tcga_y = Data.integer_encoded
#####evaluate change the one hot encode to int encode, and not mentioned in talos doc.....
#print('Cross validation:')
#scan_object.evaluate_models(x=tcga_x,
# y=tcga_y,
# folds = 10,
# metric='val_acc',
# )
out = 'cnn_1d_tumor_type_t_' + Config.treat
if os.path.isdir(out):
shutil.rmtree(out)
ta.Deploy(scan_object, out, metric='val_acc')
print("--- %s seconds ---" % (tt.time() - start_time))
def main():
global y_map_clip
expert_path = '/home/graphics/git/SmartLoader/saved_experts/HeatMap/real_life/lift_ep_12_5/'
lift_est_model = load_model('/home/graphics/git/SmartLoader/saved_models/lift_est_model_corrected')
heat_maps = np.load(expert_path+'heatmaps.npy')
states = np.load(expert_path+'states.npy', allow_pickle=True)
actions = np.load(expert_path+'actions.npy')
ep_starts = np.load(expert_path+'starts.npy')
ep_starts[0] = True
# rand_ind = np.random.randint(len(states), size=100)
# fictive_maps = np.array([np.ones(heat_maps[0].shape)*0.1]*100)
# fictive_states = states[rand_ind]
# fictive_states[:, 5] = 165
# fictive_states[:, 4] = 140
# fictive_ep_starts = ep_starts[rand_ind]
# heat_maps = np.vstack([heat_maps, fictive_maps])
# states = np.vstack([states, fictive_states])
# ep_starts = np.hstack([ep_starts, fictive_ep_starts])
labels = []
aug_heat_maps = []
########################################
# organize data to fit mission:
########################################
label_horizon = 5 # how far to the future will the network predict
num_of_step_labels = 5 # how many future steps will the network predict
x_map_front_clip = 100 # window size in front of blade
x_map_front_offset = -60 # offset size in front of blade
y_map_clip = 30 # window size on sides of the blade
learn = False
test_name = 'lift_task_LP_model_5_pred_b_wind'
if learn:
for k in range(label_horizon+300, len(states)):
state_pred = []
if ep_starts[(k - label_horizon+1):(k + 1)].any():
continue
# aug_map = map_clipper(heat_maps[k], states[k, 1], x_map_front_clip,
# x_map_front_offset, y_map_clip)
# t_lift_shovle_pos, t_lift_pitch_state = lift_normalize_states(states[k][1], states[k][5])
# t_lift_state = lift_est_model.predict(np.hstack([t_lift_shovle_pos, t_lift_pitch_state]).reshape(1, 4))
# show_heatmap(aug_map, t_lift_state)
for j in range(k-num_of_step_labels+1, k+1):
shovle_pos = states[j][1]
pitch_state = states[j][[5]] # only pitch values
lift_shovle_pos, lift_pitch_state = lift_normalize_states(shovle_pos, pitch_state)
lift_state = lift_est_model.predict(np.hstack([lift_shovle_pos, lift_pitch_state]).reshape(1, 4))
shovle_pos, pitch_state = map_normalize_states(shovle_pos, pitch_state)
# aug_states = [lift_state, pitch_state] ## for LP model
aug_states = shovle_pos[0] ## for x model
# aug_states[0:3] -= states[k - label_horizon][1] ## for moving set-point
# aug_states = actions[j,1] ## for thrust model
state_pred.append(aug_states)
aug_map = map_clipper(heat_maps[k-label_horizon], states[k-label_horizon, 1], x_map_front_clip, x_map_front_offset, y_map_clip)
state_pred = np.array(state_pred).squeeze()
# show_heatmap(heat_maps[k - label_horizon], state_pred[-1][0] * 100 + 150)
# show_heatmap(aug_map, state_pred[-1][0]*100+150)
if not aug_map.any():
continue
labels.append(np.array(state_pred).reshape(np.prod(state_pred.shape)))
aug_heat_maps.append(aug_map)
# if (k % 20) == 0:
# plt.close()
# if ep_starts[k+label_horizon]:
# plt.close()
aug_heat_maps = np.array(aug_heat_maps)
aug_heat_maps = aug_heat_maps.reshape([aug_heat_maps.shape[0], 1, aug_heat_maps.shape[1], aug_heat_maps.shape[2]])
labels = np.array(labels)
p = {'input_conv_kernel_size': [(3, 3)],
'input_conv_filters': [16],
'conv_dropout': [0.25],
'conv_layers': [2],
'conv_kernel_size': [(2, 2)],
'conv_filters': [16],
'conv_strides': [(1, 1)],
'batch_norm': [False],
'output_conv_kernel_size': [(3, 3)],
'output_conv_filters': [16],
'input_layer_size': [256],
'hidden_layers': [1],
'layer_size': [256],
'output_layer_size': [32],
'output_bias': [False],
'batch_size': [64],
'learning_rate': [5e-5],
'epochs': [200],
'optimizer': ['adam'],
'activations': ['relu'],
'output_activations': ['sigmoid']}
fract_lim = 1
# labels = actions
# aug_heat_maps = heat_maps.reshape(heat_maps.shape[0],1,heat_maps.shape[1],heat_maps.shape[2])
t = talos.Scan(x=aug_heat_maps, y=labels, model=supervised_learning, params=p, fraction_limit=fract_lim, experiment_name=test_name)
talos.Deploy(scan_object=t, model_name=test_name+'_best_vl', metric='val_loss', asc=True)
else:
json_file = open('/home/graphics/git/SmartLoader/'+test_name+'_best_vl/'+test_name+'_best_vl_model.json', 'rb')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
# load weights into new model
model.load_weights('/home/graphics/git/SmartLoader/'+test_name+'_best_vl/'+test_name+'_best_vl_model.h5')
print(' ------------ now lets evaluate -------------')
evaluate = False
if evaluate:
loss = []
evals = 50
for k in range(evals):
index = np.random.randint(len(aug_heat_maps))
states = model.predict(aug_heat_maps[index, :, :, :].reshape(1, 1, aug_heat_maps.shape[2], aug_heat_maps.shape[3]))
arm_height = states[0][0]*50+150
plt.imshow(aug_heat_maps[index, :, :, :].squeeze(), aspect=1)
plt.text(40, 5, arm_height, fontsize=15)
print(labels[index][0] * 50 + 150)
plt.show()
model.save('/home/graphics/git/SmartLoader/saved_models/'+test_name)
print('saved')
verbose=1)
test_predict = model.predict_generator(
generator=rescale_generator.flow(x=test_X,
batch_size=predict_batch_size,
shuffle=False),
steps=test_X.shape[0] / predict_batch_size,
verbose=1)
# Write train, val, and test predictions to file
for i, predictions in enumerate([train_predict, val_predict, test_predict]):
split = ["train", "val", "test"][i]
with open(experiment_name + '_' + split + 'Predictions.txt',
'w') as predictOutput:
for i in range(len(predictions)):
_ = predictOutput.write(f"{predictions[i][0]}\n")
for i, trues in enumerate([train_y, val_y, test_y]):
split = ["train", "val", "test"][i]
with open(experiment_name + '_' + split + 'True.txt', 'w') as trueOutput:
for i in range(len(trues)):
_ = trueOutput.write(f"{trues[i]}\n")
# Save best model
scan_object.x = np.zeros(500) # necessary for model to restore properly
scan_object.y = np.zeros(500)
ta.Deploy(scan_object,
experiment_name + "_hyperopt_models",
metric="val_acc",
asc=False)
return out, model
#STEP 4. SCANNING THE HYPER PARAMETER SPACE
import talos
#Random reduction
t_random = talos.Scan(x=x_train,
y=y_trainHot,
model=cnn_model,
dataset_name='wbc',
experiment_no='2',
params=hyperP,
grid_downsample=0.5)
talos.Deploy(t_random, "talos_random4_tuning", metric='accuracy')
#try:
# talos.Deploy(t_random, "talos_random3_tuning", metric='val_acc')
#except:
# print(" ")
def OutputResults(x_data, y_data, bestmodel_json_file,
bestmodel_weights_h5_file):
from keras.models import model_from_json
file = open(bestmodel_json_file, 'r')
lines = file.read()
file.close()
model = model_from_json(lines)
model.load_weights(bestmodel_weights_h5_file)
def main():
from keras import backend as K
K.set_session(
K.tf.Session(config=K.tf.ConfigProto(intra_op_parallelism_threads=32,
inter_op_parallelism_threads=32)))
print("using 32 CPUs or 64 Cores on cluster")
if len(sys.argv) != 3:
print("incorrect number of arguments - exiting program")
sys.exit(0)
s = random.getstate()
snp_file = str(sys.argv[1])
print("filename for SNPs {}".format(snp_file))
#pheno_file=str(sys.argv[2])
#print("filename for phenotype is {}".format(pheno_file))
path_input = "/home/jgrealey/Simulations/ten_k_samples/test/"
#Maybe i want the standardised genotype
#path_pheno="/home/jgrealey/Simulations/ten_k_samples/test/"
results_dir = "/projects/jgrealey/embedding/ten_k_samples/results/dae/"
plots_dir = "/projects/jgrealey/embedding/ten_k_samples/plots/dae/"
numsam = 1000
nphenotoload = numsam - 3 #
listtoload = range(numsam)
numsnps = 300
testcols = np.arange(100)
datafile = snp_file
model_name = str(sys.argv[2]) #"dae_test_hiddenlay_model"
experiment_name = model_name + "_" + snp_file
print(experiment_name)
print(model_name)
#'weight_regulizer':[None]}
#print(listtoload)
#for x, rows are snps and columns are samples so usecoles
#x=pd.read_csv(path_input+snp_file+".csv",index_col=0,usecols=listtoload,nrows=numsnps)#rows are snps#nrows=100)#testing for faster I/O
#full laod
#cols_read=pd.read_csv(path_input+snp_file+".csv",usecols=[0])
#print(cols_read)
#sys.texit(0)
#full load
x = pd.read_csv(
path_input + snp_file + ".csv", index_col=0, dtype='object'
) #,usecols=listtoload,nrows=numsnps)#rows are snps#nrows=100)#testing for faster I/O
#testing
#TESTING#
#x=pd.read_csv(path_input+snp_file+".csv",index_col=0,usecols=listtoload)#,nrows=numsnps)#rows are snps#nrows=100)#testing for faster I/O
#ONLY TRANSPOSE IF READING GENOTYPE FILE
#STANDARDISED FILE DOESNT NEED TO BE TRANSPOSED
#x=x.T
#TESTING#
#y=pd.read_csv(path_pheno+pheno_file+".csv",index_col=0,header=None)#,dtype='float')#tasting for faster I/O
#full load
#y=pd.read_csv(path_pheno+pheno_file+".csv",index_col=0,nrows=nphenotoload,header=None)#tasting for faster I/O
#y.columns=["phenotype"]
#print(x.tail())
#print(y.tail())
index = x.index
cols = x.columns
#print(index)
#print(cols)
samples = index[2:-3]
testsamples = index[2:]
#print(testsamples)
#print(y)
#xnorm=normalise(x,[2,-3])
#del x
#x=xnorm
#print(xnorm)
genotypes = x.loc[samples]
genotypes = genotypes.astype(np.float) #pd.to_numeric(genotypes)
#print(genotypes)
#noisy_genotypes=mask_function(dataframe=genotypes,noise=0.2,batch_size=100)
#print(noisy_genotypes)
#genotypes=x.loc[testsamples]
# calculate sparsity
#sparsity = 1.0 - np.count_nonzero(genotypes) /genotypes.size
#print(sparsity)0.7520695419592109
labels = x.loc[index[-1]].astype('int')
#labels.index=xnorm.index
#print(labels)
beta = x.loc[index[-2]].astype('float')
maf = x.loc[index[-3]].astype('float')
SNP_id = x.loc[index[0]]
SNP_pos = x.loc[index[1]]
#print(genotypes)
#df=genotypes.join(y)
#print(df)
#train, test = train_test_split(df, test_size=0.60, random_state=42)#40% used in training
train, test = train_test_split(genotypes, test_size=0.80,
random_state=42) #40% used in training
train_samples = train.index
#noisy_train=mask_function(dataframe=train,noise=0.3,batch_sizes=100)
vali, test = train_test_split(test, test_size=0.9, random_state=42) #
vali_samples = vali.index
test_samples = test.index
#noisy_vali=mask_function(dataframe=vali,noise=0.3,batch_sizes=100)
noisy_test = mask_function(dataframe=test, noise=0.3, batch_sizes=100)
#print(train.head)
#print(noisy_train.head)
#print(train_samples)
#print(test_samples)
#print(train.loc[train_samples])
#print(train.shape)#(4000, 1000
#print(noisy_train.shape)
#print(test.shape)#(5400, 1000)
#print(vali.shape)#(600, 1000)
#sys.exit(0)
#print(train[samples])
#print(test)
#print(df)
num_dum = 100
numSNPs = train.shape[1]
print(numSNPs)
pfull = {
'lr': (0.07, 1.0, 5), #'lr': (0.1, 10),
'first_neuron': [200, 500, 1000],
'embedding_size': [250, 500, 750],
'batch_size': [50, 100],
'epochs': [200],
'noise': [0.3, 0.2],
'dropout': [0, 0.40],
'hidden_layers': [0, 1, 2],
'activation': [keras.activations.relu, keras.activations.elu],
#'x_val_noise':noisy_vali.values,
#'x_train_noise':noisy_train.values,
'optimizer': [
keras.optimizers.Adam, keras.optimizers.RMSprop,
keras.optimizers.Adadelta
],
'loss': ["mse"], # categorical_crossentropy, logcosh],
'last_activation': [keras.activations.relu, keras.activations.elu]
} #,
#'weight_regulizer':[None]}
ptest = {
'lr': [0.1], #'lr': (0.1, 10),
'first_neuron': [2000],
'embedding_size': [1000],
'batch_size': [100],
'epochs': [200],
'noise': [0.1],
'dropout': [0.20],
'hidden_layers': [1],
'activation': [keras.activations.tanh],
#'x_val_noise':noisy_vali.values,
#'x_train_noise':noisy_train.values,
'optimizer': [keras.optimizers.Adam],
'loss': [
keras.losses.cosine_proximity
], # keras.losses.cosine_proximity categorical_crossentropy, logcosh],
'last_activation': [keras.activations.relu]
} #
print("scanning")
p = {
'lr': (0.1, 1.0, 3), #'lr': (0.1, 10),
'first_neuron': [300],
'embedding_size': [200],
'batch_size': [50],
'epochs': [200],
'noise': [0.5],
'dropout': [
0,
],
'hidden_layers': [1, 2],
#'x_val_noise':noisy_vali.values,
#'x_train_noise':noisy_train.values,
'optimizer': [keras.optimizers.Nadam],
'activation': [keras.activations.relu],
'loss': ["mse"], # categorical_crossentropy, logcosh],
'last_activation': [keras.activations.elu]
} #,
from talos.utils.gpu_utils import force_cpu
# Force CPU use on a GPU system
force_cpu()
#h = ta.Scan(x=train.values,x_val=vali.values, y=train.values,y_val=vali.values, params=pfull, model=dae_model_hl,
#dataset_name=model_name,#'dae_test_hiddenlay_model',
#experiment_no='4',
#grid_downsample=0.012)
h = ta.Scan(
x=train.values,
x_val=vali.values,
y=train.values,
y_val=vali.values,
params=ptest,
model=dae_model_hl,
dataset_name=model_name, #'dae_test_hiddenlay_model',
experiment_no='5')
#3grid_downsample=0.012)
#h = ta.Scan(x=train.values,x_val=vali.values, y=train.values,y_val=vali.values, params=pfull, model=dae_model,
#dataset_name='dae_fullsize_model',
#experiment_no='2',
#grid_downsample=0.1)
print(h.data.head()) # accessing the results data frame
print(h.peak_epochs_df) # accessing epoch entropy values for each round
print(h.details
) # accessing summary detailspoch entropy values for each round
r = ta.Reporting(h)
# returns the saved models (json)
#print(h.saved_models)
#h.saved_models
# returns the saved model weights
#print(h.saved_weights)
#h.saved_models
#h.saved_weights
# get the number of rounds in the Scan
print("number of rounds {}".format(r.rounds()))
print("highest validation accuracy {} ".format(r.high()))
# get the highest result ('val_acc' by default)
#print(r.high())
# get the highest result for any metric
print("highest accuracy {}".format(r.high('acc')))
#print(r.high('acc'))
#r.plot_hist()
# get the round with the best result
print("round with best score {}".format(r.rounds2high()))
# get the best paramaters
print("best paramaters {}".format(r.best_params()))
print("deploying best model")
deploy_name = experiment_name + "deploy"
dep_model = ta.Deploy(h, deploy_name)
# get correlation for hyperparameters against a metric
#print(r.correlate('val_acc'))
print("loading best model and embedding snps")
#deploy_load=experiment_name+"deploy"
print(deploy_name) #name of deploy
zf = zipfile.ZipFile(deploy_name + ".zip", 'r') #accessing zip archive
print("loading Json model")
json_file = zf.open(deploy_name + '_model.json',
'r') #opening json file for architecture
loaded_model_json = json_file.read()
json_file.close()
print("loading model weights")
h5_file = zf.extract(
deploy_name + '_model.h5', path='tmp/'
) #extracting weights (not sure how to do it a different way)
loaded_model = keras.models.model_from_json(loaded_model_json)
loaded_model.load_weights(h5_file) #model fully loaded
#encodings=Model(inputs=test.values,outputs=loaded_model.get_layer('bottleneck').output)
#print(encodings)
print(loaded_model.layers)
num_layers = len(loaded_model.layers)
#input_img = Input(shape=(input_dim,))
#encoder_layer1 = loaded_model.layers[0]
#if num_layers>=5:
# encoder_layer2 = loaded_model.layers[1]
inputs = Input(shape=(test.shape[1], ))
ncomps = loaded_model.get_layer('bottleneck').output_shape[1]
print(ncomps)
layer_names = [layer.name for layer in loaded_model.layers]
#layer_names=[layer_names[i] for i in layer_names if re.search(layer_names[i],Denese
encoder_layers = loaded_model.layers[0:int((float(num_layers) /
2))] #accessing middle layer
print(encoder_layers.name for layer in encoder_layers)
#funct=[layersfor layers in encoders_layers
#ifstatement for number of layers
print("number of layers")
print(len(encoder_layers))
#output_layers=[]
#for j in range(len(encoder_layers)):
# if j ==0:
# output_layers=encoder_layers[0](inputs)
# else:
# output_layers=output_layers(output_layers)
#print(output_layers)
#encoder=Model(inputs=inputs, outputs=output_layers)
#
if len(encoder_layers) == 5:
print("5 layers")
encoder = Model(inputs=inputs,
outputs=encoder_layers[4](encoder_layers[3](
encoder_layers[2](encoder_layers[1](
encoder_layers[0](inputs))))))
elif len(encoder_layers) == 4:
print("4 layers")
encoder = Model(inputs=inputs,
outputs=encoder_layers[3](encoder_layers[2](
encoder_layers[1](encoder_layers[0](inputs)))))
elif len(encoder_layers) == 7:
print("7 layers")
encoder = Model(inputs=inputs,
outputs=encoder_layers[6](encoder_layers[5](
encoder_layers[4](encoder_layers[3](
encoder_layers[2](encoder_layers[1](
encoder_layers[0](inputs))))))))
elif len(encoder_layers) == 6:
print("6 layers")
encoder = Model(inputs=inputs,
outputs=encoder_layers[5](encoder_layers[4](
encoder_layers[3](encoder_layers[2](
encoder_layers[1](
encoder_layers[0](inputs)))))))
elif len(encoder_layers) == 3:
print("3 layers")
encoder = Model(inputs=inputs,
outputs=encoder_layers[2](encoder_layers[1](
encoder_layers[0](inputs))))
elif len(encoder_layers) == 2:
print("2 layers")
encoder = Model(inputs=inputs,
outputs=encoder_layers[1](encoder_layers[0](inputs)))
encoder.summary()
encoded = encoder.predict(test)
print("dimensions of test set are")
print(test.shape)
embed_cols = ['embed_%i' % i for i in range(1, ncomps + 1, 1)]
print("dimension of embeddings from test set is")
print(encoded.shape)
embeddings_test = pd.DataFrame(data=encoded,
index=test.index,
columns=embed_cols)
print(embeddings_test.head)
print(embeddings_test.shape)
embeddings_test.to_csv(
"/projects/jgrealey/embedding/ten_k_samples/results/dae/" +
experiment_name + ".csv")
print(
"embeddings_saved at /projects/jgrealey/embedding/ten_k_samples/results/dae/{}.csv"
.format(experiment_name))
'''
