def test_eval_hyperopt_space():
space = {
'filters_conv_A': hp.choice('filters_conv_A', [8, 16]),
'filters_conv_B': hp.choice('filters_conv_B', [16, 24]),
'rate': hp.uniform('rate', 0, 1),
'units': hp.choice('units', [96, 128, 192]),
'rate_1': hp.uniform('rate_1', 0, 1),
'lr': hp.uniform('lr', 1e-5, 1e-4),
'momentum': hp.choice('momentum', [0.5, 0.9, 0.999]),
'nesterov': hp.choice('nesterov', [True, False])
}
test_vals = {
'filters_conv_A': [0],
'filters_conv_B': [1],
'rate': [0.1553971698387464],
'units': [1],
'rate_1': [0.4114807190252343],
'lr': [2.0215692016654265e-05],
'momentum': [2],
'nesterov': [0]
}
test_vals_unpacked = {
'filters_conv_A': 0,
'filters_conv_B': 1,
'rate': 0.1553971698387464,
'units': 1,
'rate_1': 0.4114807190252343,
'lr': 2.0215692016654265e-05,
'momentum': 2,
'nesterov': 0
}
result = {
'filters_conv_A': 8,
'filters_conv_B': 24,
'rate': 0.1553971698387464,
'units': 128,
'rate_1': 0.4114807190252343,
'lr': 2.0215692016654265e-05,
'momentum': 0.999,
'nesterov': True
}
assert eval_hyperopt_space(space, test_vals) == result
assert eval_hyperopt_space(space, test_vals_unpacked) == result
def test_eval_hyperopt_space():
space = {
'filters_conv_A': hp.choice('filters_conv_A', [8, 16]),
'filters_conv_B': hp.choice('filters_conv_B', [16, 24]),
'rate': hp.uniform('rate', 0, 1),
'units': hp.choice('units', [96, 128, 192]),
'rate_1': hp.uniform('rate_1', 0, 1),
'lr': hp.uniform('lr', 1e-5, 1e-4),
'momentum': hp.choice('momentum', [0.5, 0.9, 0.999]),
'nesterov': hp.choice('nesterov', [True, False])
}
test_vals = {
'filters_conv_A': [0],
'filters_conv_B': [1],
'rate': [0.1553971698387464],
'units': [1],
'rate_1': [0.4114807190252343],
'lr': [2.0215692016654265e-05],
'momentum': [2],
'nesterov': [0]
}
test_vals_unpacked = {
'filters_conv_A': 0,
'filters_conv_B': 1,
'rate': 0.1553971698387464,
'units': 1,
'rate_1': 0.4114807190252343,
'lr': 2.0215692016654265e-05,
'momentum': 2,
'nesterov': 0
}
result = {
'filters_conv_A': 8,
'filters_conv_B': 24,
'rate': 0.1553971698387464,
'units': 128,
'rate_1': 0.4114807190252343,
'lr': 2.0215692016654265e-05,
'momentum': 0.999,
'nesterov': True
}
assert eval_hyperopt_space(space, test_vals) == result
assert eval_hyperopt_space(space, test_vals_unpacked) == result
### json_string = model.to_json()
### Return the optimizer and the architecture
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
if __name__ == '__main__':
trials = Trials()
max_evals = 4
best_run, best_model, space = optim.minimize(model=cnn_model,
data=data,
algo=tpe.suggest,
max_evals=max_evals,
trials=trials,
eval_space=True,
return_space=True)
### x_train, y_train, x_test, y_test, _, _ = data()
print("Best performing model chosen hyper-parameters:")
print(best_run)
print("\n")
for t, trial in enumerate(trials):
vals = trial.get('misc').get('vals')
### print("Trial %s vals: %s" % (t, vals))
print("Trial {}".format(t), eval_hyperopt_space(space, vals))
best_run, best_model, space = optim.minimize(model=create_model,
data=data,
rseed=random_seed,
algo=tpe.suggest,
max_evals=10,
trials=trials,
eval_space=True,
return_space=True)
print("------------------ Done ------------------")
# params for the different trials
best_trial = ("NONE", 0) # : (String, acc)
for t, trial in enumerate(trials):
vals = trial.get('misc').get('vals')
acc = trial.get('result').get('loss') * -1
values = eval_hyperopt_space(space, vals)
lr = LR_FUN(values['lr_exp'])
d = DECAY_FUN(values['decay_exp'])
batch = values['batch_size']
summary = "Trial {}:\nAcc={}, Learning rate: {:.2e}, decay: {:.2e}, batch size: {}".format(
t, acc, lr, d, batch)
print(summary)
with open(output_file, 'a+') as log:
log.write('{},{},{},{},{}\n'.format(t, acc, lr, d, batch))
if best_trial[1] < acc:
best_trial = (summary, acc)
# Print summary for best trial
with open(output_file, 'a+') as log:
log.write('Best run:\n'.format(NR_EPOCHS))
log.write(best_trial[0])
train_out_file='NeMO_train.txt',
valid_image_path='../Images/Valid_Patches/',
valid_label_path='../Images/ValidRef_Patches/',
valid_out_file='NeMO_valid.txt',
pixel_mean=[127.5, 127.5, 127.5],
pixel_std=[127.5, 127.5, 127.5],
num_classes=4,
model=FCN,
model_name="NeMO_FCN")
trials = Trials()
best_run, best_model, space = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=3,
trials=trials,
eval_space=True,
return_space=True)
train_generator, validation_generator = data()
print("validation_generator_size: ", validation_generator.batch_size)
print("Evalutation of best performing model:")
print("Parameters of best run", best_run)
print(best_model.evaluate(validation_generator))
json.dump(best_run,
open('./output/best_run' + optModel.model_name + '.txt', 'w'))
for t, trial in enumerate(trials):
vals = trial.get('misc').get('vals')
print("Trial %s vals: %s" % (t, vals))
print(eval_hyperopt_space(space, vals))
print('F1 score:', f1_sc)
print('Test score:', score)
print('accuracy:', acc)
return {'loss': -f1_sc, 'status': STATUS_OK, 'model': cnn}
if __name__ == '__main__':
trials=Trials()
best_run, best_model,space = optim.minimize(model=MCNN,
data=data,
functions=[f1_score],
algo=tpe.suggest,
max_evals=100,
trials=trials,
eval_space=True,
return_space=True)
trainX1,trainX2,trainY1,valX1,valX2,valY1,testX1,testX2,testY = data()
print("Evalutation of best performing model:")
print(best_model.evaluate([valX1, valX2], valY1, verbose=0))
print("Best performing model chosen hyper-parameters:")
print(best_run)
best_model.save('best_hyperas_model_secstr.h5')
hyperas_dict = dict()
for t, trial in enumerate(trials):
vals = trial.get('misc').get('vals')
hyperas_dict[t] = vals
print("Trial %s vals: %s" % (t, vals)) # <-- indices
print(eval_hyperopt_space(space, vals)) # <-- values
##from hyperas.utils import eval_hyperopt_space
##print(eval_hyperopt_space(space, vals))
pickle.dump(hyperas_dict, open( "./hyperas_dict_100trials", "wb" ))
dataloaders, dataset_sizes = DATASETTER[opt.data.dataset](
batch_size=opt.data.batch_size,
valid_size=opt.data.valid_size,
root=opt.data.root,
fixed_valid=opt.data.fixed_valid,
autoaugment=opt.data.autoaugment,
aug_policy=opt.data.aug_policy)
for t, trial in enumerate(trials):
vals = trial.get('misc').get('vals')
print("Trial %s vals: %s" % (t, vals))
tmp = {}
for k, v in list(vals.items()):
tmp[k] = v[0]
logger.info('Trial %d : %s' % (t, eval_hyperopt_space(space, tmp)))
criterion = CRITERION[opt.criterion.algo](
**opt.criterion.param) if opt.criterion.get('param') else CRITERION[
opt.criterion.algo]()
optimizer = OPTIMIZER[opt.optimizer.algo](
best_model.parameters(), **opt.optimizer.param) if opt.optimizer.get(
'param') else OPTIMIZER[opt.optimizer.algo](model.parameters())
# if not use scheduler, you can skip in config json file
if opt.scheduler.get('enabled', False):
scheduler_type = lr_scheduler.MultiStepLR if opt.scheduler.type == 'multistep' else lr_scheduler.CosineAnnealingLR if opt.scheduler.type == 'cosine' else lr_scheduler.StepLR
scheduler = scheduler_type(optimizer, **opt.scheduler.param)
else:
scheduler = None
trials = Trials()
best_run, best_model, space = optim.minimize(
model=create_model,
data=data,
algo=tpe.suggest,
max_evals=100,
trials=trials,
eval_space=True,
return_space=True,
#rseed=100
)
X_train1, X_test1, y_train, y_test = data()
# root
from hyperas.utils import eval_hyperopt_space
# H5
H5_path = "/ASTRAL186/astral_train1_hyperas_1.h5"
param_path = "/ASTRAL186/astral_train1_hyperas_param_1.csv"
print("Best performing model chosen hyper-parameters:")
print(best_run)
best_model.save(H5_path)
#print(space)
param = []
for t, trial in enumerate(trials):
vals = trial.get('misc').get('vals')
print("Trial %s vals: %s" % (t, vals))
print(eval_hyperopt_space(space, vals))
param_csv = eval_hyperopt_space(space, vals)
param.append(param_csv)
data_csv = pd.DataFrame(data=param)
data_csv.to_csv(param_path, index=0)