def hyperparam_optimization( x, y ) :
nBits = x.shape[1]
frac_test = 0.3
x_train, x_test, y_train, y_test = train_test_split( x, y, test_size=frac_test )
TRAIN_EPOCHS = 20
MAX_TRIALS=20
EXECUTIONS_PER_TRIAL = 5
b_tuner = BayesianOptimization(
make_optimizer_model(nBits),
objective = 'val_mean_squared_error',
max_trials = MAX_TRIALS,
executions_per_trial=EXECUTIONS_PER_TRIAL,
directory='test_dir',
project_name='tune_optimizer',
seed=1
)
b_tuner.search_space_summary()
b_tuner.search( x=x_train, y=y_train, epochs=TRAIN_EPOCHS, validation_data=(x_test, y_test))
b_tuner.results_summary()
best_model = b_tuner.get_best_models()[0]
return best_model
def train(args):
print(args)
global_conf.config_tf2(args)
checkpoint_dir, log_dir, export_dir = create_env_directories(
args, get_experiment_name(args))
train_dataset = dataloader.get_dataset(
args['dataloader'],
transformation_list=args['dataloader']['train_list'],
num_classes=args["num_classes"],
split=args['dataloader']['train_split_id'])
val_dataset = dataloader.get_dataset(
args['dataloader'],
transformation_list=args['dataloader']['val_list'],
num_classes=args["num_classes"],
split=args['dataloader']['val_split_id'])
setup_mp(args)
build_model_fn = get_model(args)
callbacks = get_callbacks(args, log_dir)
# tuner = Hyperband(build_model_fn,
# objective='val_accuracy',
# max_epochs=args['num_epochs'],
# hyperband_iterations=10e100,
# directory=checkpoint_dir)
tuner = BayesianOptimization(build_model_fn,
objective='val_accuracy',
max_trials=100000,
num_initial_points=10,
directory=checkpoint_dir)
tuner.search_space_summary()
tuner.search(x=train_dataset,
validation_data=val_dataset,
callbacks=callbacks,
epochs=args['num_epochs'])
tuner.results_summary()
def BuildDNN(train_samples,
dev_samples,
test_samples,
norm_id,
model_path,
lags=None,
seed=None,
batch_size=512,
n_epochs=5,
max_trials=5,
executions_per_trial=3,
max_hidden_layers=3,
min_units=16,
max_units=64,
unit_step=16,
min_droprate=0.0,
max_droprate=0.5,
droprate_step=0.05,
min_learnrate=1e-4,
max_learnrate=1e-1,
n_tune_epochs=5,
cast_to_zero=True,
early_stop=True,
early_stop_patience=10,
retrain=False,
warm_up=False,
initial_epoch=None,
measurement_time='day',
measurement_unit='$m^3/s$'):
if not os.path.exists(model_path):
os.makedirs(model_path)
setting_info = {
"model_path": model_path,
"lags": lags,
"seed": seed,
"batch_size": batch_size,
"n_epoch": n_epochs,
"max_trials": max_trials,
"executions_per_trial": executions_per_trial,
"max_hidden_layers": max_hidden_layers,
"min_units": min_units,
"max_units": max_units,
"unit_step": unit_step,
"min_droprate": min_droprate,
"max_droprate": max_droprate,
"droprate_step": droprate_step,
"min_learnrate": min_learnrate,
"max_learnrate": max_learnrate,
"n_tune_epochs": n_tune_epochs,
"cast_to_zero": cast_to_zero,
"early_stop": early_stop,
"early_stop_patience": early_stop_patience,
"retrain": retrain,
}
with open(model_path + 'setting.json', 'w') as outfile:
json.dump(setting_info, outfile)
sMin = norm_id['series_min']
sMax = norm_id['series_max']
# sMin = train_samples.min(axis=0)
# sMax = train_samples.max(axis=0)
# train_samples = 2*(train_samples-sMin)/(sMax-sMin)-1
# dev_samples = 2*(dev_samples-sMin)/(sMax-sMin)-1
# test_samples = 2*(test_samples-sMin)/(sMax-sMin)-1
cal_samples = pd.concat([train_samples, dev_samples], axis=0)
cal_samples = cal_samples.sample(frac=1)
cal_samples = cal_samples.reset_index(drop=True)
train_samples = cal_samples.iloc[:train_samples.shape[0]]
dev_samples = cal_samples.iloc[train_samples.shape[0]:]
X = cal_samples
y = (cal_samples.pop('Y')).values
train_x = train_samples
train_y = train_samples.pop('Y')
train_y = train_y.values
dev_x = dev_samples
dev_y = dev_samples.pop('Y')
dev_y = dev_y.values
test_x = test_samples
test_y = test_samples.pop('Y')
test_y = test_y.values
# Config path to save optimal results
opt_path = model_path + '\\optimal\\'
cp_path = model_path + '\\optimal\\checkpoints\\'
if not os.path.exists(cp_path):
os.makedirs(cp_path)
# restore only the latest checkpoint after every update
checkpoint_path = cp_path + 'cp.h5'
checkpoint_dir = os.path.dirname(checkpoint_path)
# Define callbacks
cp_callback = keras.callbacks.ModelCheckpoint(checkpoint_path,
save_best_only=True,
mode='min',
save_weights_only=True,
verbose=1)
reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
min_lr=0.00001,
factor=0.2,
verbose=1,
patience=10,
mode='min')
early_stopping = keras.callbacks.EarlyStopping(
monitor='val_loss',
mode='min',
verbose=1,
patience=early_stop_patience,
restore_best_weights=True)
def build_model(hp):
input_shape = (train_x.shape[1], )
model = keras.Sequential()
num_layers = hp.Int('num_layers',
min_value=1,
max_value=max_hidden_layers,
step=1,
default=1)
for i in range(num_layers):
units = hp.Int('units_' + str(i),
min_value=min_units,
max_value=max_units,
step=unit_step)
dropout_rate = hp.Float('drop_rate_' + str(i),
min_value=min_droprate,
max_value=max_droprate,
step=droprate_step)
if i == 0:
model.add(
layers.Dense(units=units,
activation='relu',
input_shape=input_shape))
else:
model.add(layers.Dense(units=units, activation='relu'))
model.add(
layers.Dropout(rate=dropout_rate, noise_shape=None, seed=seed))
model.add(layers.Dense(1))
model.compile(optimizer=keras.optimizers.Adam(
hp.Float('learning_rate',
min_value=min_learnrate,
max_value=max_learnrate,
sampling='LOG',
default=1e-2)),
loss='mean_squared_error',
metrics=['mean_absolute_error', 'mean_squared_error'])
return model
tuner = BayesianOptimization(build_model,
objective='mean_squared_error',
max_trials=max_trials,
executions_per_trial=executions_per_trial,
directory=model_path,
project_name='BayesianOpt')
tuner.search_space_summary()
start = time.process_time()
tuner.search(x=train_x,
y=train_y,
epochs=n_tune_epochs,
validation_data=(dev_x, dev_y),
callbacks=[early_stopping])
end = time.process_time()
time_cost = end - start
tuner.results_summary()
best_hps = tuner.oracle.get_best_trials(num_trials=1)[0].hyperparameters
model = build_model(best_hps)
if retrain or not os.path.exists(checkpoint_path):
history = model.fit(X,
y,
epochs=n_epochs,
batch_size=batch_size,
validation_data=(X, y),
verbose=1,
callbacks=[
cp_callback,
early_stopping,
])
hist = pd.DataFrame(history.history)
hist.to_csv(opt_path + 'PARAMS-CAL-HISTORY.csv')
plot_history(history, opt_path + 'MAE-HISTORY.png',
opt_path + 'MSE-HISTORY.png')
else:
model.load_weights(checkpoint_path)
train_predictions = model.predict(train_x).flatten()
dev_predictions = model.predict(dev_x).flatten()
test_predictions = model.predict(test_x).flatten()
sMax = sMax[sMax.shape[0] - 1]
sMin = sMin[sMin.shape[0] - 1]
train_y = np.multiply(train_y + 1, sMax - sMin) / 2 + sMin
dev_y = np.multiply(dev_y + 1, sMax - sMin) / 2 + sMin
test_y = np.multiply(test_y + 1, sMax - sMin) / 2 + sMin
train_predictions = np.multiply(train_predictions + 1,
sMax - sMin) / 2 + sMin
dev_predictions = np.multiply(dev_predictions + 1, sMax - sMin) / 2 + sMin
test_predictions = np.multiply(test_predictions + 1,
sMax - sMin) / 2 + sMin
if cast_to_zero:
train_predictions[train_predictions < 0.0] = 0.0
dev_predictions[dev_predictions < 0.0] = 0.0
test_predictions[test_predictions < 0.0] = 0.0
dump_pred_results(
path=opt_path + '/opt_pred.csv',
train_y=train_y,
train_predictions=train_predictions,
dev_y=dev_y,
dev_predictions=dev_predictions,
test_y=test_y,
test_predictions=test_predictions,
time_cost=time_cost,
)
plot_rela_pred(train_y,
train_predictions,
measurement_time=measurement_time,
measurement_unit=measurement_unit,
fig_savepath=opt_path + 'TRAIN-PRED.png')
plot_rela_pred(dev_y,
dev_predictions,
measurement_time=measurement_time,
measurement_unit=measurement_unit,
fig_savepath=opt_path + "DEV-PRED.png")
plot_rela_pred(test_y,
test_predictions,
measurement_time=measurement_time,
measurement_unit=measurement_unit,
fig_savepath=opt_path + "TEST-PRED.png")
plot_error_distribution(test_predictions, test_y,
opt_path + 'TEST-ERROR-DSTRI.png')
plt.show()
input_img, training_class.network_obj(input_img, hp))
training_class.compile()
return training_class.local_model
# This is where we set the searching strategy
tuner = BayesianOptimization(build_model,
objective='val_mae',
seed=40,
max_trials=500,
executions_per_trial=1,
directory=jobdir)
tuner.search_space_summary()
training_class.cache_validation()
# replacement for model.fit
tuner.search(
training_class.local_generator,
validation_data=training_class.local_test_generator,
steps_per_epoch=training_class.steps_per_epoch,
epochs=training_class.epochs,
max_queue_size=4, # 32,
workers=training_class.workers,
shuffle=False,
use_multiprocessing=True,
callbacks=training_class.callbacks_list,
initial_epoch=0,
default=1e-3)),
loss='mse',
metrics=['mse'])
return model
hypermodel = RGModel(n_hidden=2)
HYPERBAND_MAX_EPOCHS = 40
MAX_TRIALS = 40
EXECUTION_PER_TRIAL = 4
tuner = BayesianOptimization(hypermodel,
objective='val_mean_squared_error',
seed=1,
max_trials=MAX_TRIALS,
executions_per_trial=EXECUTION_PER_TRIAL,
directory='random_search',
project_name='RGBFV8')
print(tuner.search_space_summary())
N_EPOCH_SEARCH = 10
# train_generator, steps_per_epoch=200, epochs=60, validation_data=validation_generator
tuner.search(train_gen_bf, epochs=N_EPOCH_SEARCH, validation_data=valid_gen_bf)
print(tuner.results_summary())
best_model = tuner.get_best_models(num_models=1)[0]
best_model.save('/DFS-L/DATA/pritchard/ankitesg/models/BFv12.h5')