def objective(trial):
# Clear clutter from previous session graphs.
keras.backend.clear_session()
# The data is split between train and validation sets.
(x_train, y_train), (x_valid, y_valid) = mnist.load_data()
x_train = x_train.reshape(60000,
784)[:N_TRAIN_EXAMPLES].astype("float32") / 255
x_valid = x_valid.reshape(10000,
784)[:N_VALID_EXAMPLES].astype("float32") / 255
# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train[:N_TRAIN_EXAMPLES], CLASSES)
y_valid = keras.utils.to_categorical(y_valid[:N_VALID_EXAMPLES], CLASSES)
# Generate our trial model.
model = create_model(trial)
# Fit the model on the training data.
# The KerasPruningCallback checks for pruning condition every epoch.
model.fit(
x_train,
y_train,
batch_size=BATCHSIZE,
callbacks=[KerasPruningCallback(trial, "val_acc")],
epochs=EPOCHS,
validation_data=(x_valid, y_valid),
verbose=1,
)
# Evaluate the model accuracy on the validation set.
score = model.evaluate(x_valid, y_valid, verbose=0)
return score[1]
def objective(trial):
discard_state_train, discard_state_test, discard_ans_vector_train, discard_ans_vector_test = train_test_split(
discard_state_nml, discard_ans_vector_nml, test_size=0.25)
one_hot_discard_state_train = one_hot(discard_state_train, num)
one_hot_discard_state_test = one_hot(discard_state_test, num)
# Generate our trial model.
model = create_model(trial)
# Fit the model on the training data.
# The KerasPruningCallback checks for pruning condition every epoch.
model.fit(one_hot_discard_state_train,
discard_ans_vector_train,
epochs=EPOCHS,
callbacks=[KerasPruningCallback(trial, 'val_acc')],
validation_data=(one_hot_discard_state_test,
discard_ans_vector_test),
verbose=0)
# Evaluate the model accuracy on the test set.
score = model.evaluate(one_hot_discard_state_test,
discard_ans_vector_test,
verbose=0)
return score[1]
def objective(trial):
# Clear clutter form previous session graphs.
keras.backend.clear_session()
discard_state_train, discard_state_test, discard_ans_vector_train, discard_ans_vector_test = train_test_split(
discard_state_nml, discard_ans_vector_nml, test_size=0.25)
one_hot_discard_state_train = one_hot(
discard_state_train, num).reshape(len(discard_state_train),
input_shape[0], input_shape[1],
1)
one_hot_discard_state_test = one_hot(discard_state_test, num).reshape(
len(discard_state_test), input_shape[0], input_shape[1], 1)
# Generate our trial model.
model = create_model(trial)
# Fit the model on the training data.
# The KerasPruningCallback checks for pruning condition every epoch.
model.fit(one_hot_discard_state_train,
discard_ans_vector_train,
epochs=EPOCHS,
callbacks=[KerasPruningCallback(trial, 'val_acc')],
validation_data=(one_hot_discard_state_test,
discard_ans_vector_test),
verbose=0)
# Evaluate the model accuracy on the test set.
score = model.evaluate(one_hot_discard_state_test,
discard_ans_vector_test,
verbose=0)
return score[1]
def test_keras_pruning_callback_monitor_is_invalid() -> None:
study = optuna.create_study(pruner=DeterministicPruner(True))
trial = study.ask()
callback = KerasPruningCallback(trial, "InvalidMonitor")
with pytest.warns(UserWarning):
callback.on_epoch_end(0, {"loss": 1.0})
def test_keras_pruning_callback_observation_isnan() -> None:
study = optuna.create_study(pruner=DeterministicPruner(True))
trial = study.ask()
callback = KerasPruningCallback(trial, "loss")
with pytest.raises(optuna.TrialPruned):
callback.on_epoch_end(0, {"loss": 1.0})
with pytest.raises(optuna.TrialPruned):
callback.on_epoch_end(0, {"loss": float("nan")})
def test_keras_pruning_callback_observation_isnan():
# type: () -> None
study = optuna.create_study(pruner=DeterministicPruner(True))
trial = create_running_trial(study, 1.0)
callback = KerasPruningCallback(trial, "loss")
with pytest.raises(optuna.exceptions.TrialPruned):
callback.on_epoch_end(0, {"loss": 1.0})
with pytest.raises(optuna.exceptions.TrialPruned):
callback.on_epoch_end(0, {"loss": float("nan")})
def test_keras_pruning_callback_observation_isnan():
# type: () -> None
study = optuna.create_study(pruner=DeterministicPruner(True))
trial = study._run_trial(func=lambda _: 1.0, catch=(Exception, ))
callback = KerasPruningCallback(trial, 'loss')
with pytest.raises(optuna.structs.TrialPruned):
callback.on_epoch_end(0, {'loss': 1.0})
with pytest.raises(optuna.structs.TrialPruned):
callback.on_epoch_end(0, {'loss': float('nan')})
def objective(trial):
# type: (optuna.trial.Trial) -> float
model = Sequential()
model.add(Dense(1, activation='sigmoid', input_dim=20))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
model.fit(
np.zeros((16, 20), np.float32),
np.zeros((16, ), np.int32),
batch_size=1,
epochs=1,
callbacks=[KerasPruningCallback(trial, 'accuracy')],
verbose=0)
return 1.0
def objective(trial: optuna.trial.Trial) -> float:
model = Sequential()
model.add(Dense(1, activation="sigmoid", input_dim=20))
model.compile(optimizer="rmsprop", loss="binary_crossentropy", metrics=["accuracy"])
model.fit(
np.zeros((16, 20), np.float32),
np.zeros((16,), np.int32),
batch_size=1,
epochs=epochs,
callbacks=[KerasPruningCallback(trial, "accuracy", interval=interval)],
verbose=0,
)
return 1.0
def test_keras_pruning_callback_observation_isnan():
# type: () -> None
# TODO(higumachan): remove this "if" section after Tensorflow supports Python 3.7.
if not _available:
pytest.skip(
'This test requires keras '
'but this version can not install keras(tensorflow) with pip.')
study = optuna.create_study(pruner=DeterministicPruner(True))
trial = study._run_trial(func=lambda _: 1.0, catch=(Exception, ))
callback = KerasPruningCallback(trial, 'loss')
with pytest.raises(optuna.structs.TrialPruned):
callback.on_epoch_end(0, {'loss': 1.0})
with pytest.raises(optuna.structs.TrialPruned):
callback.on_epoch_end(0, {'loss': float('nan')})
def objective(trial):
# Open data file
f_in = h5py.File(DT_FL_IN, "r")
dt_in = f_in[DT_DST_IN]
f_out = h5py.File(DT_FL_OUT, "r")
dt_out = f_out[DT_DST_OUT]
WD = 2
# Dummy y_data
x_data, _ = format_data(dt_in, wd=WD, get_y=True)
_, y_data = format_data(dt_out, wd=WD, get_y=True)
x_data = np.squeeze(x_data)
# Split data and get slices
idxs = split(x_data.shape[0],
N_TRAIN,
N_VALID,
test_last=dt_in.attrs["idx"])
slc_trn, slc_vld, slc_tst = slicer(x_data.shape, idxs)
# Get data
x_train = x_data[slc_trn[0]]
y_train = y_data[slc_trn[0]]
x_val = x_data[slc_vld[0]]
y_val = y_data[slc_vld[0]]
conv_shape = y_train.shape[1:3]
# Strides cfg
strd = [2, 2, 5, 5]
# Limits and options
epochs = 60
# Filters
flt_lm = [[4, 128], [4, 128], [4, 128]]
d_lm = [1, 50]
# Kernel
k_lm = [3, 5]
# Regularizer
l2_lm = [1e-7, 1e-3]
# Activation functions
act_opts = ["relu", "elu", "tanh", "linear"]
# Latent space cfg
lt_sz = [5, 150]
lt_dv = [0.3, 0.7]
# Learning rate
lm_lr = [1e-5, 1e-1]
# Clear tensorflow session
tf.keras.backend.clear_session()
# Input
inputs = layers.Input(shape=x_train.shape[1:])
d = inputs
# Decoder
n_layers = trial.suggest_int("n_layers", 1, 3)
flt = trial.suggest_int("nl_flt", d_lm[0], d_lm[1])
# Reduction from output
red = np.prod(strd[:n_layers])
# Decoder first shape
lt_shp = (np.array(conv_shape) / red).astype(int)
# Decoder dense size
n_flat = np.prod(lt_shp) * flt
# Format stride list
strd = strd[::-1][-n_layers:]
# Latent -> Decoder layer
# Activation
act_lt = trial.suggest_categorical("lt_activation", act_opts)
# Regularization
l2_lt = int(trial.suggest_loguniform("lt_l2", l2_lm[0], l2_lm[1]))
l2_reg = regularizers.l2(l=l2_lt)
# Flat input to the decoder
d = layers.Dense(n_flat,
activation=act_lt,
kernel_regularizer=l2_reg,
name="l1_dense_decoder")(inputs)
# Reshape to the output of the encoder
d = layers.Reshape(list(lt_shp) + [flt])(d)
# Generate the convolutional layers
for i in range(n_layers):
# Get number of filters
flt = trial.suggest_int("n{}_flt".format(i), flt_lm[i][0],
flt_lm[i][1])
# Get the kernel size
k_sz = trial.suggest_categorical("d{}_kernel_size".format(i), k_lm)
# Get the activation function
act = trial.suggest_categorical("d{}_activation".format(i), act_opts)
# Regularization value
l2 = trial.suggest_loguniform("d{}_l2".format(i), l2_lm[0], l2_lm[1])
l2_reg = regularizers.l2(l=l2)
# Convolutional layer
d = layers.Conv2DTranspose(
flt,
(k_sz, k_sz),
strides=strd[i],
activation=act,
padding="same",
kernel_regularizer=l2_reg,
name="{}_decoder".format(i + 1),
)(d)
dp = 0
# Dropout layers
if dp > 0:
d = layers.Dropout(dp, name="{}_dropout_decoder".format(i + 1))(d)
decoded = layers.Conv2DTranspose(
y_train.shape[3],
(5, 5),
activation="linear",
padding="same",
name="output_decoder",
)(d)
ae = Model(inputs, decoded, name="Decoder_nxt")
# Earling stopping monitoring the loss of the validation dataset
monitor = "val_loss_norm_error"
patience = int(epochs * 0.3)
es = EarlyStopping(monitor=monitor,
mode="min",
patience=patience,
restore_best_weights=True)
opt = "adam"
if opt == "adam":
k_optf = optimizers.Adam
elif opt == "nadam":
k_optf = optimizers.Nadam
elif opt == "adamax":
k_optf = optimizers.Adamax
lr = trial.suggest_loguniform("lr", lm_lr[0], lm_lr[1])
if lr > 0:
k_opt = k_optf(learning_rate=lr)
else:
k_opt = k_optf()
ae.compile(optimizer=k_opt,
loss=loss_norm_error,
metrics=["mse", loss_norm_error])
batch_size = int(trial.suggest_uniform("batch_sz", 2, 32))
ae.summary()
hist = ae.fit(
x_train,
y_train,
epochs=epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(x_val, y_val),
callbacks=[KerasPruningCallback(trial, "val_loss_norm_error"), es],
verbose=1,
)
txt = PREFIX + SUFFIX
ae.save(txt.format(RUN_VERSION, trial.number))
return min(hist.history["val_loss_norm_error"])
def objective(trial):
# Open data file
f = h5py.File(DT_FL, "r")
dt = f[DT_DST]
# Split data and get slices
idxs = split(dt.shape[0], N_TRAIN, N_VALID)
slc_trn, slc_vld, slc_tst = slicer(dt.shape, idxs)
# Get data
x_train = dt[slc_trn]
x_val = dt[slc_vld]
# Limits and options
# Filters
# flt_lm = [4, 128]
flt_lm = [[4, 128], [4, 128], [4, 128]]
# Kernel
k_lm = [3, 5]
# Regularizer
l2_lm = [1e-7, 1e-3]
# Activation functions
act_opts = ["relu", "elu", "tanh", "linear"]
# Latent space cfg
lt_sz = [5, 150]
lt_dv = [0.3, 0.7]
# Learning rate
lm_lr = [1e-5, 1e-2]
# Clear tensorflow session
tf.keras.backend.clear_session()
# Input
inputs = layers.Input(shape=x_train.shape[1:])
e = inputs
# Encoder
flt, k_sz, act, l2 = [], [], [], []
strd = [2, 2, 5]
# n_layers = trial.suggest_int("n_layers", 2, 3)
n_layers = 3
for i in range(n_layers):
# Get values
flt += [
trial.suggest_int("n{}_flts".format(i), flt_lm[i][0], flt_lm[i][1])
]
k_sz += [trial.suggest_categorical("e{}_kernel_size".format(i), k_lm)]
act += [
trial.suggest_categorical("e{}_activation".format(i), act_opts)
]
l2 += [
trial.suggest_loguniform("e{}_l2".format(i), l2_lm[0], l2_lm[1])
]
l2_reg = regularizers.l2(l=l2[-1])
# l2_reg = regularizers.l2(l=0)
# Set layer
e = layers.Conv2D(
flt[-1],
(k_sz[-1], k_sz[-1]),
strides=strd[i],
activation=act[-1],
padding="same",
kernel_regularizer=l2_reg,
name="{}_encoder".format(i + 1),
)(e)
# Add layers
if i == 0:
ed = layers.Conv2D(
1,
(1, 1),
padding="same",
kernel_regularizer=l2_reg,
name="l2_input".format(i),
)(e)
# Dropout
dp = 0
if dp > 0:
e = layers.Dropout(dp, name="{}_dropout_encoder".format(i + 1))(e)
# Latent space
act_lt = trial.suggest_categorical("lt_activation", act_opts)
l2_lt = int(trial.suggest_loguniform("lt_l2", l2_lm[0], l2_lm[1]))
l2_reg = regularizers.l2(l=l2_lt)
sz_lt = trial.suggest_int("lt_sz", lt_sz[0], lt_sz[1])
dv_lt = trial.suggest_uniform("lt_div", lt_dv[0], lt_dv[1])
# Dense latent sizes
latent_1 = int(sz_lt * dv_lt)
latent_2 = sz_lt - latent_1
lt1 = layers.Flatten()(e)
lt1 = layers.Dense(latent_1,
activation=act_lt,
kernel_regularizer=l2_reg,
name="l1_latent")(lt1)
lt2 = layers.Flatten()(ed)
lt2 = layers.Dense(latent_2,
activation=act_lt,
kernel_regularizer=l2_reg,
name="l2_latent")(lt2)
# Dencoder
# Flat input to the decoder
n_flat = np.prod(backend.int_shape(e)[1:])
d = layers.Dense(n_flat,
activation=act_lt,
kernel_regularizer=l2_reg,
name="l1_dense_decoder")(lt1)
# Consider uses only one filter with convolution
# Reshape to the output of the encoder
d = layers.Reshape(backend.int_shape(e)[1:])(d)
# Generate the convolutional layers
for i in range(n_layers):
# Settings index
j = -i - 1
# Set the regularizer
l2_reg = regularizers.l2(l=l2[j])
# Add the latent space
if i == n_layers - 1:
d1 = layers.Dense(
5000,
activation="linear",
kernel_regularizer=l2_reg,
name="l2_dense_decoder",
)(lt2)
d1 = layers.Reshape(backend.int_shape(ed)[1:],
name="l2_reshape_decoder")(d1)
d1 = layers.Conv2D(
flt[j + 1],
(1, 1),
padding="same",
name="l2_compat_decoder",
kernel_regularizer=l2_reg,
)(d1)
d = layers.Add()([d1, d])
# Convolutional layer
d = layers.Conv2DTranspose(
flt[j],
(k_sz[j], k_sz[j]),
strides=strd[j],
activation=act[j],
padding="same",
kernel_regularizer=l2_reg,
name="{}_decoder".format(i + 1),
)(d)
# Dropout layers
if dp > 0:
d = layers.Dropout(dp, name="{}_dropout_decoder".format(i + 1))(d)
decoded = layers.Conv2DTranspose(
x_train.shape[-1],
(5, 5),
activation="linear",
padding="same",
kernel_regularizer=l2_reg,
name="output_decoder",
)(d)
ae = Model(inputs, decoded, name="auto_encoder_add")
opt = "adam"
if opt == "adam":
k_optf = optimizers.Adam
elif opt == "nadam":
k_optf = optimizers.Nadam
elif opt == "adamax":
k_optf = optimizers.Adamax
lr = trial.suggest_loguniform("lr", lm_lr[0], lm_lr[1])
if lr > 0:
k_opt = k_optf(learning_rate=lr)
else:
k_opt = k_optf()
ae.compile(optimizer=k_opt,
loss=loss_norm_error,
metrics=["mse", loss_norm_error])
batch_size = int(trial.suggest_uniform("batch_sz", 2, 32))
ae.summary()
hist = ae.fit(
x_train,
x_train,
epochs=30,
batch_size=batch_size,
shuffle=True,
validation_data=(x_val, x_val),
callbacks=[KerasPruningCallback(trial, "val_loss_norm_error")],
verbose=1,
)
txt = PREFIX + SUFFIX
ae.save(txt.format(RUN_VERSION, trial.number))
return hist.history["val_loss_norm_error"][-1]
def _do_set_callbacks_task(
self, base_model, train_set, validation_set, trial):
best_model_name = "best_model.h5"
model_save_file = os.path.join(self.config.model_path, best_model_name)
if self.config.multi_gpu:
checkpoint = ncc.callbacks.MultiGPUCheckpointCallback(
filepath=model_save_file,
base_model=base_model,
save_best_only=True,
)
else:
checkpoint = keras.callbacks.ModelCheckpoint(
filepath=model_save_file, save_best_only=True
)
if self.config.cosine_decay:
ncc_scheduler = ncc.schedulers.Scheduler(
self.config.cosine_lr_max,
self.config.cosine_lr_min,
self.config.epochs
)
scheduler = keras.callbacks.LearningRateScheduler(
ncc_scheduler.cosine_decay)
else:
scheduler = keras.callbacks.ReduceLROnPlateau(
factor=0.5, patience=10, verbose=1)
plot_history = ncc.callbacks.PlotHistory(
self.config.learning_path,
['loss', 'acc', 'iou_score', 'categorical_crossentropy']
)
callbacks = [checkpoint, scheduler, plot_history]
if self.config.task == ncc.tasks.Task.SEMANTIC_SEGMENTATION:
iou_history = ncc.callbacks.IouHistory(
save_dir=self.config.learning_path,
validation_files=validation_set,
class_names=self.config.class_names,
height=self.config.height,
width=self.config.width,
train_colors=self.config.train_colors
)
val_save_dir = os.path.join(self.config.image_path, "validation")
generate_sample_result = ncc.callbacks.GenerateSampleResult(
val_save_dir=val_save_dir,
validation_files=validation_set,
nb_classes=self.config.nb_classes,
height=self.config.height,
width=self.config.width,
train_colors=self.config.train_colors,
segmentation_val_step=self.config.segmentation_val_step
)
callbacks.extend([iou_history, generate_sample_result])
if self.config.optuna:
callbacks.append(KerasPruningCallback(trial, 'dice'))
elif self.config.task == ncc.tasks.Task.CLASSIFICATION:
if self.config.input_data_type == "video":
batch_checkpoint = ncc.callbacks.BatchCheckpoint(
self.config.learning_path,
f'{self.config.model_path}/batch_model.h5',
token=self.config.slack_token,
channel=self.config.slack_channel,
period=self.config.batch_period
)
callbacks.append(batch_checkpoint)
if self.config.optuna:
callbacks.append(KerasPruningCallback(trial, 'val_acc'))
if self.config.slack_channel and self.config.slack_token:
if self.config.task == ncc.tasks.Task.SEMANTIC_SEGMENTATION:
file_name = os.path.join(self.config.learning_path, "IoU.png")
else:
file_name = os.path.join(
self.config.learning_path, "acc.png"
)
slack_logging = ncc.callbacks.SlackLogger(
logger_file=file_name,
token=self.config.slack_token,
channel=self.config.slack_channel,
title=self.config.model_name,
)
callbacks.append(slack_logging)
return callbacks
def objective(trial):
# Open data file
f = h5py.File(DT_FL, "r")
dt = f[DT_DST]
# Format data for LSTM training
x_data, y_data = format_data(dt, wd=WD, get_y=True)
x_data = np.squeeze(x_data)
# Split data and get slices
idxs = split(x_data.shape[0], N_TRAIN, N_VALID)
slc_trn, slc_vld, slc_tst = slicer(x_data.shape, idxs)
# Get data
x_train = x_data[slc_trn[0]]
y_train = y_data[slc_trn[0]] - x_train
x_val = x_data[slc_vld[0]]
y_val = y_data[slc_vld[0]] - x_val
# Limits and options
# Filters
# n_lstm = [[4, 128], [4, 128], [4, 128]]
n_lstm = [[4, 196], [4, 196], [4, 196]]
# Regularizer
l2_lm = [1e-7, 1e-3]
# Activation functions
act_opts = ["relu", "elu", "tanh", "linear"]
# Latent space cfg
lt_sz = [5, 150]
lt_dv = [0.3, 0.7]
# Learning rate
lm_lr = [1e-5, 1]
# Clear tensorflow session
tf.keras.backend.clear_session()
# Input
inputs = layers.Input(shape=x_train.shape[1:])
p = inputs
# Dense layers
# n_lyr_dense = trial.suggest_int("n_lyr_dense", 0, 2)
n_lyr_dense = trial.suggest_int("n_lyr_dense", 1, 3)
for i in range(n_lyr_dense):
# For the current layer
# Get number of filters
l = trial.suggest_int("n{}_dense".format(i), n_lstm[i][0],
n_lstm[i][1])
# Get the activation function
act = trial.suggest_categorical("d{}_activation".format(i), act_opts)
# Regularization value
l2 = trial.suggest_loguniform("d{}_l2".format(i), l2_lm[0], l2_lm[1])
l2_reg = regularizers.l2(l=l2)
# Set layer
p = layers.Dense(
l,
activation=act,
# kernel_regularizer=l2_reg,
name="{}_dense".format(i + 1),
)(p)
# Dropout
dp = trial.suggest_uniform("d{}_dropout".format(i), 0, 1)
p = layers.Dropout(dp, name="{}_dropout_dense".format(i + 1))(p)
bn = trial.suggest_categorical("d{}_batchnorm".format(i), [0, 1])
if bn == 1:
p = layers.BatchNormalization(name="{}_bnorm_dense".format(i +
1))(p)
out = layers.Dense(y_data.shape[1], activation="linear")(p)
pred = Model(inputs, out, name="auto_encoder_add")
# opt_opts = ["adam", "nadam", "adamax", "RMSprop"]
# opt = trial.suggest_categorical("optimizer", opt_opts)
opt = "adam"
if opt == "adam":
k_optf = optimizers.Adam
elif opt == "nadam":
k_optf = optimizers.Nadam
elif opt == "adamax":
k_optf = optimizers.Adamax
elif opt == "RMSprop":
k_optf = optimizers.RMSprop
lr = trial.suggest_loguniform("lr", lm_lr[0], lm_lr[1])
if lr > 0:
k_opt = k_optf(learning_rate=lr)
else:
k_opt = k_optf()
pred.compile(optimizer=k_opt, loss="mse", metrics=["mse", loss_norm_error])
batch_size = int(trial.suggest_uniform("batch_sz", 2, 32))
pred.summary()
hist = pred.fit(
x_train,
y_train,
epochs=100,
batch_size=batch_size,
shuffle=True,
validation_data=(x_val, y_val),
callbacks=[KerasPruningCallback(trial, "val_mse")],
verbose=1,
)
txt = PREFIX + SUFFIX
pred.save(txt.format(RUN_VERSION, trial.number))
return hist.history["val_mse"][-1]
def objective(trial):
# Open data file
f = h5py.File(DT_FL, "r")
dt = f[DT_DST]
y_data = np.empty_like(dt)
for idx in dt.attrs['idx']:
y_data[idx[0]:idx[1]] = np.gradient(dt[idx[0]:idx[1]], 10, axis=0)
# Split data file
idxs = split(dt.shape[0], N_TRAIN, N_VALID, test_last=dt.attrs['idx'])
slc_trn, slc_vld, slc_tst = slicer(dt.shape, idxs)
# Slice data
x_train = dt[slc_trn]
y_train = y_data[slc_trn]
x_val = dt[slc_vld]
y_val = y_data[slc_vld]
# Limits and options
epochs = 500
# Filters
n_n = [[30, 150], [30, 150]]
# Regularizer
l2_lm = [1e-7, 1e-2]
# Activation functions
act_opts = ["relu", "elu", "tanh", "linear"]
# Learning rate
lm_lr = [1e-5, 1e-1]
# Clear tensorflow session
tf.keras.backend.clear_session()
# Input
inputs = layers.Input(shape=x_train.shape[1:])
d = inputs
# FCNN
n_layers = trial.suggest_int("n_layers", 1, 3)
for i in range(n_layers):
# For the current layer
# Get number of filters
n = trial.suggest_int("l{}_n_neurons".format(i), n_n[i][0], n_n[i][1])
# Get the activation function
act = trial.suggest_categorical("l{}_activation".format(i), act_opts)
# Regularization value
l2 = trial.suggest_loguniform("l{}_l2".format(i), l2_lm[0], l2_lm[1])
l2_reg = regularizers.l2(l=l2)
# Set layer
d = layers.Dense(
n,
activation=act,
kernel_regularizer=l2_reg,
name="l{}_fc".format(i),
)(d)
dd = layers.Dense(x_train.shape[1], activation='linear')(d)
fcnn = Model(inputs, dd, name="FCNN")
monitor = "val_loss_norm_error"
patience = int(epochs * 0.1)
es = EarlyStopping(monitor=monitor,
mode="min",
patience=patience,
restore_best_weights=True)
opt = "adam"
if opt == "adam":
k_optf = optimizers.Adam
elif opt == "nadam":
k_optf = optimizers.Nadam
elif opt == "adamax":
k_optf = optimizers.Adamax
lr = trial.suggest_loguniform("lr", lm_lr[0], lm_lr[1])
if lr > 0:
k_opt = k_optf(learning_rate=lr)
else:
k_opt = k_optf()
fcnn.compile(optimizer=k_opt,
loss=loss_norm_error,
metrics=["mse", loss_norm_error])
batch_size = int(trial.suggest_uniform("batch_sz", 2, 32))
fcnn.summary()
hist = fcnn.fit(
x_train,
y_train,
epochs=epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(x_val, y_val),
callbacks=[KerasPruningCallback(trial, "val_loss_norm_error"), es],
verbose=1,
)
txt = PREFIX + SUFFIX
fcnn.save(txt.format(RUN_VERSION, trial.number))
return hist.history["val_loss_norm_error"][-1]