def evaluator(l2_reg):
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=800, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=800))
model.add(Softmax())
ns = 800
# Define callbacks
mt = MetricTracker() # Stores training evolution info
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-3,
lr_max=1e-1,
ns=ns) # Modifies lr while training
callbacks = [mt, lrs]
# Fit model
iterations = 4 * ns
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=100,
iterations=iterations,
l2_reg=l2_reg,
shuffle_minibatch=True,
callbacks=callbacks)
model.save("models/yes_dropout_test")
# Test model
val_acc = model.get_metric_loss(x_val, y_val)[0]
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "L2 param: " + str(l2_reg) + ", Test acc: " + str(test_acc)
mt.plot_training_progress(show=True,
save=True,
name="figures/l2reg_optimization/" + str(l2_reg),
subtitle=subtitle)
print("Val accuracy:", val_acc)
print("Test accuracy:", test_acc)
return val_acc
def evaluator(x_train,
y_train,
x_val,
y_val,
x_test,
y_test,
experiment_name="",
**kwargs):
# Saving directories
figure_file = "figures/" + experiment_name + "/" + dict_to_string(kwargs)
model_file = "models/" + experiment_name + "/" + dict_to_string(kwargs)
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=50, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=50))
model.add(Softmax())
# Pick metaparams
batch_size = 100
ns = 2 * np.floor(x_train.shape[1] / batch_size)
iterations = 4 * ns # 2 cycles
# Define callbacks
mt = MetricTracker() # Stores training evolution info
# bms = BestModelSaver(save_dir=None)
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-5,
lr_max=1e-1,
ns=ns)
# callbacks = [mt, bms, lrs]
callbacks = [mt, lrs]
# Adjust logarithmic
kwargs["l2_reg"] = 10**kwargs["l2_reg"]
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=batch_size,
epochs=None,
iterations=iterations,
**kwargs,
callbacks=callbacks)
# Write results
# best_model = bms.get_best_model()
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "l2_reg: " + str(
kwargs["l2_reg"]) + ", Test Acc: " + str(test_acc)
mt.plot_training_progress(show=False,
save=True,
name=figure_file,
subtitle=subtitle)
# Maximizing value: validation accuracy
# val_metric = bms.best_metric
val_metric = model.get_metric_loss(x_val, y_val)[0]
return val_metric
ns=ns) # Modifies lr while training
callbacks = [mt, lrs]
# Fit model
iterations = 6 * ns
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=100,
iterations=iterations,
l2_reg=10**-1.85,
shuffle_minibatch=True,
callbacks=callbacks)
# model.save("models/yes_dropout_test")
# # Test model
val_acc = model.get_metric_loss(x_val, y_val)[0]
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "Test acc: " + str(test_acc)
mt.plot_training_progress(show=True,
save=True,
name="figures/lr_limits/final_train",
subtitle=subtitle)
# mt.save("limits_test")
# lrs = np.load("limits_test_lr.npy")
# plt.plot(lrs)
# plt.show()
# mt.plot_acc_vs_lr(show=True, save=True, name="figures/lr_limits/lr_test", subtitle="")
