def evaluator(l2_reg): # Define model model = Sequential(loss=CrossEntropy(), metric=Accuracy()) model.add(Dense(nodes=800, input_dim=x_train.shape[0])) model.add(Relu()) model.add(Dense(nodes=10, input_dim=800)) model.add(Softmax()) ns = 800 # Define callbacks mt = MetricTracker() # Stores training evolution info lrs = LearningRateScheduler(evolution="cyclic", lr_min=1e-3, lr_max=1e-1, ns=ns) # Modifies lr while training callbacks = [mt, lrs] # Fit model iterations = 4 * ns model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val, batch_size=100, iterations=iterations, l2_reg=l2_reg, shuffle_minibatch=True, callbacks=callbacks) model.save("models/yes_dropout_test") # Test model val_acc = model.get_metric_loss(x_val, y_val)[0] test_acc = model.get_metric_loss(x_test, y_test)[0] subtitle = "L2 param: " + str(l2_reg) + ", Test acc: " + str(test_acc) mt.plot_training_progress(show=True, save=True, name="figures/l2reg_optimization/" + str(l2_reg), subtitle=subtitle) print("Val accuracy:", val_acc) print("Test accuracy:", test_acc) return val_acc
def evaluator(x_train, y_train, x_val, y_val, x_test, y_test, experiment_name="", **kwargs): # Saving directories figure_file = "figures/" + experiment_name + "/" + dict_to_string(kwargs) model_file = "models/" + experiment_name + "/" + dict_to_string(kwargs) # Define model model = Sequential(loss=CrossEntropy(), metric=Accuracy()) model.add(Dense(nodes=50, input_dim=x_train.shape[0])) model.add(Relu()) model.add(Dense(nodes=10, input_dim=50)) model.add(Softmax()) # Pick metaparams batch_size = 100 ns = 2 * np.floor(x_train.shape[1] / batch_size) iterations = 4 * ns # 2 cycles # Define callbacks mt = MetricTracker() # Stores training evolution info # bms = BestModelSaver(save_dir=None) lrs = LearningRateScheduler(evolution="cyclic", lr_min=1e-5, lr_max=1e-1, ns=ns) # callbacks = [mt, bms, lrs] callbacks = [mt, lrs] # Adjust logarithmic kwargs["l2_reg"] = 10**kwargs["l2_reg"] # Fit model model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val, batch_size=batch_size, epochs=None, iterations=iterations, **kwargs, callbacks=callbacks) # Write results # best_model = bms.get_best_model() test_acc = model.get_metric_loss(x_test, y_test)[0] subtitle = "l2_reg: " + str( kwargs["l2_reg"]) + ", Test Acc: " + str(test_acc) mt.plot_training_progress(show=False, save=True, name=figure_file, subtitle=subtitle) # Maximizing value: validation accuracy # val_metric = bms.best_metric val_metric = model.get_metric_loss(x_val, y_val)[0] return val_metric
ns=ns) # Modifies lr while training callbacks = [mt, lrs] # Fit model iterations = 6 * ns model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val, batch_size=100, iterations=iterations, l2_reg=10**-1.85, shuffle_minibatch=True, callbacks=callbacks) # model.save("models/yes_dropout_test") # # Test model val_acc = model.get_metric_loss(x_val, y_val)[0] test_acc = model.get_metric_loss(x_test, y_test)[0] subtitle = "Test acc: " + str(test_acc) mt.plot_training_progress(show=True, save=True, name="figures/lr_limits/final_train", subtitle=subtitle) # mt.save("limits_test") # lrs = np.load("limits_test_lr.npy") # plt.plot(lrs) # plt.show() # mt.plot_acc_vs_lr(show=True, save=True, name="figures/lr_limits/lr_test", subtitle="")