def evaluator(x_train, y_train, x_val, y_val, x_test, y_test, experiment_name="", init="fixed", **kwargs):
# Define model
model = Sequential(loss=CrossEntropy())
model.add(Dense(nodes=10, input_dim=x_train.shape[0]))
model.add(Softmax())
# Fit model
model_save_path = "models/" + experiment_name + "/" + dict_to_string(kwargs) + "_" + init
best_model = model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val,
save_path=model_save_path, **kwargs)
# Plot results
test_acc = best_model.get_classification_metrics(x_test, y_test)[0]
subtitle = "l2_reg: " + str(kwargs["l2_reg"]) + ", lr: " + str(kwargs["lr"]) +\
", weight_init:" + init + ", Test Acc: " + str(test_acc)
best_model.plot_training_progress(show=False,
save=True,
name="figures/" + experiment_name + "/" + dict_to_string(kwargs) + "_" + init,
subtitle=subtitle)
montage(W=np.array(best_model.layers[0].weights[:, :-1]),
title=subtitle,
path="figures/" + experiment_name + "/weights/" + dict_to_string(kwargs) + "_" + init)
# Minimizing value: validation accuracy
val_acc = best_model.get_classification_metrics(x_val, y_val)[0] # Get accuracy
result = {"value": val_acc, "model": best_model} # Save score and model
return result
def base(train_dp, valid_dp, logger, learning_rate):
# learning_rate = 0.01
rng = numpy.random.RandomState([2016,02,26])
max_epochs = 1000
cost = CECost()
stats = list()
test_dp = deepcopy(valid_dp)
train_dp.reset()
valid_dp.reset()
test_dp.reset()
# NETWORK TOPOLOGY:
model = MLP(cost=cost)
model.add_layer(Relu(idim=125, odim=125, irange=1.6, rng=rng))
model.add_layer(Softmax(idim=125, odim=19, rng=rng))
# define the optimiser, here stochasitc gradient descent
# with fixed learning rate and max_epochs
lr_scheduler = LearningRateFixed(
learning_rate=learning_rate, max_epochs=max_epochs)
optimiser = SGDOptimiser(lr_scheduler=lr_scheduler)
logger.info('Training started...')
tr_stats_b, valid_stats_b = optimiser.train(model, train_dp, valid_dp)
logger.info('Testing the model on test set:')
tst_cost, tst_accuracy = optimiser.validate(model, test_dp)
logger.info('ACL test set accuracy is %.2f %%, cost (%s) is %.3f' %
(tst_accuracy*100., cost.get_name(), tst_cost))
def get_cur_discr_model(model, cur_layer_id):
cur_layer = model.layers[cur_layer_id]
assert isinstance(cur_layer, Linear), (
"Expected current layer to be Linear or its subclass")
get_targets = lambda inputs, t, pure: t
if cur_layer_id == 0:
get_inputs = lambda x: (x, None)
else:
prev_layers = model.layers[:cur_layer_id]
prev_mds = MLP_fast(MSECost())
prev_mds.set_layers(prev_layers)
get_inputs = lambda x: (prev_mds.fprop(x), None)
last_layer = cur_layer_id == len(model.layers) - 1
cur_model = MLP_fast(CECost())
cur_model.add_layer(cur_layer)
if last_layer:
assert isinstance(cur_layer, Softmax), (
"final layer must be softmax for MNIST digits classification"
)
#here it automatically matches output of previous layer
else:
#get final layer for the MNIST dataset
cur_model.add_layer(Softmax(cur_layer.odim, 10))
return cur_model, get_inputs, get_targets
def evaluator(l2_reg):
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=800, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=800))
model.add(Softmax())
ns = 800
# Define callbacks
mt = MetricTracker() # Stores training evolution info
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-3,
lr_max=1e-1,
ns=ns) # Modifies lr while training
callbacks = [mt, lrs]
# Fit model
iterations = 4 * ns
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=100,
iterations=iterations,
l2_reg=l2_reg,
shuffle_minibatch=True,
callbacks=callbacks)
model.save("models/yes_dropout_test")
# Test model
val_acc = model.get_metric_loss(x_val, y_val)[0]
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "L2 param: " + str(l2_reg) + ", Test acc: " + str(test_acc)
mt.plot_training_progress(show=True,
save=True,
name="figures/l2reg_optimization/" + str(l2_reg),
subtitle=subtitle)
print("Val accuracy:", val_acc)
print("Test accuracy:", test_acc)
return val_acc
class_sum = np.sum(y_train, axis=1)*y_train.shape[0]
class_count = np.reciprocal(class_sum, where=abs(class_sum) > 0)
print(class_count)
print(type(x_train[0, 0, 0]))
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Conv2D(num_filters=2, kernel_shape=(4, 4), stride=3, dilation_rate=2, input_shape=x_train.shape[0:-1]))
model.add(Relu())
model.add(MaxPool2D((2, 2), stride=3))
model.add(Flatten())
model.add(Dense(nodes=y_train.shape[0]))
model.add(Relu())
model.add(Softmax())
print(np.min(np.abs(model.layers[0].filters)))
reg = 0.0
# Fit model
anal_time = time.time()
model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val,
batch_size=200, epochs=1, lr=0, momentum=0, l2_reg=reg)
analytical_grad_weight = model.layers[0].filter_gradients
analytical_grad_bias = model.layers[0].bias_gradients
# print(analytical_grad_weight)
print(analytical_grad_bias)
anal_time = time.time() - anal_time
def evaluator(x_train, y_train, x_val, y_val, experiment_name="", **kwargs):
print(kwargs)
# Saving directories
figure_file = "figures/" + experiment_name + "/" + dict_to_string(kwargs)
model_file = "models/" + experiment_name + "/" + dict_to_string(kwargs)
mt = MetricTracker() # Stores training evolution info (losses and metrics)
# Define model
d = x_train.shape[0]
n1 = kwargs["n1"] # Filters of first Conv2D
k1 = kwargs["k1"] # First kernel y size
n2 = kwargs["n2"] # Filters of second Conv2D
k2 = kwargs["k2"] # Second kernel y size
batch_size = kwargs["batch_size"]
try:
# Define model
model = Sequential(loss=CrossEntropy(class_count=None),
metric=Accuracy())
model.add(
Conv2D(num_filters=n1,
kernel_shape=(d, k1),
input_shape=x_train.shape[:-1]))
model.add(Relu())
model.add(Conv2D(num_filters=n2, kernel_shape=(1, k2)))
model.add(Relu())
model.add(Flatten())
model.add(Dense(nodes=y_train.shape[0]))
model.add(Softmax())
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=batch_size,
epochs=1000,
lr=1e-2,
momentum=0.8,
l2_reg=0.001,
compensate=True,
callbacks=[mt])
except Exception as e:
print(e)
return -1 # If configuration impossible
model.save(model_file)
# Write results
n1 = str(n1)
n2 = str(n2)
k1 = str(k1)
k2 = str(k2)
batch_size = str(batch_size)
subtitle = "n1:" + n1 + ", n2:" + n2 + ", k1:" + k1 + ", k2:" + k1 +\
", batch_size:" + batch_size
mt.plot_training_progress(show=False,
save=True,
name=figure_file,
subtitle=subtitle)
# Maximizing value: validation accuracy
return model.val_metric
def evaluator(x_train,
y_train,
x_val,
y_val,
x_test,
y_test,
experiment_name="",
**kwargs):
# Saving directories
figure_file = "figures/" + experiment_name + "/" + dict_to_string(kwargs)
model_file = "models/" + experiment_name + "/" + dict_to_string(kwargs)
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=50, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=50))
model.add(Softmax())
# Pick metaparams
batch_size = 100
ns = 2 * np.floor(x_train.shape[1] / batch_size)
iterations = 4 * ns # 2 cycles
# Define callbacks
mt = MetricTracker() # Stores training evolution info
# bms = BestModelSaver(save_dir=None)
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-5,
lr_max=1e-1,
ns=ns)
# callbacks = [mt, bms, lrs]
callbacks = [mt, lrs]
# Adjust logarithmic
kwargs["l2_reg"] = 10**kwargs["l2_reg"]
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=batch_size,
epochs=None,
iterations=iterations,
**kwargs,
callbacks=callbacks)
# Write results
# best_model = bms.get_best_model()
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "l2_reg: " + str(
kwargs["l2_reg"]) + ", Test Acc: " + str(test_acc)
mt.plot_training_progress(show=False,
save=True,
name=figure_file,
subtitle=subtitle)
# Maximizing value: validation accuracy
# val_metric = bms.best_metric
val_metric = model.get_metric_loss(x_val, y_val)[0]
return val_metric
tsk8_2_model.add_layer(
ConvRelu(num_inp_feat_maps=1,
num_out_feat_maps=5,
image_shape=(28, 28),
kernel_shape=(5, 5),
stride=(1, 1),
rng=rng))
tsk8_2_model.add_layer(
ConvMaxPool2D(num_feat_maps=5,
conv_shape=(24, 24),
pool_shape=(2, 2),
pool_stride=(2, 2)))
#idim, odim,
tsk8_2_model.add_layer(Relu(idim=5 * 12 * 12, odim=80, rng=rng))
tsk8_2_model.add_layer(Softmax(idim=80, odim=10, rng=rng))
#one can stack more layers here
# define the optimiser, here stochasitc gradient descent
# with fixed learning rate and max_epochs as stopping criterion
lr_scheduler = LearningRateFixed(learning_rate=0.01, max_epochs=30)
optimiser = SGDOptimiser(lr_scheduler=lr_scheduler)
logger.info('Initialising data providers...')
train_dp = MNISTDataProvider(dset='train',
batch_size=250,
max_num_batches=-10,
randomize=True,
conv_reshape=True)
valid_dp = MNISTDataProvider(dset='valid',
batch_size=250,
max_epochs = 30
cost = CECost()
stats = list()
test_dp = deepcopy(valid_dp)
train_dp.reset()
valid_dp.reset()
test_dp.reset()
#define the model
model = MLP(cost=cost)
#model.add_layer(ComplexLinear(idim=125, odim=125, irange=1.6, rng=rng))
#model.add_layer(Sigmoid(idim=2*125, odim=125, irange=1.6, rng=rng))
model.add_layer(Sigmoid(idim=125, odim=125, irange=1.6, rng=rng))
model.add_layer(Softmax(idim=125, odim=19, rng=rng))
# define the optimiser, here stochasitc gradient descent
# with fixed learning rate and max_epochs
lr_scheduler = LearningRateFixed(learning_rate=learning_rate,
max_epochs=max_epochs)
optimiser = SGDOptimiser(lr_scheduler=lr_scheduler)
logger.info('Training started...')
tr_stats, valid_stats = optimiser.train(model, train_dp, valid_dp)
logger.info('Testing the model on test set:')
tst_cost, tst_accuracy = optimiser.validate(model, test_dp)
logger.info('MNIST test set accuracy is %.2f %%, cost (%s) is %.3f' %
(tst_accuracy * 100., cost.get_name(), tst_cost))
irange=0.2,
rng=None,
conv_fwd=my_conv_fwd,
conv_bck=my_conv_bck,
conv_grad=my_conv_grad)
"""
tsk8_1_1_model.add_layer(ConvSigmoid(num_inp_feat_maps=1,
num_out_feat_maps=1,
image_shape=(28,28),
kernel_shape=(10, 10),
stride=(1,1),
rng=rng))
#idim, odim,
tsk8_1_1_model.add_layer(Relu(idim=1*19*19, odim=100, rng=rng))
tsk8_1_1_model.add_layer(Softmax(idim=100, odim=10, rng=rng))
#one can stack more layers here
# define the optimiser, here stochasitc gradient descent
# with fixed learning rate and max_epochs as stopping criterion
lr_scheduler = LearningRateFixed(learning_rate=0.1, max_epochs=30)
optimiser = SGDOptimiser(lr_scheduler=lr_scheduler)
logger.info('Initialising data providers...')
train_dp = MNISTDataProvider(dset='train', batch_size=100, max_num_batches=-10, randomize=True, conv_reshape=True)
valid_dp = MNISTDataProvider(dset='valid', batch_size=100, max_num_batches=-10, randomize=False, conv_reshape=True)
logger.info('Training started...')
tsk8_1_1_tr_stats, tsk8_1_1_valid_stats = optimiser.train(tsk8_1_1_model, train_dp, valid_dp)
logger.info('Testing the model on test set:')