def evaluator(l2_reg):
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=800, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=800))
model.add(Softmax())
ns = 800
# Define callbacks
mt = MetricTracker() # Stores training evolution info
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-3,
lr_max=1e-1,
ns=ns) # Modifies lr while training
callbacks = [mt, lrs]
# Fit model
iterations = 4 * ns
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=100,
iterations=iterations,
l2_reg=l2_reg,
shuffle_minibatch=True,
callbacks=callbacks)
model.save("models/yes_dropout_test")
# Test model
val_acc = model.get_metric_loss(x_val, y_val)[0]
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "L2 param: " + str(l2_reg) + ", Test acc: " + str(test_acc)
mt.plot_training_progress(show=True,
save=True,
name="figures/l2reg_optimization/" + str(l2_reg),
subtitle=subtitle)
print("Val accuracy:", val_acc)
print("Test accuracy:", test_acc)
return val_acc
return grads_w, grads_b
if __name__ == "__main__":
x_train, y_train, x_val, y_val, x_test, y_test = read_cifar_10(n_train=3, n_val=5, n_test=2)
# x_train, y_train, x_val, y_val, x_test, y_test = read_mnist(n_train=2, n_val=5, n_test=2)
# x_train, y_train, x_val, y_val, x_test, y_test = read_names(n_train=500)
class_sum = np.sum(y_train, axis=1)*y_train.shape[0]
class_count = np.reciprocal(class_sum, where=abs(class_sum) > 0)
print(class_count)
print(type(x_train[0, 0, 0]))
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Conv2D(num_filters=2, kernel_shape=(4, 4), stride=3, dilation_rate=2, input_shape=x_train.shape[0:-1]))
model.add(Relu())
model.add(MaxPool2D((2, 2), stride=3))
model.add(Flatten())
model.add(Dense(nodes=y_train.shape[0]))
model.add(Relu())
model.add(Softmax())
print(np.min(np.abs(model.layers[0].filters)))
reg = 0.0
# Fit model
anal_time = time.time()
model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val,
a = np.divide(
np.abs(analytical_grad_weight-numerical_grad_w),
np.multiply(denom, (denom > _EPS)) + np.multiply(_EPS*np.ones(denom.shape), (denom <= _EPS)))
np.set_printoptions(suppress=True)
print(np.round(a*100,decimals=2))
av_error = np.average(a)
max_error = np.max(a)
print("Averaged Element-Wise Relative Error:", av_error*100, "%")
print("Max Element-Wise Relative Error:", max_error*100, "%")
# np.set_printoptions(suppress=False)
if __name__ == "__main__":
# Define model
v_rnn = VanillaRNN(state_size=state_size, input_size=K, output_size=K)
model = Sequential(loss=CrossEntropy(class_count=None), metric=Accuracy())
model.add(v_rnn)
model.layers[0].reset_state(copy.deepcopy(state))
# print(model.layers[0].c)
# Fit model
l2_reg = 0.0
model.fit(X=encoded_data, epochs=1, lr = 2e-2, momentum=0.95, l2_reg=l2_reg,
batcher=RnnBatcher(seq_length), callbacks=[])
print(model.layers[0].dl_dc)
anal = copy.deepcopy(model.layers[0].dl_dc)
model.layers[0].reset_state(copy.deepcopy(state))
learning_rate=3.0, batch_size=20,
epoch=50, loss=loss_SGD, activation=activation_SGD, l2=l2, size_of_valid=0.2)
model = Mlp()
model.add_module(InputLayer(784))
model.add_module(LinearActivation())
model.add_module(Dense(784, 30, initializer, initializer))
model.add_module(Sigmoid())
model.add_module(Dense(30, 10, initializer, initializer))
model.add_module(Sigmoid())
SGD.train(model, train_data.T)
predict_data = (model.forward(test_input.T)).T
accuracy = Accuracy()
cost = Cost(loss_SGD, l2, model)
precision = Precision()
recall = Recall()
f1 = F1()
tab_of_predict = TabOfPredict()
accuracy.evaluate(test_output, predict_data)
cost.evaluate(test_output, predict_data)
recall.evaluate(test_output, predict_data)
precision.evaluate(test_output, predict_data)
f1.evaluate(recall.history[0], precision.history[0])
tab_of_predict.evaluate(test_output, predict_data)
print("Accuracy: ", accuracy.history[0], "\n",
"Cost: ", cost.history[0], "\n",
"Recall: ", "\n", recall.history[0], "\n",
def evaluator(x_train, y_train, x_val, y_val, experiment_name="", **kwargs):
print(kwargs)
# Saving directories
figure_file = "figures/" + experiment_name + "/" + dict_to_string(kwargs)
model_file = "models/" + experiment_name + "/" + dict_to_string(kwargs)
mt = MetricTracker() # Stores training evolution info (losses and metrics)
# Define model
d = x_train.shape[0]
n1 = kwargs["n1"] # Filters of first Conv2D
k1 = kwargs["k1"] # First kernel y size
n2 = kwargs["n2"] # Filters of second Conv2D
k2 = kwargs["k2"] # Second kernel y size
batch_size = kwargs["batch_size"]
try:
# Define model
model = Sequential(loss=CrossEntropy(class_count=None),
metric=Accuracy())
model.add(
Conv2D(num_filters=n1,
kernel_shape=(d, k1),
input_shape=x_train.shape[:-1]))
model.add(Relu())
model.add(Conv2D(num_filters=n2, kernel_shape=(1, k2)))
model.add(Relu())
model.add(Flatten())
model.add(Dense(nodes=y_train.shape[0]))
model.add(Softmax())
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=batch_size,
epochs=1000,
lr=1e-2,
momentum=0.8,
l2_reg=0.001,
compensate=True,
callbacks=[mt])
except Exception as e:
print(e)
return -1 # If configuration impossible
model.save(model_file)
# Write results
n1 = str(n1)
n2 = str(n2)
k1 = str(k1)
k2 = str(k2)
batch_size = str(batch_size)
subtitle = "n1:" + n1 + ", n2:" + n2 + ", k1:" + k1 + ", k2:" + k1 +\
", batch_size:" + batch_size
mt.plot_training_progress(show=False,
save=True,
name=figure_file,
subtitle=subtitle)
# Maximizing value: validation accuracy
return model.val_metric
def evaluator(x_train,
y_train,
x_val,
y_val,
x_test,
y_test,
experiment_name="",
**kwargs):
# Saving directories
figure_file = "figures/" + experiment_name + "/" + dict_to_string(kwargs)
model_file = "models/" + experiment_name + "/" + dict_to_string(kwargs)
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=50, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=50))
model.add(Softmax())
# Pick metaparams
batch_size = 100
ns = 2 * np.floor(x_train.shape[1] / batch_size)
iterations = 4 * ns # 2 cycles
# Define callbacks
mt = MetricTracker() # Stores training evolution info
# bms = BestModelSaver(save_dir=None)
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-5,
lr_max=1e-1,
ns=ns)
# callbacks = [mt, bms, lrs]
callbacks = [mt, lrs]
# Adjust logarithmic
kwargs["l2_reg"] = 10**kwargs["l2_reg"]
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=batch_size,
epochs=None,
iterations=iterations,
**kwargs,
callbacks=callbacks)
# Write results
# best_model = bms.get_best_model()
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "l2_reg: " + str(
kwargs["l2_reg"]) + ", Test Acc: " + str(test_acc)
mt.plot_training_progress(show=False,
save=True,
name=figure_file,
subtitle=subtitle)
# Maximizing value: validation accuracy
# val_metric = bms.best_metric
val_metric = model.get_metric_loss(x_val, y_val)[0]
return val_metric
def train(self, model, training_data):
dataset = Dataset()
train_accuracy = Accuracy()
valid_accuracy = Accuracy()
train_cost = Cost(self.loss, self.l2, model)
valid_cost = Cost(self.loss, self.l2, model)
for j in range(self.epoch):
np.random.shuffle(training_data.T)
size = int(training_data.shape[1] * (1 - self.size_of_valid))
train_input = training_data[1:, :size]
train_output = training_data[0:1, :size]
train_input = dataset.normalization(train_input, 255)
train_output = dataset.hot_one(train_output, 10)
valid_input = training_data[1:, size:]
valid_output = training_data[0:1, size:]
valid_input = dataset.normalization(valid_input, 255)
valid_output = dataset.hot_one(valid_output, 10)
for k in range(0, len(train_output.T), self.batch_size):
mini_input = train_input[:,
self.batch_size:2 * self.batch_size]
mini_output = train_output[:,
self.batch_size:2 * self.batch_size]
self.update(model, mini_input, mini_output,
len(train_output.T))
train_model_predict = model.forward(train_input).T
valid_model_predict = model.forward(valid_input).T
train_accuracy.evaluate(train_output.T, train_model_predict)
valid_accuracy.evaluate(valid_output.T, valid_model_predict)
train_cost.evaluate(train_output.T, train_model_predict)
valid_cost.evaluate(train_output.T, valid_model_predict)
print("Epoch: ", j + 1)
print(
"Accuracy train: ",
train_accuracy.history[j],
"Accuracy valid: ",
valid_accuracy.history[j],
)
print("Cost train: ", train_cost.history[j], "Cost valid: ",
valid_cost.history[j])
epochs = range(1, self.epoch + 1)
plt.plot(epochs,
train_accuracy.history,
'g',
label='Training accuracy')
plt.plot(epochs,
valid_accuracy.history,
'b',
label='Validation accuracy')
plt.title('Training and Validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()
epochs = range(1, self.epoch + 1)
plt.plot(epochs, train_cost.history, 'g', label='Training loss')
plt.plot(epochs, valid_cost.history, 'b', label='Validation loss')
plt.title('Training and Validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()