def evaluator(x_train, y_train, x_val, y_val, x_test, y_test, experiment_name="", init="fixed", **kwargs):
# Define model
model = Sequential(loss=CrossEntropy())
model.add(Dense(nodes=10, input_dim=x_train.shape[0]))
model.add(Softmax())
# Fit model
model_save_path = "models/" + experiment_name + "/" + dict_to_string(kwargs) + "_" + init
best_model = model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val,
save_path=model_save_path, **kwargs)
# Plot results
test_acc = best_model.get_classification_metrics(x_test, y_test)[0]
subtitle = "l2_reg: " + str(kwargs["l2_reg"]) + ", lr: " + str(kwargs["lr"]) +\
", weight_init:" + init + ", Test Acc: " + str(test_acc)
best_model.plot_training_progress(show=False,
save=True,
name="figures/" + experiment_name + "/" + dict_to_string(kwargs) + "_" + init,
subtitle=subtitle)
montage(W=np.array(best_model.layers[0].weights[:, :-1]),
title=subtitle,
path="figures/" + experiment_name + "/weights/" + dict_to_string(kwargs) + "_" + init)
# Minimizing value: validation accuracy
val_acc = best_model.get_classification_metrics(x_val, y_val)[0] # Get accuracy
result = {"value": val_acc, "model": best_model} # Save score and model
return result
from mlp.losses import CrossEntropy
from mlp.layers import Conv2D, Dense, Softmax, Relu, Flatten, Dropout, MaxPool2D
from mlp.callbacks import MetricTracker, BestModelSaver, LearningRateScheduler
from mlp.utils import plot_confusion_matrix
np.random.seed(1)
if __name__ == "__main__":
# Load data
x_test, y_test, names = read_names_test()
classes = read_names_countries()
print(x_test.shape)
# Load model model
model = Sequential(loss=CrossEntropy())
model.load("models/names_test")
# model.load("models/names_no_compensation")
y_pred_prob_test = model.predict(x_test)
y_pred_test = model.predict_classes(x_test)
print(y_pred_prob_test)
print(y_test)
plot_confusion_matrix(y_pred_test, y_test, classes, "figures/conf_test")
import matplotlib.pyplot as plt
plt.title("Prediction Vectors")
pos = plt.imshow(y_pred_prob_test.T)
plt.xticks(range(len(classes)), classes, rotation=45, ha='right')
plt.yticks(range(len(names)), names)
# Define callbacks
mt = MetricTracker() # Stores training evolution info (losses and metrics)
# lrs = LearningRateScheduler(evolution="linear", lr_min=1e-3, lr_max=9e-1)
# lrs = LearningRateScheduler(evolution="constant", lr_min=1e-3, lr_max=9e-1)
# callbacks = [mt, lrs]
callbacks = [mt]
# Define hyperparams
d = x_train.shape[0]
n1 = 40 # Filters of first Conv2D
k1 = 6 # First kernel y size
n2 = 20 # Filters of second Conv2D
k2 = 4 # Second kernel y size
# Define model
model = Sequential(loss=CrossEntropy(class_count=None), metric=Accuracy())
model.add(
Conv2D(num_filters=n1,
kernel_shape=(d, k1),
input_shape=x_train.shape[:-1]))
model.add(Relu())
model.add(Conv2D(num_filters=n2, kernel_shape=(1, k2)))
model.add(Relu())
model.add(Flatten())
model.add(Dense(nodes=y_train.shape[0]))
model.add(Softmax())
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
def evaluator(x_train, y_train, x_val, y_val, experiment_name="", **kwargs):
print(kwargs)
# Saving directories
figure_file = "figures/" + experiment_name + "/" + dict_to_string(kwargs)
model_file = "models/" + experiment_name + "/" + dict_to_string(kwargs)
mt = MetricTracker() # Stores training evolution info (losses and metrics)
# Define model
d = x_train.shape[0]
n1 = kwargs["n1"] # Filters of first Conv2D
k1 = kwargs["k1"] # First kernel y size
n2 = kwargs["n2"] # Filters of second Conv2D
k2 = kwargs["k2"] # Second kernel y size
batch_size = kwargs["batch_size"]
try:
# Define model
model = Sequential(loss=CrossEntropy(class_count=None),
metric=Accuracy())
model.add(
Conv2D(num_filters=n1,
kernel_shape=(d, k1),
input_shape=x_train.shape[:-1]))
model.add(Relu())
model.add(Conv2D(num_filters=n2, kernel_shape=(1, k2)))
model.add(Relu())
model.add(Flatten())
model.add(Dense(nodes=y_train.shape[0]))
model.add(Softmax())
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=batch_size,
epochs=1000,
lr=1e-2,
momentum=0.8,
l2_reg=0.001,
compensate=True,
callbacks=[mt])
except Exception as e:
print(e)
return -1 # If configuration impossible
model.save(model_file)
# Write results
n1 = str(n1)
n2 = str(n2)
k1 = str(k1)
k2 = str(k2)
batch_size = str(batch_size)
subtitle = "n1:" + n1 + ", n2:" + n2 + ", k1:" + k1 + ", k2:" + k1 +\
", batch_size:" + batch_size
mt.plot_training_progress(show=False,
save=True,
name=figure_file,
subtitle=subtitle)
# Maximizing value: validation accuracy
return model.val_metric
if __name__ == "__main__":
# Load data
x_train, y_train = LoadXY("data_batch_1")
x_val, y_val = LoadXY("data_batch_2")
x_test, y_test = LoadXY("test_batch")
# Preprocessing
mean_x = np.mean(x_train)
std_x = np.std(x_train)
x_train = (x_train - mean_x) / std_x
x_val = (x_val - mean_x) / std_x
x_test = (x_test - mean_x) / std_x
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=800, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dropout(ones_ratio=0.50))
model.add(Dense(nodes=10, input_dim=800))
model.add(Softmax())
ns = 500
# Define callbacks
mt = MetricTracker() # Stores training evolution info
# bms = BestModelSaver(save_dir=None) # Saves model with highest val_metric
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-3,
lr_max=1e-1,
ns=ns) # Modifies lr while training
def evaluator(x_train,
y_train,
x_val,
y_val,
x_test,
y_test,
experiment_name="",
**kwargs):
# Saving directories
figure_file = "figures/" + experiment_name + "/" + dict_to_string(kwargs)
model_file = "models/" + experiment_name + "/" + dict_to_string(kwargs)
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=50, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=50))
model.add(Softmax())
# Pick metaparams
batch_size = 100
ns = 2 * np.floor(x_train.shape[1] / batch_size)
iterations = 4 * ns # 2 cycles
# Define callbacks
mt = MetricTracker() # Stores training evolution info
# bms = BestModelSaver(save_dir=None)
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-5,
lr_max=1e-1,
ns=ns)
# callbacks = [mt, bms, lrs]
callbacks = [mt, lrs]
# Adjust logarithmic
kwargs["l2_reg"] = 10**kwargs["l2_reg"]
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=batch_size,
epochs=None,
iterations=iterations,
**kwargs,
callbacks=callbacks)
# Write results
# best_model = bms.get_best_model()
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "l2_reg: " + str(
kwargs["l2_reg"]) + ", Test Acc: " + str(test_acc)
mt.plot_training_progress(show=False,
save=True,
name=figure_file,
subtitle=subtitle)
# Maximizing value: validation accuracy
# val_metric = bms.best_metric
val_metric = model.get_metric_loss(x_val, y_val)[0]
return val_metric
if __name__ == "__main__":
# Load data
x_train, y_train, x_val, y_val, x_test, y_test = read_mnist(n_train=200,
n_val=200,
n_test=2)
# Define callbacks
mt = MetricTracker() # Stores training evolution info (losses and metrics)
# lrs = LearningRateScheduler(evolution="linear", lr_min=1e-3, lr_max=9e-1)
# lrs = LearningRateScheduler(evolution="constant", lr_min=1e-3, lr_max=9e-1)
# callbacks = [mt, lrs]
callbacks = [mt]
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(
Conv2D(num_filters=64, kernel_shape=(4, 4), input_shape=(28, 28, 1)))
model.add(Relu())
model.add(MaxPool2D(kernel_shape=(2, 2)))
# model.add(Conv2D(num_filters=32, kernel_shape=(3, 3)))
# model.add(Relu())
model.add(Flatten())
# model.add(Flatten(input_shape=(28, 28, 1)))
model.add(Dense(nodes=400))
model.add(Relu())
model.add(Dense(nodes=10))
model.add(Softmax())
# for filt in model.layers[0].filters:
# print(filt)
a = np.divide(
np.abs(analytical_grad_weight-numerical_grad_w),
np.multiply(denom, (denom > _EPS)) + np.multiply(_EPS*np.ones(denom.shape), (denom <= _EPS)))
np.set_printoptions(suppress=True)
print(np.round(a*100,decimals=2))
av_error = np.average(a)
max_error = np.max(a)
print("Averaged Element-Wise Relative Error:", av_error*100, "%")
print("Max Element-Wise Relative Error:", max_error*100, "%")
# np.set_printoptions(suppress=False)
if __name__ == "__main__":
# Define model
v_rnn = VanillaRNN(state_size=state_size, input_size=K, output_size=K)
model = Sequential(loss=CrossEntropy(class_count=None), metric=Accuracy())
model.add(v_rnn)
model.layers[0].reset_state(copy.deepcopy(state))
# print(model.layers[0].c)
# Fit model
l2_reg = 0.0
model.fit(X=encoded_data, epochs=1, lr = 2e-2, momentum=0.95, l2_reg=l2_reg,
batcher=RnnBatcher(seq_length), callbacks=[])
print(model.layers[0].dl_dc)
anal = copy.deepcopy(model.layers[0].dl_dc)
model.layers[0].reset_state(copy.deepcopy(state))
def evaluator(l2_reg):
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=800, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=800))
model.add(Softmax())
ns = 800
# Define callbacks
mt = MetricTracker() # Stores training evolution info
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-3,
lr_max=1e-1,
ns=ns) # Modifies lr while training
callbacks = [mt, lrs]
# Fit model
iterations = 4 * ns
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=100,
iterations=iterations,
l2_reg=l2_reg,
shuffle_minibatch=True,
callbacks=callbacks)
model.save("models/yes_dropout_test")
# Test model
val_acc = model.get_metric_loss(x_val, y_val)[0]
test_acc = model.get_metric_loss(x_test, y_test)[0]
subtitle = "L2 param: " + str(l2_reg) + ", Test acc: " + str(test_acc)
mt.plot_training_progress(show=True,
save=True,
name="figures/l2reg_optimization/" + str(l2_reg),
subtitle=subtitle)
print("Val accuracy:", val_acc)
print("Test accuracy:", test_acc)
return val_acc
if __name__ == "__main__":
# Load data
x_train, y_train = LoadXY("data_batch_1")
x_val, y_val = LoadXY("data_batch_2")
x_test, y_test = LoadXY("test_batch")
# Preprocessing
mean_x = np.mean(x_train)
std_x = np.std(x_train)
x_train = (x_train - mean_x) / std_x
x_val = (x_val - mean_x) / std_x
x_test = (x_test - mean_x) / std_x
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Dense(nodes=50, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=50))
model.add(Softmax())
ns = 800
# Define callbacks
mt = MetricTracker() # Stores training evolution info
lrs = LearningRateScheduler(evolution="cyclic",
lr_min=1e-7,
lr_max=1e-2,
ns=ns) # Modifies lr while training
callbacks = [mt, lrs]
# print(y_train.shape)
# for i in range(200):
# cv2.imshow("image", x_train[..., i])
# cv2.waitKey()
# Define callbacks
mt = MetricTracker(file_name="cifar_test_3") # Stores training evolution info (losses and metrics)
# bms = BestModelSaver("models/best_cifar") # Stores training evolution info (losses and metrics)
lrs = LearningRateScheduler(evolution="cyclic", lr_min=1e-3, lr_max=0.2, ns=500)
# lrs = LearningRateScheduler(evolution="constant", lr_min=1e-3, lr_max=9e-1)
# callbacks = [mt, lrs]
callbacks = [mt, lrs]
# Define architecture (copied from https://appliedmachinelearning.blog/2018/03/24/achieving-90-accuracy-in-object-recognition-task-on-cifar-10-dataset-with-keras-convolutional-neural-networks/)
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Conv2D(num_filters=32, kernel_shape=(3, 3), stride=2, input_shape=(32, 32, 3)))
model.add(Relu())
model.add(Conv2D(num_filters=64, kernel_shape=(3, 3)))
model.add(Relu())
model.add(MaxPool2D(kernel_shape=(2, 2), stride=2))
model.add(Conv2D(num_filters=128, kernel_shape=(2, 2)))
model.add(Relu())
model.add(MaxPool2D(kernel_shape=(2, 2)))
model.add(Flatten())
model.add(Dense(nodes=200))
model.add(Relu())
model.add(Dense(nodes=10))
model.add(Softmax())
x_val, y_val = LoadXY("data_batch_2")
x_test, y_test = LoadXY("test_batch")
# Preprocessing
mean_x = np.mean(x_train)
std_x = np.std(x_train)
x_train = (x_train - mean_x) / std_x
x_val = (x_val - mean_x) / std_x
x_test = (x_test - mean_x) / std_x
x = x_train[:, 0:20]
y = y_train[:, 0:20]
reg = 0.1
# Define model
model = Sequential(loss=CrossEntropy())
model.add(Dense(nodes=50, input_dim=x_train.shape[0]))
model.add(Relu())
model.add(Dense(nodes=10, input_dim=50))
model.add(Softmax())
anal_time = time.time()
model.fit(
x,
y,
batch_size=None,
epochs=1,
lr=0, # 0 lr will not change weights
momentum=0,
l2_reg=reg)
analytical_grad = model.layers[0].gradient
y_train = np.concatenate((y_train, y), axis=1)
x_val = x_train[:, -1000:]
y_val = y_train[:, -1000:]
x_train = x_train[:, :-1000]
y_train = y_train[:, :-1000]
x_test, y_test = getXY(LoadBatch("test_batch"))
# Preprocessing
mean_x = np.mean(x_train)
std_x = np.std(x_train)
x_train = (x_train - mean_x) / std_x
x_val = (x_val - mean_x) / std_x
x_test = (x_test - mean_x) / std_x
# Modelling
model = Sequential(loss="categorical_hinge")
model.add(
Dense(nodes=10, input_dim=x.shape[0], weight_initialization="fixed"))
best_model = model.fit(
X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=20,
epochs=100,
lr=0.001, # 0 lr will not change weights
momentum=0.5,
l2_reg=0.05,
save_path="models/svm/test_2")
best_model.plot_training_progress(show=False,
# Put it in a data folder
# Load data
x_train, y_train = LoadXY("data_batch_1")
x_val, y_val = LoadXY("data_batch_2")
x_test, y_test = LoadXY("test_batch")
# Preprocessing
mean_x = np.mean(x_train)
std_x = np.std(x_train)
x_train = (x_train - mean_x) / std_x
x_val = (x_val - mean_x) / std_x
x_test = (x_test - mean_x) / std_x
# Define model
model = Sequential(loss=CrossEntropy())
model.add(Dense(nodes=10, input_dim=x_train.shape[0]))
model.add(Softmax())
# Fit model
# model.load("models/mlp_test")
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=100,
epochs=40,
lr=0.001,
momentum=0.0,
l2_reg=0.0,
shuffle_minibatch=False)
# x_val, y_val = getXY(LoadBatch("data_batch_2"))
# x_test, y_test = getXY(LoadBatch("test_batch"))
# Preprocessing
mean_x = np.mean(x_train)
std_x = np.std(x_train)
x_train = (x_train - mean_x)/std_x
# x_val = (x_val - mean_x)/std_x
# x_test = (x_test - mean_x)/std_x
x = x_train[:, 0:5]
y = y_train[:, 0:5]
reg = 0.1
# Define model
model = Sequential(loss="categorical_hinge")
model.add(Dense(nodes=10, input_dim=x.shape[0], weight_initialization="fixed"))
anal_time = time.time()
model.fit(x, y,
batch_size=10000, epochs=1, lr=0, # 0 lr will not change weights
momentum=0, l2_reg=reg)
analytical_grad = model.layers[0].gradient
anal_time = anal_time - time.time()
# Get Numerical gradient
num_time = time.time()
numerical_grad = ComputeGradsNum(x, y, model, l2_reg=reg, h=0.001)
print(numerical_grad.shape)
num_time = num_time - time.time()
x_val, y_val = getXY(LoadBatch("data_batch_2"))
x_test, y_test = getXY(LoadBatch("test_batch"))
# Preprocessing
mean_x = np.mean(x_train)
std_x = np.std(x_train)
x_train = (x_train - mean_x) / std_x
x_val = (x_val - mean_x) / std_x
x_test = (x_test - mean_x) / std_x
x = x_train[:, 0]
y = y_train[:, 0]
reg = 0.1
# Define model
model = Sequential(loss="cross_entropy")
model.add(
Dense(nodes=10, input_dim=x.shape[0], weight_initialization="fixed"))
model.add(Activation("softmax"))
anal_time = time.time()
model.fit(
x,
y,
batch_size=10000,
epochs=1,
lr=0, # 0 lr will not change weights
momentum=0,
l2_reg=reg)
analytical_grad = model.layers[0].gradient
anal_time = anal_time - time.time()
def evaluator(x_train, y_train, x_val, y_val, **kwargs):
# Define model
model = Sequential(loss="cross_entropy")
model.add(
Dense(nodes=10, input_dim=x_train.shape[0], weight_initialization="fixed"))
model.add(Activation("softmax"))
# Fit model
model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val, **kwargs)
model.plot_training_progress(show=False, save=True, name="figures/" + dict_to_string(kwargs))
# model.save("models/" + dict_to_string(kwargs))
# Minimizing value:
value = model.get_classification_metrics(x_val, y_val)[0] # Get accuracy
result = {"value": value, "model": model} # Save score and model
return result
# Put it in a Data folder
# Load data
x_train, y_train = getXY(LoadBatch("data_batch_1"))
x_val, y_val = getXY(LoadBatch("data_batch_2"))
x_test, y_test = getXY(LoadBatch("test_batch"))
# Preprocessing
mean_x = np.mean(x_train)
std_x = np.std(x_train)
x_train = (x_train - mean_x) / std_x
x_val = (x_val - mean_x) / std_x
x_test = (x_test - mean_x) / std_x
# Define SVM multi-class model
model = Sequential(loss="categorical_hinge")
model.add(Dense(nodes=10, input_dim=x_train.shape[0]))
model.add(Activation("softmax"))
# Fit model
model.fit(X=x_train,
Y=y_train,
X_val=x_val,
Y_val=y_val,
batch_size=100,
epochs=100,
lr=0.0001,
momentum=0.1,
l2_reg=0.1)
model.plot_training_progress()
return grads_w, grads_b
if __name__ == "__main__":
x_train, y_train, x_val, y_val, x_test, y_test = read_cifar_10(n_train=3, n_val=5, n_test=2)
# x_train, y_train, x_val, y_val, x_test, y_test = read_mnist(n_train=2, n_val=5, n_test=2)
# x_train, y_train, x_val, y_val, x_test, y_test = read_names(n_train=500)
class_sum = np.sum(y_train, axis=1)*y_train.shape[0]
class_count = np.reciprocal(class_sum, where=abs(class_sum) > 0)
print(class_count)
print(type(x_train[0, 0, 0]))
# Define model
model = Sequential(loss=CrossEntropy(), metric=Accuracy())
model.add(Conv2D(num_filters=2, kernel_shape=(4, 4), stride=3, dilation_rate=2, input_shape=x_train.shape[0:-1]))
model.add(Relu())
model.add(MaxPool2D((2, 2), stride=3))
model.add(Flatten())
model.add(Dense(nodes=y_train.shape[0]))
model.add(Relu())
model.add(Softmax())
print(np.min(np.abs(model.layers[0].filters)))
reg = 0.0
# Fit model
anal_time = time.time()
model.fit(X=x_train, Y=y_train, X_val=x_val, Y_val=y_val,
"/Toy-DeepLearning-Framework/")
from mlp.metrics import Accuracy
from mlp.models import Sequential
from mlp.losses import CrossEntropy
from mlp.layers import Conv2D, Dense, Softmax, Relu, Flatten, Dropout, MaxPool2D
from mlp.callbacks import MetricTracker, BestModelSaver, LearningRateScheduler
from mlp.utils import plot_confusion_matrix
np.random.seed(1)
if __name__ == "__main__":
# Load data
x_train, y_train, x_val, y_val, _, _ = read_names(n_train=-1)
print(x_train.shape)
classes = read_names_countries()
# Load model model
model = Sequential(loss=CrossEntropy())
# model.load("models/names_test")
model.load(
"models/name_metaparam_search_2/n1-39_n2-33_k1-2_k2-10_batch_size-50")
y_pred_train = model.predict_classes(x_train)
y_pred_val = model.predict_classes(x_val)
plot_confusion_matrix(y_pred_train, y_train, classes,
"figures/conf_best_model")
plot_confusion_matrix(y_pred_val, y_val, classes,
"figures/conf_best_model_val")
