def default_model():
"""
Function containing the code definition for training and evaluating the default required model
"""
model = Sequential(Linear(2, 25), ReLU(), Linear(25, 25), ReLU(),
Linear(25, 25), ReLU(), Linear(25, 2))
train_input, train_target = generate_disc_set(1000)
test_input, test_target = generate_disc_set(1000)
values = {"lr": [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]}
cross_validate = args.cross_val
best_lr = 1e-4
optimizer = SGDCV(model,
nb_epochs=50,
mini_batch_size=1,
lr=best_lr,
criterion=LossMSE())
if cross_validate:
optimizer.cross_validate(k=5, values=values, verbose=True)
optimizer.set_params()
optimizer.train(train_input, train_target, verbose=True)
evaluator = Evaluator(model)
print("Train accuracy: {:.1f}%".format(
(evaluator.compute_accuracy(train_input, train_target) * 100).item()))
print("Test accuracy: {:.1f}%".format(
(evaluator.compute_accuracy(test_input, test_target) * 100).item()))
def test_backward(self):
x = np.random.random(100)
x[::2] *= -1
layer = ReLU()
layer.forward(x)
grad = layer.backward(1)
assert np.all(grad[::2] == 0) and np.all(grad[1::2] == 1)
def build_model():
model = Sequential(MSE(), input_size=2)
model.add_layer(Linear(2, 25))
model.add_layer(ReLU(25))
model.add_layer(Linear(25, 25))
model.add_layer(ReLU(25))
model.add_layer(Linear(25, 25))
model.add_layer(Tanh(25))
model.add_layer(Linear(25, 2))
return model
def test_relu_calculate_gradient(self):
# Given
pre_activation = np.array([[1, 2], [3, -3], [0, -2]])
relu = ReLU()
# When
grad = relu.calculate_gradient(pre_activation, None)
# Then
expected = np.array([[1, 1], [1, 0], [0, 0]])
self.assertTrue(np.array_equal(grad, expected))
def test_relu_apply_activation(self):
# Given
pre_activation = np.array([[1], [0], [-4]])
relu = ReLU()
# When
activation = relu.apply_activation(pre_activation)
# Then
expected = np.array([[1], [0], [0]])
self.assertTrue(np.array_equal(activation, expected))
def test_relu_activation(N=None):
from activations import ReLU
N = np.inf if N is None else N
mine = ReLU()
gold = lambda z: F.relu(torch.FloatTensor(z)).numpy()
i = 0
while i < N:
n_dims = np.random.randint(1, 100)
z = random_stochastic_matrix(1, n_dims)
assert_almost_equal(mine.fn(z), gold(z))
print("PASSED")
i += 1
def _build_encoder(self):
"""
CNN encoder
Conv1 -> ReLU -> MaxPool1 -> Conv2 -> ReLU -> MaxPool2 ->
Flatten -> FC1 -> ReLU -> FC2
"""
self.encoder = OrderedDict()
self.encoder["Conv1"] = Conv2D(
act_fn=ReLU(),
init=self.init,
pad=self.enc_conv1_pad,
optimizer=self.optimizer,
out_ch=self.enc_conv1_out_ch,
stride=self.enc_conv1_stride,
kernel_shape=self.enc_conv1_kernel_shape,
)
self.encoder["Pool1"] = Pool2D(
mode="max",
optimizer=self.optimizer,
stride=self.enc_pool1_stride,
kernel_shape=self.enc_pool1_kernel_shape,
)
self.encoder["Conv2"] = Conv2D(
act_fn=ReLU(),
init=self.init,
pad=self.enc_conv2_pad,
optimizer=self.optimizer,
out_ch=self.enc_conv2_out_ch,
stride=self.enc_conv2_stride,
kernel_shape=self.enc_conv2_kernel_shape,
)
self.encoder["Pool2"] = Pool2D(
mode="max",
optimizer=self.optimizer,
stride=self.enc_pool2_stride,
kernel_shape=self.enc_pool2_kernel_shape,
)
self.encoder["Flatten3"] = Flatten(optimizer=self.optimizer)
self.encoder["FC4"] = FullyConnected(
n_out=self.latent_dim, act_fn=ReLU(), optimizer=self.optimizer
)
self.encoder["FC5"] = FullyConnected(
n_out=self.T * 2,
optimizer=self.optimizer,
act_fn=Affine(slope=1, intercept=0),
init=self.init,
)
def set_params(self, summary_dict):
cids = self.hyperparameters["component_ids"]
for k, v in summary_dict["parameters"].items():
if k == "components":
for c, cd in summary_dict["parameters"][k].items():
if c in cids:
getattr(self, c).set_params(cd)
elif k in self.parameters:
self.parameters[k] = v
for k, v in summary_dict["hyperparameters"].items():
if k == "components":
for c, cd in summary_dict["hyperparameters"][k].items():
if c in cids:
getattr(self, c).set_params(cd)
if k in self.hyperparameters:
if k == "act_fn" and v == "ReLU":
self.hyperparameters[k] = ReLU()
elif v == "act_fn" and v == "Sigmoid":
self.hyperparameters[k] = Sigmoid()
elif v == "act_fn" and v == "Tanh":
self.hyperparameters[k] = Tanh()
elif v == "act_fn" and "Affine" in v:
r = r"Affine\(slope=(.*), intercept=(.*)\)"
slope, intercept = re.match(r, v).groups()
self.hyperparameters[k] = Affine(float(slope),
float(intercept))
elif v == "act_fn" and "Leaky ReLU" in v:
r = r"Leaky ReLU\(alpha=(.*)\)"
alpha = re.match(r, v).groups()[0]
self.hyperparameters[k] = LeakyReLU(float(alpha))
else:
self.hyperparameters[k] = v
def test_relu_grad(N=None):
from activations import ReLU
N = np.inf if N is None else N
mine = ReLU()
gold = torch_gradient_generator(F.relu)
i = 0
while i < N:
n_ex = np.random.randint(1, 100)
n_dims = np.random.randint(1, 100)
z = random_tensor((n_ex, n_dims))
assert_almost_equal(mine.grad(z), gold(z))
print("PASSED")
i += 1
def test_neural_net_back_forward(self):
n_in, n_out = 3, 2
weights = np.array([[0, -1, 2], [-3, 4, -5]])
bias = np.arange(n_out)[:, np.newaxis]
nn = NeuralNet(MeanSquaredError(),
1e-3,
layers=[Linear(n_in, 2, weights, bias),
ReLU()])
x = np.array([[[0], [1], [2]]])
y = np.array([[[2], [3]]])
assert y.shape[1] == n_out
# |0 -1 2| |0| |0| | 3| |0| | 3| |3|
# |-3 4 -5| |1| + |1| = |-6| + |1| = |-5| -> |0|
# |2|
pred = nn.forward(x)
assert np.array_equal(pred, [[[3], [0]]])
nn.compute_loss(pred, y)
dL_dx = nn.backward()
# |0 -1 2| |0 + dx1| | 3 + 0 - dx2 + 2dx3| | 3 + ...| |3 - dx2 + 2dx3|
# |-3 4 -5| |1 + dx2| = |-6 - 3dx1 + 4dx2 - 5dx3| = |-5 + ...| -> |0|
# |2 + dx3| The second component is ReLU'ed away
# MSE loss results in 2( ... ) so dL = -2dx2 + 4dx3, dL/dx = |0, -2, 4|
assert np.array_equal(dL_dx, [[[0], [-2], [4]]])
def sequential(self,
network=[1, 1, 1],
activation=[ReLU(), Softmax()],
loss=Cross_entropy(),
regu=Default(),
weight_type='default'):
self.net = network
self.activation1 = activation[0]
self.activation2 = activation[1]
self.loss = loss
self.regu = regu
self.init_weight(weight_type)
def build_model(self, prospective):
print(
"initializing..., learing rate %s, n_layers %s, n_hidden %s, dropout rate %s."
% (self.lr, self.n_layers, self.n_hidden, self.rate_dropout))
self.model = Sequential()
self.model.add(GaussianNoise(stddev=0.01,
input_shape=self.input_shape))
# self.model.add(LocallyConnected1D(self.input_shape[1] * 2, 3))
# self.model.add(Dropout(rate=0.5))
for i in range(0, self.n_layers - 1):
self.model.add(
LSTM(self.n_hidden * 4,
return_sequences=True,
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
dropout=self.rate_dropout,
recurrent_dropout=self.rate_dropout))
self.model.add(
LSTM(self.n_hidden,
return_sequences=False,
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
dropout=self.rate_dropout,
recurrent_dropout=self.rate_dropout))
self.model.add(
Dense(prospective,
kernel_initializer=initializers.glorot_uniform()))
self.model.add(
BatchNormalization(
axis=-1,
moving_mean_initializer=Constant(value=0.5),
moving_variance_initializer=Constant(value=0.25)))
# if self.loss == risk_estimation:
# self.model.add(BatchNormalization(axis=-1, beta_initializer='ones'))
self.model.add(ReLU(alpha=0.0))
opt = RMSprop(lr=self.lr)
self.model.compile(loss="mse",
optimizer=opt,
metrics=['mean_squared_error'])
def _build_decoder(self):
"""
MLP decoder
FC1 -> ReLU -> FC2 -> Sigmoid
"""
self.decoder = OrderedDict()
self.decoder["FC1"] = FullyConnected(
act_fn=ReLU(),
init=self.init,
n_out=self.latent_dim,
optimizer=self.optimizer,
)
# NB. `n_out` is dependent on the dimensionality of X. we use a
# placeholder for now, and update it within the `forward` method
self.decoder["FC2"] = FullyConnected(
n_out=None, act_fn=Sigmoid(), optimizer=self.optimizer, init=self.init
)
def init_from_str(self, act_str):
act_str = act_str.lower()
if act_str == "relu":
act_fn = ReLU()
elif act_str == "tanh":
act_fn = Tanh()
elif act_str == "sigmoid":
act_fn = Sigmoid()
elif "affine" in act_str:
r = r"affine\(slope=(.*), intercept=(.*)\)"
slope, intercept = re.match(r, act_str).groups()
act_fn = Affine(float(slope), float(intercept))
elif "leaky relu" in act_str:
r = r"leaky relu\(alpha=(.*)\)"
alpha = re.match(r, act_str).groups()[0]
act_fn = LeakyReLU(float(alpha))
else:
raise ValueError("Unknown activation: {}".format(act_str))
return act_fn
def main():
# 先讀取資料，並建立模型。
# 輸入的維度為一個資料的長度，因為資料量小，batch size即為資料總數。
x, y = ParityBits(8).load_data()
batch_size, input_dim = x.shape
model = Sequential(
[Dense(64, activation=ReLU()),
Dense(32, activation=Tanh()),
Dense(16, activation=Tanh()),
Dense(4, activation=None),
Dense(1, activation=Sigmoid())],
input_dim=input_dim,
# 使用GD為優化器，MSE為損失函式。
optimizer=GradientDescent(learning_rate=0.01, momentum=0.0),
loss=MeanSquaredError())
# 設定好epochs後訓練模型，訓練完後取得預測結果和每個epoch的損失值。
y_pred, losses = model.train(
x, y, batch_size=batch_size, epochs=200, verbose_step=10)
# 因為答案皆為整數0或1，因此訓練的成果為模型預測的結果取整數。
result = np.around(y_pred).astype(int)
# 將答案與訓練成果相減。
diff = np.subtract(y, result)
print(pd.DataFrame({
# 印出表格時，須將輸入的資料的每項陣列例如`[0 0 0 0 0 0 0 0]`轉成字串，
# 因為Pandas的DataFrame的每一項不能吃陣列。
"Data": [np.array_str(v) for v in x],
"Answer": y[:, 0],
"Prediction": [f'{v:.8f}' for v in y_pred[:, 0]],
"Result": result[:, 0],
# 如果答案與訓練成果在相減之後為0的話代表預測正確，否則失敗。
"Correct": [True if v == 0 else False for v in diff[:, 0]]
}, index=np.arange(1, len(x) + 1)).to_string())
# 輸出最後的損失值和訓練成果與答案差了幾項，並繪製每個epoch與其損失值的變化圖表。
print(f'loss: {losses[-1]:.8f}, difference: {np.count_nonzero(diff)}')
plt.figure(figsize=(8, 4))
plt.plot(losses)
plt.xlabel('epoch')
plt.ylabel('loss')
plt.show()
def plot_activations():
fig, axes = plt.subplots(2, 3, sharex=True, sharey=True)
fns = [Affine(), Tanh(), Sigmoid(), ReLU(), LeakyReLU(), ELU()]
for ax, fn in zip(axes.flatten(), fns):
X = np.linspace(-3, 3, 100).astype(float).reshape(100, 1)
ax.plot(X, fn(X), label=r"$y$", alpha=0.7)
ax.plot(X, fn.grad(X), label=r"$\frac{dy}{dx}$", alpha=0.7)
ax.plot(X, fn.grad2(X), label=r"$\frac{d^2 y}{dx^2}$", alpha=0.7)
ax.hlines(0, -3, 3, lw=1, linestyles="dashed", color="k")
ax.vlines(0, -1.2, 1.2, lw=1, linestyles="dashed", color="k")
ax.set_ylim(-1.1, 1.1)
ax.set_xlim(-3, 3)
ax.set_xticks([])
ax.set_yticks([-1, 0, 1])
ax.xaxis.set_visible(False)
# ax.yaxis.set_visible(False)
ax.set_title("{}".format(fn))
ax.legend(frameon=False)
sns.despine(left=True, bottom=True)
fig.set_size_inches(8, 5)
plt.tight_layout()
plt.savefig("plot.png", dpi=300)
plt.close("all")
def main():
"""
Function containing the main code definition, display all functionalities provided by the framework
"""
# Different activation functions and setting of automatic Xavier parameter initialization
relu_model = Sequential(Linear(2, 25),
ReLU(),
Linear(25, 25),
ReLU(),
Linear(25, 25),
ReLU(),
Linear(25, 2),
xavier_init=True)
leaky_relu_model = Sequential(Linear(2, 25),
LeakyReLU(),
Linear(25, 25),
LeakyReLU(),
Linear(25, 25),
LeakyReLU(),
Linear(25, 2),
xavier_init=True)
tanh_model = Sequential(Linear(2, 25),
Tanh(),
Linear(25, 25),
Tanh(),
Linear(25, 25),
Tanh(),
Linear(25, 2),
xavier_init=True)
sigmoid_model = Sequential(Linear(2, 25),
Sigmoid(),
Linear(25, 25),
Sigmoid(),
Linear(25, 25),
Sigmoid(),
Linear(25, 2),
xavier_init=False)
model_names = ["ReLU", "Leaky", "Tanh", "Sigmoid"]
train_input, train_target = generate_disc_set(1000)
test_input, test_target = generate_disc_set(1000)
# Model training without cross-validation of the optimizer parameters
optimizer = SGDCV(leaky_relu_model, nb_epochs=25)
optimizer.train(train_input, train_target)
evaluator = Evaluator(leaky_relu_model)
print("Train accuracy using LeakyReLU: {:.1f}%".format(
(evaluator.compute_accuracy(train_input, train_target) * 100).item()))
print("Test accuracy using LeakyReLU: {:.1f}%".format(
(evaluator.compute_accuracy(test_input, test_target) * 100).item()))
models = (relu_model, leaky_relu_model, tanh_model, sigmoid_model)
sgd_cross_val_param_grid = {"lr": [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]}
adam_cross_val_param_grid = {
"lr": [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1],
"b1": [0.9, 0.8],
"b2": [0.999, 0.888],
"epsilon": [1e-8, 1e-7, 1e-6]
}
adam_params = {
"ReLU": {
"lr": [0.001],
"b1": [0.9],
"b2": [0.999],
"epsilon": [1e-08]
},
"Leaky": {
"lr": [0.001],
"b1": [0.9],
"b2": [0.999],
"epsilon": [1e-08]
},
"Tanh": {
"lr": [0.001],
"b1": [0.9],
"b2": [0.999],
"epsilon": [1e-08]
},
"Sigmoid": {
"lr": [0.001],
"b1": [0.9],
"b2": [0.999],
"epsilon": [1e-08]
}
}
sgd_params = {
"ReLU": {
"lr": [0.001]
},
"Leaky": {
"lr": [0.001]
},
"Tanh": {
"lr": [0.001]
},
"Sigmoid": {
"lr": [0.01]
}
}
mse_loss = not args.CE
optimizer_sgd = not args.Adam
cross_validate = args.cross_val
# Different loss functions
if mse_loss:
criterion = LossMSE()
else:
criterion = LossCrossEntropy()
for name, model in zip(model_names, models):
if optimizer_sgd:
# SGD optimizer parameter cross-validation
optimizer = SGDCV(model, mini_batch_size=10, criterion=criterion)
if cross_validate:
params = sgd_cross_val_param_grid
else:
params = sgd_params[name]
cross_val_results, best_params_score = optimizer.cross_validate(
values=params)
print("Best params for model using {} : (lr={:.3f})".format(
name, best_params_score["lr"]))
else:
# Adam optimizer parameter cross-validation
optimizer = AdamCV(model, mini_batch_size=10, criterion=criterion)
if cross_validate:
params = adam_cross_val_param_grid
else:
params = adam_params[name]
cross_val_results, best_params_score = optimizer.cross_validate(
values=params)
print(
"Best params for model using {} : (lr={:.3f}, b1={:.3f}, b2={:.3f}, epsilon={:.1e})"
.format(name, best_params_score["lr"], best_params_score["b1"],
best_params_score["b2"], best_params_score["epsilon"]))
print("Best score for model using {} : {:.3f} (+/- {:.3f})".format(
name, best_params_score["mean"], best_params_score["std"]))
def cross_val_results(verbose=True):
"""
Function for generating the accuracy results of four models presented in the report with their best parameters,
averaged over 10 runs and using different combinations of the available optimizers and loss
:param verbose: whether to print average results for each (Model, Optimizer, Loss) combination,
boolean, optional, default is True
:returns: list of tuples containing (mean, std) of each (Model, Optimizer, Loss) combination, each tuple in [0, 1]^2
"""
datasets = []
for i in range(10):
datasets.append((generate_disc_set(1000), generate_disc_set(1000)))
relu_model = Sequential(Linear(2, 25), ReLU(),
Linear(25, 25), ReLU(),
Linear(25, 25), ReLU(),
Linear(25, 2), xavier_init=True)
leaky_relu_model = Sequential(Linear(2, 25), LeakyReLU(),
Linear(25, 25), LeakyReLU(),
Linear(25, 25), LeakyReLU(),
Linear(25, 2), xavier_init=True)
tanh_model = Sequential(Linear(2, 25), Tanh(),
Linear(25, 25), Tanh(),
Linear(25, 25), Tanh(),
Linear(25, 2), xavier_init=True)
sigmoid_model = Sequential(Linear(2, 25), Sigmoid(),
Linear(25, 25), Sigmoid(),
Linear(25, 25), Sigmoid(),
Linear(25, 2))
models = [relu_model, leaky_relu_model, tanh_model, sigmoid_model]
final_scores = []
optimizers_names = ["SGD", "Adam"]
models_names = ["ReLU", "Leaky", "Tanh", "Sigmoid"]
losses_names = ["MSE", "CrossEntropy"]
losses = [LossMSE(), LossCrossEntropy()]
adam_params = {"ReLU": {"lr": 0.001, "b1": 0.9, "b2": 0.999, "epsilon": 1e-08},
"Leaky": {"lr": 0.001, "b1": 0.9, "b2": 0.999, "epsilon": 1e-08},
"Tanh": {"lr": 0.001, "b1": 0.9, "b2": 0.999, "epsilon": 1e-08},
"Sigmoid": {"lr": 0.001, "b1": 0.9, "b2": 0.999, "epsilon": 1e-08}}
sgd_params = {"ReLU": {"lr": 0.001},
"Leaky": {"lr": 0.001},
"Tanh": {"lr": 0.001},
"Sigmoid": {"lr": 0.01}}
for optim_name in optimizers_names:
for loss_name, loss in zip(losses_names, losses):
for model_name, model in zip(models_names, models):
if verbose:
print("Validating model {} with {} and {} loss...".format(model_name, optim_name, loss_name),
end='')
scores = []
if optim_name == "Adam":
params = adam_params[model_name]
optim = Adam(model, criterion=loss, nb_epochs=50, mini_batch_size=10, lr=params["lr"],
b1=params["b1"], b2=params["b2"], epsilon=params["epsilon"])
else:
params = sgd_params[model_name]
optim = SGD(relu_model, criterion=loss, nb_epochs=50, mini_batch_size=10, lr=params["lr"])
for ((train_input, train_target), (test_input, test_target)) in datasets:
optim.model = copy.deepcopy(model)
optim.train(train_input, train_target, verbose=False)
evaluator = Evaluator(optim.model)
accuracy = evaluator.compute_accuracy(test_input, test_target)
scores.append(accuracy)
scores = torch.FloatTensor(scores)
scores_mean = torch.mean(scores).item()
scores_var = torch.std(scores).item()
if verbose:
print("Score : {0:.3f} (+/- {1:.3f}) ".format(scores_mean, scores_var))
final_scores.append((scores_mean, scores_var))
return final_scores
def test_forward(self):
x = np.random.random(100)
x[::2] *= -1
y = ReLU().forward(x)
assert np.all(y[::2] == 0) and np.all(y[1::2] == x[1::2])
from loss import CrossEntropyLoss
from utils import save_weights, load_weights
from evaluation import Evaluation
from visualize import img_viewer_examples
from cnn import *
import time
import numpy as np
batch_size = 1
dataset = MNIST_dataset("./datasets/train.csv")
dataloader = Data_Loader(dataset, batch_size)
lenet = Model()
lenet.add(conv(1, 6, 5, padding=2))
lenet.add(ReLU())
lenet.add(AvgPool2D(kernel_size=2))
lenet.add(conv(6, 16, 5))
lenet.add(ReLU())
lenet.add(AvgPool2D(kernel_size=2))
lenet.add(Flatten())
lenet.add(Dense(16 * 5 * 5, 120))
lenet.add(ReLU())
lenet.add(Dense(120, 84))
lenet.add(ReLU())
lenet.add(Dense(84, 10))
lenet.set_loss(CrossEntropyLoss())
optimizer = GradientDecent(lenet.parameters(), learning_rate=0.01)
print("weight=\n", np.array(weight[l]), file=outputfile)
print("bias=\n", np.array(bias[l]), file=outputfile)
addlayer = onelayer(inputvector=layervector,
activation=self.activation,
weight=np.array(weight[l]),
bias=np.array(bias[l]))
preoutput = addlayer.preoutput()
preoutputsequence.append(preoutput)
layervector = addlayer.output()
outputsequence.append(layervector)
return layervector, outputsequence, preoutputsequence
"""
test the output
"""
if __name__ == "__main__":
L = 3 #number of hidden layers#
n = np.random.randint(
1, 5,
size=L) #network size for each hidden layer n[0]=n_1, ..., m[L-1]=n_L#
print("hidden layer sizes=", n, file=outputfile)
network = fullnetwork(L=L, n=n, activation=ReLU())
weight, bias = network.setparameter()
networkoutput, outputsequence, preoutputsequence = network.output(
1, weight, bias)
print("\nnetwork output=", float(networkoutput), file=outputfile)
print("outputsequence=", outputsequence, file=outputfile)
print("preoutoputsequence=", preoutputsequence, file=outputfile)
outputfile.close()
dataloader = Data_Loader(dataset, batch_size)
test_dataset = MNIST_dataset("./datasets/train.csv")
dataloader_test = Data_Loader(test_dataset, batch_size)
image, labels = dataloader.__getitem__(1)
images = image.T.reshape(batch_size, 1, 28, 28)
images = np.asarray(images)
img_viewer_examples(images, labels.tolist()[0], greyscale= True)
model = Model()
model.add(Dense(784, 90))
model.add(ReLU())
model.add(Dense(90, 45))
model.add(ReLU())
model.add(Dense(45, 10))
model.set_loss(CrossEntropyLoss())
optimizer = Adam(model.parameters(), learning_rate = 0.01)
lr_schedular = StepLR(optimizer, step_size = 1, gamma=0.1)
# weights path
path = "./checkpoints/Linear_MINST_weights.sav"
# model = load_weights(path)
epochs = 6
from loss import CrossEntropyLoss
from utils import save_weights, load_weights
from evaluation import Evaluation
from visualize import img_viewer_examples
from cnn import *
import time
import numpy as np
batch_size = 1
dataset = MNIST_dataset("./datasets/train.csv")
dataloader = Data_Loader(dataset, batch_size)
model = Model()
model.add(conv(1, 4, 3, padding=1))
model.add(MaxPool2D(kernel_size=(2, 2)))
model.add(ReLU())
model.add(conv(4, 8, 3, padding=1))
model.add(MaxPool2D(kernel_size=2))
model.add(ReLU())
model.add(Flatten())
model.add(Dense(8 * 7 * 7, 10))
model.set_loss(CrossEntropyLoss())
optimizer = Adam(model.parameters(), learning_rate=0.01)
lr_schedular = StepLR(optimizer, step_size=1, gamma=0.1)
epochs = 1
for epoch in range(epochs):
i = 0
for image, label in dataloader:
image = image / 255
