def Model_Linear_Relu_1_SoftmaxCrossEntropyLoss():
name = '1_Relu_SoftmaxCrossEntropyLoss'
model = Network()
model.add(Linear('fc1', 784, 256, 0.01))
model.add(Relu('a1'))
model.add(Linear('fc2', 256, 10, 0.01))
loss = SoftmaxCrossEntropyLoss(name='loss')
return name, model, loss
def Model_Linear_Gelu_2_SoftmaxCrossEntropyLoss():
name = '2_Gelu_SoftmaxCrossEntropyLoss'
model = Network()
model.add(Linear('fc1', 784, 441, 0.01))
model.add(Gelu('a1'))
model.add(Linear('fc2', 441, 196, 0.01))
model.add(Gelu('a2'))
model.add(Linear('fc3', 196, 10, 0.01))
loss = SoftmaxCrossEntropyLoss(name='loss')
return name, model, loss
def basicConv2Layer():
model = Network()
model.add(Conv2D('conv1', 1, 4, 3, 1, 1))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 4, 4, 3, 1, 1))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 4 x 7 x 7
model.add(Reshape('flatten', (-1, 196)))
model.add(Linear('fc3', 196, 10, 0.1))
loss = SoftmaxCrossEntropyLoss(name='loss')
return model, loss
def LeNet():
model = Network()
model.add(Conv2D('conv1', 1, 6, 5, 2, 1))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 6 x 14 x 14
model.add(Conv2D('conv2', 6, 16, 5, 0, 1))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 16 x 5 x 5
model.add(Reshape('flatten', (-1, 400)))
model.add(Linear('fc1', 400, 120, 0.1))
model.add(Relu('relu3'))
model.add(Linear('fc2', 120, 84, 0.1))
model.add(Relu('relu4'))
model.add(Linear('fc3', 84, 10, 0.1))
loss = SoftmaxCrossEntropyLoss(name='loss')
return model, loss
def evaluate(model, data):
x_data = data['x']
y_data = data['y']
batch_size = 100
size = len(x_data)
correct = 0
loss_value = 0
loss = SoftmaxCrossEntropyLoss('loss')
for start_idx in range(0, size, batch_size):
end_idx = min(start_idx + batch_size, size)
x = np.array(x_data[start_idx:end_idx])
y = y_data[start_idx:end_idx]
ans = model.forward(x)
output = softmax(ans)
loss_value += len(y) * loss.forward(ans, onehot_encoding(y, 5))
correct += len(y) * calculate_acc(output, y)
return loss_value / size, correct / size
from solve_net import show4category
train_data, test_data, train_label, test_label = load_mnist_4d('data')
# Your model defintion here
# You should explore different model architecture
model = Network()
model.add(Conv2D('conv1', 1, 4, 3, 1, 0.01))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 4, 8, 3, 1, 0.01))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 8 x 7 x 7
model.add(Reshape('flatten', (-1, 392)))
model.add(Linear('fc3', 392, 10, 0.01))
loss = SoftmaxCrossEntropyLoss(name='loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': 0.01,
'weight_decay': 0,
'momentum': 0.7,
'batch_size': 100,
'max_epoch': 300,
'disp_freq': 5,
'test_epoch': 2
model4 = Network(name='model4')
model4.add(Linear('m4_fc1', 784, 512, 0.01))
model4.add(Relu('m4_fc2'))
model4.add(Linear('m4_fc3', 512, 128, 0.01))
model4.add(Relu('m4_fc4'))
model4.add(Linear('m4_fc5', 128, 10, 0.01))
model5 = Network(name='model5')
model5.add(Linear('m5_fc1', 784, 392, 0.01))
model5.add(Relu('m5_fc2'))
model5.add(Linear('m5_fc3', 392, 196, 0.01))
model5.add(Relu('m5_fc4'))
model5.add(Linear('m5_fc5', 196, 10, 0.01))
loss1 = EuclideanLoss(name='Euclidean')
loss2 = SoftmaxCrossEntropyLoss(name='XEntropy')
#models = [model1, model2, model3, model4, model5]
#losses = [loss1, loss2]
model = model4
loss = loss2
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': 0.01,
'weight_decay': 0.0,
t = np.array(time_list)
return [final_acc, end_time - start_time, x, ya, yl, t]
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--train_one_layer", default=False)
parser.add_argument("--train_two_layer", default=False)
parser.add_argument("--modified_gd", default=False)
parser.add_argument("--stop_time", default=0, type=int)
args = parser.parse_args()
train_data, test_data, train_label, test_label = load_mnist_2d('data')
loss1 = EuclideanLoss(name="euclidean loss")
loss2 = SoftmaxCrossEntropyLoss(name="softmax cross entropy loss")
config = {
'learning_rate': 0.01,
'weight_decay': 0.001,
'momentum': 0.8,
'batch_size': 64,
'max_epoch': 50,
'disp_freq': 1000,
'test_epoch': 2,
'stop_time': args.stop_time
}
if Type(args.train_one_layer):
config['max_epoch'] = 50
train_data, test_data, train_label, test_label = load_mnist_4d('data')
# Your model defintion here
# You should explore different model architecture
model = Network('CNN_test')
model.add(Conv2D('conv1', 1, 4, 3, 1, 0.1))
model.add(Relu('relu1'))
model.add(AvgPool2D('pool1', 2, 0)) # output shape: N x 4 x 14 x 14
model.add(Conv2D('conv2', 4, 4, 3, 1, 0.1))
model.add(Relu('relu2'))
model.add(AvgPool2D('pool2', 2, 0)) # output shape: N x 4 x 7 x 7
model.add(Reshape('flatten', (-1, 196)))
model.add(Linear('fc3', 196, 10, 0.1))
loss = SoftmaxCrossEntropyLoss(name='SoftmaxCrossEntropy')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': 0.01,
'weight_decay': 0.0,
'momentum': 0.9,
'batch_size': 100,
'max_epoch': 2,
'disp_freq': 100,
'layer_vis': 'relu1'
elif args.layers == 1:
model.add(Linear('fc1', 784, 256, args.std))
model.add(activation('act'))
model.add(Linear('fc2', 256, 10, args.std))
else:
model.add(Linear('fc1', 784, 256, args.std))
model.add(activation('act'))
model.add(Linear('fc2', 256, 128, args.std))
model.add(activation('act'))
model.add(Linear('fc3', 128, 10, args.std))
if args.loss == 'mse':
model.add(Sigmoid('sigmoid'))
loss = EuclideanLoss('loss')
else:
loss = SoftmaxCrossEntropyLoss('loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': args.lr,
'weight_decay': args.weight_decay,
'momentum': args.momentum,
'batch_size': args.batch_size,
'max_epoch': args.max_epoch,
'disp_freq': 50,
'test_epoch': 1
