def get_model(args):
loss_dict = {}
softmaxLoss = SoftmaxCrossEntropyLoss("softmax")
euclideanLoss = EuclideanLoss("euclidean")
loss_dict['softmax'] = softmaxLoss
loss_dict['euclidean'] = euclideanLoss
config = {
'learning_rate': args.learning_rate,
'weight_decay': args.weight_decay,
'momentum': args.momentum,
'batch_size': args.batch_size,
'max_epoch': args.max_epoch,
'disp_freq': args.disp_freq,
'test_epoch': args.test_epoch
}
loss = loss_dict[args.loss]
model = Network()
layer = args.hidden_layer
if layer == 1:
model.add(Linear('fc1', 784, args.hidden_size, 0.01))
model.add(get_activation(args.activation, 0))
model.add(Linear('fc2', args.hidden_size, 10, 0.01))
model.add(get_activation(args.activation, 1))
else:
model.add(Linear('fc1', 784, args.hidden_size, 0.01))
model.add(get_activation(args.activation, 0))
model.add(Linear('fc2', args.hidden_size, args.hidden_size//2, 0.01))
model.add(get_activation(args.activation, 1))
model.add(Linear('fc2', args.hidden_size//2, 10, 0.01))
model.add(get_activation(args.activation, 2))
return model, config, loss
def build_model(config):
model = Network()
layer_num = 0
for layer in config['use_layer']:
if layer['type'] == "Linear":
in_num = layer['in_num']
out_num = layer['out_num']
if "init_std" in layer.keys():
model.add(
Linear(layer['type'] + str(layer_num),
in_num,
out_num,
init_std=layer['init_std']))
else:
model.add(
Linear(layer['type'] + str(layer_num), in_num, out_num))
layer_num += 1
elif layer['type'] == 'Relu':
model.add(Relu(layer['type'] + str(layer_num)))
layer_num += 1
elif layer['type'] == 'Sigmoid':
model.add(Sigmoid(layer['type'] + str(layer_num)))
layer_num += 1
else:
assert 0
loss_name = config['use_loss']
if loss_name == 'EuclideanLoss':
loss = EuclideanLoss(loss_name)
elif loss_name == 'SoftmaxCrossEntropyLoss':
loss = SoftmaxCrossEntropyLoss(loss_name)
else:
assert 0
return model, loss
def getNetwork():
'''
to obtain network structure from specified file
'''
file_name = "models/structure.json"
if len(sys.argv)>1:
file_name = sys.argv[1]
f = file(file_name, "r")
s = f.read()
f.close()
networks = json.loads(s)
for network in networks:
config = network['config']
dis_model = network['model']
model = Network()
for layer in dis_model:
if layer['type'] == 'Linear':
model.add(Linear(layer['name'], layer['in_num'], layer['out_num'], layer['std']))
if layer['type'] == 'Relu':
model.add(Relu(layer['name']))
if layer['type'] == 'Sigmoid':
model.add(Sigmoid(layer['name']))
if layer['type'] == 'Softmax':
model.add(Softmax(layer['name']))
loss = EuclideanLoss('loss')
if 'loss' in config:
if config['loss'] == 'CrossEntropyLoss':
loss = CrossEntropyLoss('loss')
yield network['name'], model, config, loss
def Model_Linear_Relu_1_EuclideanLoss():
name = '1_Relu_EuclideanLoss'
model = Network()
model.add(Linear('fc1', 784, 256, 0.01))
model.add(Relu('a1'))
model.add(Linear('fc2', 256, 10, 0.01))
loss = EuclideanLoss(name='loss')
return name, model, loss
def Model_Linear_Relu_2_EuclideanLoss():
name = '2_Relu_EuclideanLoss'
model = Network()
model.add(Linear('fc1', 784, 441, 0.01))
model.add(Relu('a1'))
model.add(Linear('fc2', 441, 196, 0.01))
model.add(Relu('a2'))
model.add(Linear('fc3', 196, 10, 0.01))
loss = EuclideanLoss(name='loss')
return name, model, loss
from layers import Relu, Sigmoid, Linear
from loss import EuclideanLoss
from solve_net import train_net, test_net
from load_data import load_mnist_2d
import numpy as np
train_data, test_data, train_label, test_label = load_mnist_2d('data')
# Your model defintion here
# You should explore different model architecture
model = Network()
model.add(Linear('fc1', 784, 100, 0.01))
model.add(Sigmoid('Sigmoid1'))
model.add(Linear('fc2', 100, 10, 0.01))
loss = EuclideanLoss(name='loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': 0.08,
'weight_decay': 0.001,
'momentum': 0.9,
'batch_size': 80,
'max_epoch': 100,
'disp_freq': 10,
'test_epoch': 1
model4 = Network(name='model4')
model4.add(Linear('m4_fc1', 784, 512, 0.01))
model4.add(Relu('m4_fc2'))
model4.add(Linear('m4_fc3', 512, 128, 0.01))
model4.add(Relu('m4_fc4'))
model4.add(Linear('m4_fc5', 128, 10, 0.01))
model5 = Network(name='model5')
model5.add(Linear('m5_fc1', 784, 392, 0.01))
model5.add(Relu('m5_fc2'))
model5.add(Linear('m5_fc3', 392, 196, 0.01))
model5.add(Relu('m5_fc4'))
model5.add(Linear('m5_fc5', 196, 10, 0.01))
loss1 = EuclideanLoss(name='Euclidean')
loss2 = SoftmaxCrossEntropyLoss(name='XEntropy')
#models = [model1, model2, model3, model4, model5]
#losses = [loss1, loss2]
model = model4
loss = loss2
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': 0.01,
yl = np.array(loss_list)
t = np.array(time_list)
return [final_acc, end_time - start_time, x, ya, yl, t]
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--train_one_layer", default=False)
parser.add_argument("--train_two_layer", default=False)
parser.add_argument("--modified_gd", default=False)
parser.add_argument("--stop_time", default=0, type=int)
args = parser.parse_args()
train_data, test_data, train_label, test_label = load_mnist_2d('data')
loss1 = EuclideanLoss(name="euclidean loss")
loss2 = SoftmaxCrossEntropyLoss(name="softmax cross entropy loss")
config = {
'learning_rate': 0.01,
'weight_decay': 0.001,
'momentum': 0.8,
'batch_size': 64,
'max_epoch': 50,
'disp_freq': 1000,
'test_epoch': 2,
'stop_time': args.stop_time
}
if Type(args.train_one_layer):
config['max_epoch'] = 50
if args.layers == 0:
model.add(Linear('fc', 784, 10, args.std))
elif args.layers == 1:
model.add(Linear('fc1', 784, 256, args.std))
model.add(activation('act'))
model.add(Linear('fc2', 256, 10, args.std))
else:
model.add(Linear('fc1', 784, 256, args.std))
model.add(activation('act'))
model.add(Linear('fc2', 256, 128, args.std))
model.add(activation('act'))
model.add(Linear('fc3', 128, 10, args.std))
if args.loss == 'mse':
model.add(Sigmoid('sigmoid'))
loss = EuclideanLoss('loss')
else:
loss = SoftmaxCrossEntropyLoss('loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': args.lr,
'weight_decay': args.weight_decay,
'momentum': args.momentum,
'batch_size': args.batch_size,
'max_epoch': args.max_epoch,