def main(args): # Define a convolutional neural network the same as above net = builder.Sequential( builder.Convolution((7, 7), 32), builder.ReLU(), builder.Pooling('max', (2, 2), (2, 2)), builder.Reshape((flattened_input_size,)) builder.Affine(hidden_size), builder.Affine(num_classes), ) # Cast the definition to a model compatible with minpy solver model = builder.Model(net, 'softmax', (3 * 32 * 32,)) data = get_CIFAR10_data(args.data_dir) train_dataiter = NDArrayIter(data['X_train'], data['y_train'], batch_size=batch_size, shuffle=True) test_dataiter = NDArrayIter(data['X_test'], data['y_test'], batch_size=batch_size, shuffle=False) solver = Solver(model, train_dataiter, test_dataiter, num_epochs=10, init_rule='gaussian', init_config={ 'stdvar': 0.001 }, update_rule='sgd_momentum', optim_config={ 'learning_rate': 1e-3, 'momentum': 0.9 }, verbose=True, print_every=20) solver.init() solver.train()
def main(args): # Define a 2-layer perceptron MLP = builder.Sequential( builder.Affine(512), builder.ReLU(), builder.Affine(10) ) # Cast the definition to a model compatible with minpy solver model = builder.Model(MLP, 'softmax', (3 * 32 * 32,)) data = get_CIFAR10_data(args.data_dir) data['X_train'] = data['X_train'].reshape([data['X_train'].shape[0], 3 * 32 * 32]) data['X_val'] = data['X_val'].reshape([data['X_val'].shape[0], 3 * 32 * 32]) data['X_test'] = data['X_test'].reshape([data['X_test'].shape[0], 3 * 32 * 32]) train_dataiter = NDArrayIter(data['X_train'], data['y_train'], batch_size=100, shuffle=True) test_dataiter = NDArrayIter(data['X_test'], data['y_test'], batch_size=100, shuffle=False) solver = Solver(model, train_dataiter, test_dataiter, num_epochs=10, init_rule='gaussian', init_config={ 'stdvar': 0.001 }, update_rule='sgd_momentum', optim_config={ 'learning_rate': 1e-5, 'momentum': 0.9 }, verbose=True, print_every=20) solver.init() solver.train()
data = load_cifar10(path='../utilities/cifar/', center=True, rescale=True) X = data[0][:16] hidden_layers = 4 shapes = (1024, ) * hidden_layers + (10, ) activation = builder.ReLU storage = {} mlp = builder.Sequential() for i, shape in enumerate(shapes[:-1]): mlp.append(builder.Affine(shape)) mlp.append(builder.Export('affine%d' % i, storage)) mlp.append(activation()) mlp.append(builder.Affine(shapes[-1])) model = builder.Model(mlp, 'softmax', (3072, )) ''' for key, value in model.param_configs.items(): if 'weight' in key: value['init_rule'] = 'gaussian' value['init_config'] = {'stdvar' : 1} ''' initialize(model) for key, value in model.params.items(): if 'weight' in key: print np.std(value) result = model.forward(X, 'train') print 'result', np.std(result)
from GPU_utility import GPU_availability from minpy.context import set_context, gpu set_context(gpu(GPU_availability()[0])) ACTIVATION = 'ReLU' SHAPE = (1024, ) * 3 + (10, ) BATCH_SIZE = 64 X_SHAPE = (3072, ) activation = getattr(builder, ACTIVATION) mlp = builder.Sequential() for shape in SHAPE[:-1]: mlp.append(builder.Affine(shape)) mlp.append(activation()) mlp.append(builder.Affine(SHAPE[-1])) model = builder.Model(mlp, 'softmax', X_SHAPE) initialize(model) updater = Updater(model, 'sgd', {'learning_rate': 0.01}) training_X, training_Y, validation_X, validation_Y, test_X, test_Y, = \ load_cifar10(path='../../cifar10/utilities/cifar/', center=True, rescale=True) X_batches = Batches(training_X, BATCH_SIZE) Y_batches = Batches(training_Y, BATCH_SIZE) ITERATIONS = 20000 LOGGING_INTERVAL = 10 VALIDATION_INTERVAL = 50 loss_table = [] validation_accuracy_table = [] sample = lambda N, D: np.random.uniform(-1, 1, (N, D)) for i in range(ITERATIONS):
load_cifar10(path='../../utilities/cifar/', center=True, rescale=True) HIDDEN_LAYERS = 4 activation = sys.argv[1] # activation = 'ReLU' storage = {} mlp = MLP(*((1024, ) * HIDDEN_LAYERS + (10, )), activation=activation, affine_monitor=False, activation_monitor=False, storage=storage) ini_mode = sys.argv[2] # ini_mode = 'layer-by-layer' if ini_mode == 'layer-by-layer': model = builder.Model(mlp, 'softmax', (3072, ), training_X) else: model = builder.Model(mlp, 'softmax', (3072, )) solver = Solver(model, NDArrayIter(training_X, training_Y), NDArrayIter(test_X, test_Y), init_rule='xavier') solver.init() parameter_keys = list(model.params.keys()) parameter_values = list(model.params.values()) def loss_function(*args):
N, D = 50000, 16 data, p = generate_data(N, D) BATCH_SIZE = 100 X_batches = Batches(data, BATCH_SIZE) p_batches = Batches(p.reshape((N, 1)), BATCH_SIZE) ACTIVATION = 'ReLU' activation = getattr(builder, ACTIVATION) DSHAPE = (16,) * 4 + (1,) dmlp = builder.Sequential() for shape in DSHAPE[:-1]: dmlp.append(builder.Affine(shape)) dmlp.append(activation()) dmlp.append(builder.Affine(DSHAPE[-1])) dmodel = builder.Model(dmlp, 'l2', (D,)) initialize(dmodel) dupdater = Updater(dmodel, 'sgd', {'learning_rate' : -0.01}) GSHAPE = (16,) * 4 + (D,) gmlp = builder.Sequential() for shape in GSHAPE[:-1]: gmlp.append(builder.Affine(shape)) gmlp.append(activation()) gmlp.append(builder.Affine(GSHAPE[-1])) gmodel = builder.Model(gmlp, 'l2', (D,)) initialize(gmodel) gupdater = Updater(gmodel, 'sgd', {'learning_rate' : 0.01}) ITERATIONS = 1000 INTERVAL = 10
sys.path.append('../') from utilities.data_utility import load_mnist data = load_mnist(path='../utilities') hidden_layers = 4 shapes = (1024,) * hidden_layers + (10,) activation = builder.ReLU storage = {} mlp = builder.Sequential() for i, shape in enumerate(shapes[:-1]): mlp.append(builder.Affine(shape)) mlp.append(builder.Export('affine%d' % i, storage)) mlp.append(activation()) mlp.append(builder.Affine(shapes[-1])) mlp.append(builder.Export('affine%d' % (len(shapes) - 1), storage)) model = builder.Model(mlp, 'softmax', (28 * 28,)) batch_size = 50 batches = len(data[0]) // batch_size batch_index = 0 iterations = 10000 interval = 10 validation_interval = 1000 validation_X, validation_Y = data[2 : 4] validation_X = validation_X[:1024] validation_Y = validation_Y[:1024] settings = {'learning_rate' : 0.01} initialize(model) updater = Updater(model, 'sgd', settings)
DReLU(), builder.Convolution((1, 1), 192), DReLU(), builder.Convolution((1, 1), 10), DReLU(), builder.Pooling('avg', (8, 8)), builder.Reshape((10,)) ) data = load_cifar10(path='../utilities/cifar/', reshape=True, center=True, rescale=True) ini_mode = sys.argv[2] # ini_mode = 'normal' if ini_mode == 'layer-by-layer': model = builder.Model(network_in_network, 'softmax', (3, 32, 32,), data[2]) solver = Solver( model, NDArrayIter(data[0], data[1]), NDArrayIter(data[0], data[1]), ) solver.init() else: model = builder.Model(network_in_network, 'softmax', (3, 32, 32,)) for arg, setting in model.param_configs.items(): print arg shape = setting['shape'] if 'weight' in arg: if len(shape) == 2: n = shape[0] elif len(shape) == 4: