def main(args):
# Define a convolutional neural network the same as above
net = builder.Sequential(
builder.Convolution((7, 7), 32),
builder.ReLU(),
builder.Pooling('max', (2, 2), (2, 2)),
builder.Reshape((flattened_input_size,))
builder.Affine(hidden_size),
builder.Affine(num_classes),
)
# Cast the definition to a model compatible with minpy solver
model = builder.Model(net, 'softmax', (3 * 32 * 32,))
data = get_CIFAR10_data(args.data_dir)
train_dataiter = NDArrayIter(data['X_train'],
data['y_train'],
batch_size=batch_size,
shuffle=True)
test_dataiter = NDArrayIter(data['X_test'],
data['y_test'],
batch_size=batch_size,
shuffle=False)
solver = Solver(model,
train_dataiter,
test_dataiter,
num_epochs=10,
init_rule='gaussian',
init_config={
'stdvar': 0.001
},
update_rule='sgd_momentum',
optim_config={
'learning_rate': 1e-3,
'momentum': 0.9
},
verbose=True,
print_every=20)
solver.init()
solver.train()
def main(args):
# Define a 2-layer perceptron
MLP = builder.Sequential(
builder.Affine(512),
builder.ReLU(),
builder.Affine(10)
)
# Cast the definition to a model compatible with minpy solver
model = builder.Model(MLP, 'softmax', (3 * 32 * 32,))
data = get_CIFAR10_data(args.data_dir)
data['X_train'] = data['X_train'].reshape([data['X_train'].shape[0], 3 * 32 * 32])
data['X_val'] = data['X_val'].reshape([data['X_val'].shape[0], 3 * 32 * 32])
data['X_test'] = data['X_test'].reshape([data['X_test'].shape[0], 3 * 32 * 32])
train_dataiter = NDArrayIter(data['X_train'],
data['y_train'],
batch_size=100,
shuffle=True)
test_dataiter = NDArrayIter(data['X_test'],
data['y_test'],
batch_size=100,
shuffle=False)
solver = Solver(model,
train_dataiter,
test_dataiter,
num_epochs=10,
init_rule='gaussian',
init_config={
'stdvar': 0.001
},
update_rule='sgd_momentum',
optim_config={
'learning_rate': 1e-5,
'momentum': 0.9
},
verbose=True,
print_every=20)
solver.init()
solver.train()
data = load_cifar10(path='../utilities/cifar/', center=True, rescale=True)
X = data[0][:16]
hidden_layers = 4
shapes = (1024, ) * hidden_layers + (10, )
activation = builder.ReLU
storage = {}
mlp = builder.Sequential()
for i, shape in enumerate(shapes[:-1]):
mlp.append(builder.Affine(shape))
mlp.append(builder.Export('affine%d' % i, storage))
mlp.append(activation())
mlp.append(builder.Affine(shapes[-1]))
model = builder.Model(mlp, 'softmax', (3072, ))
'''
for key, value in model.param_configs.items():
if 'weight' in key:
value['init_rule'] = 'gaussian'
value['init_config'] = {'stdvar' : 1}
'''
initialize(model)
for key, value in model.params.items():
if 'weight' in key:
print np.std(value)
result = model.forward(X, 'train')
print 'result', np.std(result)
from GPU_utility import GPU_availability
from minpy.context import set_context, gpu
set_context(gpu(GPU_availability()[0]))
ACTIVATION = 'ReLU'
SHAPE = (1024, ) * 3 + (10, )
BATCH_SIZE = 64
X_SHAPE = (3072, )
activation = getattr(builder, ACTIVATION)
mlp = builder.Sequential()
for shape in SHAPE[:-1]:
mlp.append(builder.Affine(shape))
mlp.append(activation())
mlp.append(builder.Affine(SHAPE[-1]))
model = builder.Model(mlp, 'softmax', X_SHAPE)
initialize(model)
updater = Updater(model, 'sgd', {'learning_rate': 0.01})
training_X, training_Y, validation_X, validation_Y, test_X, test_Y, = \
load_cifar10(path='../../cifar10/utilities/cifar/', center=True, rescale=True)
X_batches = Batches(training_X, BATCH_SIZE)
Y_batches = Batches(training_Y, BATCH_SIZE)
ITERATIONS = 20000
LOGGING_INTERVAL = 10
VALIDATION_INTERVAL = 50
loss_table = []
validation_accuracy_table = []
sample = lambda N, D: np.random.uniform(-1, 1, (N, D))
for i in range(ITERATIONS):
load_cifar10(path='../../utilities/cifar/', center=True, rescale=True)
HIDDEN_LAYERS = 4
activation = sys.argv[1]
# activation = 'ReLU'
storage = {}
mlp = MLP(*((1024, ) * HIDDEN_LAYERS + (10, )),
activation=activation,
affine_monitor=False,
activation_monitor=False,
storage=storage)
ini_mode = sys.argv[2]
# ini_mode = 'layer-by-layer'
if ini_mode == 'layer-by-layer':
model = builder.Model(mlp, 'softmax', (3072, ), training_X)
else:
model = builder.Model(mlp, 'softmax', (3072, ))
solver = Solver(model,
NDArrayIter(training_X, training_Y),
NDArrayIter(test_X, test_Y),
init_rule='xavier')
solver.init()
parameter_keys = list(model.params.keys())
parameter_values = list(model.params.values())
def loss_function(*args):
N, D = 50000, 16
data, p = generate_data(N, D)
BATCH_SIZE = 100
X_batches = Batches(data, BATCH_SIZE)
p_batches = Batches(p.reshape((N, 1)), BATCH_SIZE)
ACTIVATION = 'ReLU'
activation = getattr(builder, ACTIVATION)
DSHAPE = (16,) * 4 + (1,)
dmlp = builder.Sequential()
for shape in DSHAPE[:-1]:
dmlp.append(builder.Affine(shape))
dmlp.append(activation())
dmlp.append(builder.Affine(DSHAPE[-1]))
dmodel = builder.Model(dmlp, 'l2', (D,))
initialize(dmodel)
dupdater = Updater(dmodel, 'sgd', {'learning_rate' : -0.01})
GSHAPE = (16,) * 4 + (D,)
gmlp = builder.Sequential()
for shape in GSHAPE[:-1]:
gmlp.append(builder.Affine(shape))
gmlp.append(activation())
gmlp.append(builder.Affine(GSHAPE[-1]))
gmodel = builder.Model(gmlp, 'l2', (D,))
initialize(gmodel)
gupdater = Updater(gmodel, 'sgd', {'learning_rate' : 0.01})
ITERATIONS = 1000
INTERVAL = 10
sys.path.append('../')
from utilities.data_utility import load_mnist
data = load_mnist(path='../utilities')
hidden_layers = 4
shapes = (1024,) * hidden_layers + (10,)
activation = builder.ReLU
storage = {}
mlp = builder.Sequential()
for i, shape in enumerate(shapes[:-1]):
mlp.append(builder.Affine(shape))
mlp.append(builder.Export('affine%d' % i, storage))
mlp.append(activation())
mlp.append(builder.Affine(shapes[-1]))
mlp.append(builder.Export('affine%d' % (len(shapes) - 1), storage))
model = builder.Model(mlp, 'softmax', (28 * 28,))
batch_size = 50
batches = len(data[0]) // batch_size
batch_index = 0
iterations = 10000
interval = 10
validation_interval = 1000
validation_X, validation_Y = data[2 : 4]
validation_X = validation_X[:1024]
validation_Y = validation_Y[:1024]
settings = {'learning_rate' : 0.01}
initialize(model)
updater = Updater(model, 'sgd', settings)
DReLU(),
builder.Convolution((1, 1), 192),
DReLU(),
builder.Convolution((1, 1), 10),
DReLU(),
builder.Pooling('avg', (8, 8)),
builder.Reshape((10,))
)
data = load_cifar10(path='../utilities/cifar/', reshape=True, center=True, rescale=True)
ini_mode = sys.argv[2]
# ini_mode = 'normal'
if ini_mode == 'layer-by-layer':
model = builder.Model(network_in_network, 'softmax', (3, 32, 32,), data[2])
solver = Solver(
model,
NDArrayIter(data[0], data[1]),
NDArrayIter(data[0], data[1]),
)
solver.init()
else:
model = builder.Model(network_in_network, 'softmax', (3, 32, 32,))
for arg, setting in model.param_configs.items():
print arg
shape = setting['shape']
if 'weight' in arg:
if len(shape) == 2:
n = shape[0]
elif len(shape) == 4:
