def main():
# setup the model and run for num_epochs saving the last state only
# this is at the top so that the be is generated
mlp = gen_model(args.backend)
# setup data iterators
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir)
if args.backend == 'nervanacpu' or args.backend == 'cpu':
# limit data since cpu backend runs slower
train = DataIterator(X_train[:1000], y_train[:1000], nclass=nclass, lshape=(1, 28, 28))
valid = DataIterator(X_test[:1000], y_test[:1000], nclass=nclass, lshape=(1, 28, 28))
else:
train = DataIterator(X_train, y_train, nclass=nclass, lshape=(1, 28, 28))
valid = DataIterator(X_test, y_test, nclass=nclass, lshape=(1, 28, 28))
# serialization related
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
opt_gdm = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
checkpoint_model_path = os.path.join('./', 'test_oneshot.pkl')
checkpoint_schedule = 1 # save at every step
callbacks = Callbacks(mlp, train)
callbacks.add_serialize_callback(checkpoint_schedule, checkpoint_model_path, history=2)
# run the fit all the way through saving a checkpoint e
mlp.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
# setup model with same random seed run epoch by epoch
# serializing and deserializing at each step
mlp = gen_model(args.backend)
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
opt_gdm = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
# reset data iterators
train.reset()
valid.reset()
checkpoint_model_path = os.path.join('./', 'test_manyshot.pkl')
checkpoint_schedule = 1 # save at evey step
callbacks = Callbacks(mlp, train)
callbacks.add_serialize_callback(checkpoint_schedule,
checkpoint_model_path,
history=num_epochs)
for epoch in range(num_epochs):
# _0 points to state at end of epoch 0
mlp.fit(train, optimizer=opt_gdm, num_epochs=epoch+1, cost=cost, callbacks=callbacks)
# load saved file
prts = os.path.splitext(checkpoint_model_path)
fn = prts[0] + '_%d' % epoch + prts[1]
mlp.load_weights(fn) # load the saved weights
# compare test_oneshot_<num_epochs>.pkl to test_manyshot_<num_epochs>.pkl
try:
compare_model_pickles('test_oneshot_%d.pkl' % (num_epochs-1),
'test_manyshot_%d.pkl' % (num_epochs-1))
except:
print 'test failed....'
sys.exit(1)
def test_dataset(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator(X_train, y_train, nclass=nclass)
train_set.be = NervanaObject.be
for i in range(2):
for X_batch, y_batch in train_set:
print X_batch.shape, y_batch.shape
train_set.index = 0
def test_dataset(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator(X_train, y_train, nclass=nclass)
train_set.be = NervanaObject.be
for i in range(2):
for X_batch, y_batch in train_set:
print X_batch.shape, y_batch.shape
train_set.index = 0
def test_dataset(backend_default, data):
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=data)
train_set = ArrayIterator(X_train, y_train, nclass=nclass)
train_set.be = NervanaObject.be
for i in range(2):
for X_batch, y_batch in train_set:
neon_logger.display("Xshape: {}, yshape: {}".format(X_batch.shape, y_batch.shape))
train_set.index = 0
def load_data(self):
# load up the mnist data set
# split into train and tests sets
(X_train,
y_train), (X_test,
y_test), nclass = load_mnist(path=self.args.data_dir)
# setup a training set iterator
self.train_set = ArrayIterator(X_train,
y_train,
nclass=nclass,
lshape=(1, 28, 28))
# setup a validation data set iterator
self.valid_set = ArrayIterator(X_test,
y_test,
nclass=nclass,
lshape=(1, 28, 28))
def test_model_get_outputs(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator(X_train[:backend.bsz * 3])
init_norm = Gaussian(loc=0.0, scale=0.1)
layers = [Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
mlp = Model(layers=layers)
out_list = []
for x, t in train_set:
x = mlp.fprop(x)
out_list.append(x.get().T.copy())
ref_output = np.vstack(out_list)
train_set.reset()
output = mlp.get_outputs(train_set)
assert np.allclose(output, ref_output)
def test_model_get_outputs(backend_default):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator(X_train[:backend_default.bsz * 3])
init_norm = Gaussian(loc=0.0, scale=0.1)
layers = [Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
mlp = Model(layers=layers)
out_list = []
mlp.initialize(train_set)
for x, t in train_set:
x = mlp.fprop(x)
out_list.append(x.get().T.copy())
ref_output = np.vstack(out_list)
train_set.reset()
output = mlp.get_outputs(train_set)
assert np.allclose(output, ref_output)
def test_model_get_outputs(backend_default, data):
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=data)
train_set = ArrayIterator(X_train[:backend_default.bsz * 3])
init_norm = Gaussian(loc=0.0, scale=0.1)
layers = [Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
mlp = Model(layers=layers)
out_list = []
mlp.initialize(train_set)
for x, t in train_set:
x = mlp.fprop(x)
out_list.append(x.get().T.copy())
ref_output = np.vstack(out_list)
train_set.reset()
output = mlp.get_outputs(train_set)
assert np.allclose(output, ref_output)
# test model benchmark inference
mlp.benchmark(train_set, inference=True, niterations=5)
from neon.layers import GeneralizedCost, Affine, BranchNode, Multicost, Tree from neon.models import Model from neon.optimizers import GradientDescentMomentum from neon.transforms import Rectlin, Logistic, Misclassification, Softmax from neon.transforms import CrossEntropyBinary, CrossEntropyMulti from neon.util.argparser import NeonArgparser # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() # load up the mnist data set # split into train and tests sets (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) # setup a training set iterator train_set = DataIterator(X_train, y_train, nclass=nclass) # setup a validation data set iterator valid_set = DataIterator(X_test, y_test, nclass=nclass) # setup weight initialization function init_norm = Gaussian(loc=0.0, scale=0.01) normrelu = dict(init=init_norm, activation=Rectlin()) normsigm = dict(init=init_norm, activation=Logistic(shortcut=True)) normsoft = dict(init=init_norm, activation=Softmax()) # setup model layers b1 = BranchNode(name="b1")
from neon.layers import GeneralizedCost, Affine, BranchNode, Multicost, Tree from neon.models import Model from neon.optimizers import GradientDescentMomentum from neon.transforms import Rectlin, Logistic, Misclassification, Softmax from neon.transforms import CrossEntropyBinary, CrossEntropyMulti from neon.util.argparser import NeonArgparser # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() # load up the mnist data set # split into train and tests sets (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) # setup a training set iterator train_set = DataIterator(X_train, y_train, nclass=nclass) # setup a validation data set iterator valid_set = DataIterator(X_test, y_test, nclass=nclass) # setup weight initialization function init_norm = Gaussian(loc=0.0, scale=0.01) normrelu = dict(init=init_norm, activation=Rectlin()) normsigm = dict(init=init_norm, activation=Logistic(shortcut=True)) normsoft = dict(init=init_norm, activation=Softmax()) # setup model layers b1 = BranchNode(name="b1")
yield (inputs, targets)
class DataIterator(ArrayIterator):
"""
This class has been renamed to ArrayIterator and deprecated.
This is just a place holder until the class is removed. Please
use the ArrayIterator class.
"""
def __init__(self, *args, **kwargs):
logger.error('DataIterator class has been deprecated and renamed'
'"ArrayIterator" please use that name.')
super(DataIterator, self).__init__(*args, **kwargs)
if __name__ == '__main__':
from neon.data import load_mnist
(X_train, y_train), (X_test, y_test) = load_mnist()
from neon.backends.nervanagpu import NervanaGPU
ng = NervanaGPU(0, device_id=1)
NervanaObject.be = ng
ng.bsz = 128
train_set = ArrayIterator(
[X_test[:1000], X_test[:1000]], y_test[:1000], nclass=10)
for i in range(3):
for bidx, (X_batch, y_batch) in enumerate(train_set):
print bidx, train_set.start
pass
def test_model_serialize(backend_default, data):
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=data)
train_set = DataIterator(
[X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = Sequential([Conv((5, 5, 16), init=init_norm, bias=Constant(0), activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())])
path2 = Sequential([Affine(nout=100, init=init_norm, bias=Constant(0), activation=Rectlin()),
Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())])
layers = [MergeMultistream(layers=[path1, path2], merge="stack"),
Affine(nout=20, init=init_norm, batch_norm=True, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp.initialize(train_set, cost=mlp.cost)
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
save_obj(mlp.serialize(keep_states=True), tmp_save)
# Load model
mlp = Model(layers=layers)
mlp.load_weights(tmp_save)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert np.allclose(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert np.allclose(_s, _s_e)
else:
assert np.allclose(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
assert type(p) == type(p_e)
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert np.allclose(_p, _p_e)
elif isinstance(p, np.ndarray):
assert np.allclose(p, p_e)
else:
assert p == p_e
os.remove(tmp_save)
def test_model_serialize(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator([X_train, X_train],
y_train,
nclass=nclass,
lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = [
Conv((5, 5, 16),
init=init_norm,
bias=Constant(0),
activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
]
path2 = [
Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
]
layers = [
MergeConcat([path1, path2]),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
BatchNorm(),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))
]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1,
momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
save_obj(mlp.serialize(keep_states=True), tmp_save)
# Load model
mlp = Model(layers=layers)
mlp.load_weights(tmp_save)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert np.allclose(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert np.allclose(_s, _s_e)
else:
assert np.allclose(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert np.allclose(_p, _p_e)
else:
assert np.allclose(p, p_e)
os.remove(tmp_save)
axis=0)
if self.be.bsz > bsz:
self.ybuf[:, bsz:] = self.be.onehot(
self.ydev[:(self.be.bsz - bsz)], axis=0)
else:
self.ybuf[:, :bsz] = self.ydev[i1:i2].T
if self.be.bsz > bsz:
self.ybuf[:, bsz:] = self.ydev[:(self.be.bsz - bsz)].T
inputs = self.Xbuf[0] if len(self.Xbuf) == 1 else self.Xbuf
targets = self.ybuf if self.ybuf else inputs
yield (inputs, targets)
if __name__ == '__main__':
from neon.data import load_mnist
(X_train, y_train), (X_test, y_test) = load_mnist()
from neon.backends.nervanagpu import NervanaGPU
ng = NervanaGPU(0, device_id=1)
NervanaObject.be = ng
ng.bsz = 128
train_set = DataIterator([X_test[:1000], X_test[:1000]],
y_test[:1000],
nclass=10)
for i in range(3):
for bidx, (X_batch, y_batch) in enumerate(train_set):
print bidx, train_set.start
pass
inputs = self.Xbuf[0] if len(self.Xbuf) == 1 else self.Xbuf
targets = self.ybuf if self.ybuf else inputs
yield (inputs, targets)
class DataIterator(ArrayIterator):
"""
This class has been renamed to ArrayIterator and deprecated.
This is just a place holder until the class is removed. Please
use the ArrayIterator class.
"""
def __init__(self, *args, **kwargs):
logger.error('DataIterator class has been deprecated and renamed'
'"ArrayIterator" please use that name.')
super(DataIterator, self).__init__(*args, **kwargs)
if __name__ == '__main__':
from neon.data import load_mnist
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
from neon.backends import gen_backend
be = gen_backend('gpu', batch_size=128)
train_set = ArrayIterator(X_test[:1000], y_test[:1000], nclass=nclass)
for i in range(3):
for bidx, (X_batch, y_batch) in enumerate(train_set):
logger.display("{}".format((bidx, train_set.start)))
pass
inputs = self.Xbuf[0] if len(self.Xbuf) == 1 else self.Xbuf
targets = self.ybuf if self.ybuf else inputs
yield (inputs, targets)
class DataIterator(ArrayIterator):
"""
This class has been renamed to ArrayIterator and deprecated.
This is just a place holder until the class is removed. Please
use the ArrayIterator class.
"""
def __init__(self, *args, **kwargs):
logger.error('DataIterator class has been deprecated and renamed'
'"ArrayIterator" please use that name.')
super(DataIterator, self).__init__(*args, **kwargs)
if __name__ == '__main__':
from neon.data import load_mnist
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
from neon.backends import gen_backend
be = gen_backend('gpu', batch_size=128)
train_set = ArrayIterator(X_test[:1000], y_test[:1000], nclass=nclass)
for i in range(3):
for bidx, (X_batch, y_batch) in enumerate(train_set):
logger.display("{}".format((bidx, train_set.start)))
pass
