def test_iterator():
parser = NeonArgparser(__doc__)
args = parser.parse_args()
(X_train, y_train), (X_test, y_test), nclass = load_cifar10_imgs(path=args.data_dir)
train = DataIterator(X_train, y_train, nclass=nclass, lshape=(3, 32, 32))
test = DataIterator(X_test, y_test, nclass=nclass, lshape=(3, 32, 32))
return run(args, train, test)
def test_dataset(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator(X_train, y_train, nclass=nclass)
train_set.be = NervanaObject.be
for i in range(2):
for X_batch, y_batch in train_set:
print X_batch.shape, y_batch.shape
train_set.index = 0
def test_dataset(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator(X_train, y_train, nclass=nclass)
train_set.be = NervanaObject.be
for i in range(2):
for X_batch, y_batch in train_set:
print X_batch.shape, y_batch.shape
train_set.index = 0
def train(self):
"""Provide neon data iterator for training purposes."""
return DataIterator(
X=self.X_train,
y=self.y_train,
nclass=self.num_labels,
make_onehot=True,
lshape=(self.num_features, 1, 1))
def test_model_get_outputs(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator(X_train[:backend.bsz * 3])
init_norm = Gaussian(loc=0.0, scale=0.1)
layers = [Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
mlp = Model(layers=layers)
out_list = []
for x, t in train_set:
x = mlp.fprop(x)
out_list.append(x.get().T.copy())
ref_output = np.vstack(out_list)
train_set.reset()
output = mlp.get_outputs(train_set)
assert np.allclose(output, ref_output)
def test_model_get_outputs(backend_default):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator(X_train[:backend_default.bsz * 3])
init_norm = Gaussian(loc=0.0, scale=0.1)
layers = [Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
mlp = Model(layers=layers)
out_list = []
mlp.initialize(train_set)
for x, t in train_set:
x = mlp.fprop(x)
out_list.append(x.get().T.copy())
ref_output = np.vstack(out_list)
train_set.reset()
output = mlp.get_outputs(train_set)
assert np.allclose(output, ref_output)
def test(self):
"""Provide neon data iterator for testing purposes."""
if self.num_test_examples > 0:
return DataIterator(
X=self.X_test,
y=self.y_test,
nclass=self.num_labels,
make_onehot=True,
lshape=(self.num_features, 1, 1))
else:
return None
def visualize(model, data, max_fm, filename):
data_shape = data.shape
data = data.reshape((data.shape[0], -1))
dataset = DataIterator(data, lshape=data_shape[1:])
deconv_file = h5py.File("no_file", driver='core', backing_store=False)
deconv = DeconvCallback(deconv_file, model, dataset, dataset,
max_fm=max_fm, dataset_pct=100)
deconv.on_train_end()
deconv_data = h5_deconv_data(deconv_file)
deconv_summary_page(filename, deconv_data, max_fm)
from neon.util.argparser import NeonArgparser
# parse the command line arguments
parser = NeonArgparser(__doc__)
args = parser.parse_args()
logger = logging.getLogger()
logger.setLevel(args.log_thresh)
# load up the mnist data set
# split into train and tests sets
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir)
# setup a training set iterator
train_set = DataIterator(X_train, y_train, nclass=nclass, lshape=(1, 28, 28))
# setup a validation data set iterator
valid_set = DataIterator(X_test, y_test, nclass=nclass, lshape=(1, 28, 28))
# setup weight initialization function
init_norm = Gaussian(loc=0.0, scale=0.01)
# setup model layers
layers = [
Affine(nout=100, init=init_norm, activation=Rectlin()),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))
]
# setup cost function as CrossEntropy
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
def test_model_serialize(backend):
(X_train, y_train), (X_test, y_test), nclass = load_mnist()
train_set = DataIterator([X_train, X_train],
y_train,
nclass=nclass,
lshape=(1, 28, 28))
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = [
Conv((5, 5, 16),
init=init_norm,
bias=Constant(0),
activation=Rectlin()),
Pooling(2),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
]
path2 = [
Dropout(keep=0.5),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin())
]
layers = [
MergeConcat([path1, path2]),
Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()),
BatchNorm(),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))
]
tmp_save = 'test_model_serialize_tmp_save.pickle'
mlp = Model(layers=layers)
mlp.optimizer = GradientDescentMomentum(learning_rate=0.1,
momentum_coef=0.9)
mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary())
n_test = 3
num_epochs = 3
# Train model for num_epochs and n_test batches
for epoch in range(num_epochs):
for i, (x, t) in enumerate(train_set):
x = mlp.fprop(x)
delta = mlp.cost.get_errors(x, t)
mlp.bprop(delta)
mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch)
if i > n_test:
break
# Get expected outputs of n_test batches and states of all layers
outputs_exp = []
pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs_exp.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Serialize model
save_obj(mlp.serialize(keep_states=True), tmp_save)
# Load model
mlp = Model(layers=layers)
mlp.load_weights(tmp_save)
outputs = []
pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize]
for i, (x, t) in enumerate(train_set):
outputs.append(mlp.fprop(x, inference=True))
if i > n_test:
break
# Check outputs, states, and params are the same
for output, output_exp in zip(outputs, outputs_exp):
assert np.allclose(output.get(), output_exp.get())
for pd, pd_exp in zip(pdicts, pdicts_exp):
for s, s_e in zip(pd['states'], pd_exp['states']):
if isinstance(s, list): # this is the batch norm case
for _s, _s_e in zip(s, s_e):
assert np.allclose(_s, _s_e)
else:
assert np.allclose(s, s_e)
for p, p_e in zip(pd['params'], pd_exp['params']):
if isinstance(p, list): # this is the batch norm case
for _p, _p_e in zip(p, p_e):
assert np.allclose(_p, _p_e)
else:
assert np.allclose(p, p_e)
os.remove(tmp_save)
def main():
# setup the model and run for num_epochs saving the last state only
# this is at the top so that the be is generated
mlp = gen_model(args.backend)
# setup data iterators
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir)
if args.backend == 'nervanacpu' or args.backend == 'cpu':
# limit data since cpu backend runs slower
train = DataIterator(X_train[:1000], y_train[:1000], nclass=nclass, lshape=(1, 28, 28))
valid = DataIterator(X_test[:1000], y_test[:1000], nclass=nclass, lshape=(1, 28, 28))
else:
train = DataIterator(X_train, y_train, nclass=nclass, lshape=(1, 28, 28))
valid = DataIterator(X_test, y_test, nclass=nclass, lshape=(1, 28, 28))
# serialization related
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
opt_gdm = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
checkpoint_model_path = os.path.join('./', 'test_oneshot.pkl')
checkpoint_schedule = 1 # save at every step
callbacks = Callbacks(mlp, train)
callbacks.add_serialize_callback(checkpoint_schedule, checkpoint_model_path, history=2)
# run the fit all the way through saving a checkpoint e
mlp.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks)
# setup model with same random seed run epoch by epoch
# serializing and deserializing at each step
mlp = gen_model(args.backend)
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
opt_gdm = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9)
# reset data iterators
train.reset()
valid.reset()
checkpoint_model_path = os.path.join('./', 'test_manyshot.pkl')
checkpoint_schedule = 1 # save at evey step
callbacks = Callbacks(mlp, train)
callbacks.add_serialize_callback(checkpoint_schedule,
checkpoint_model_path,
history=num_epochs)
for epoch in range(num_epochs):
# _0 points to state at end of epoch 0
mlp.fit(train, optimizer=opt_gdm, num_epochs=epoch+1, cost=cost, callbacks=callbacks)
# load saved file
prts = os.path.splitext(checkpoint_model_path)
fn = prts[0] + '_%d' % epoch + prts[1]
mlp.load_weights(fn) # load the saved weights
# compare test_oneshot_<num_epochs>.pkl to test_manyshot_<num_epochs>.pkl
try:
compare_model_pickles('test_oneshot_%d.pkl' % (num_epochs-1),
'test_manyshot_%d.pkl' % (num_epochs-1))
except:
print 'test failed....'
sys.exit(1)
args = parser.parse_args()
# hyperparameters
batch_size = 128
num_epochs = args.epochs
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype)
(X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir)
train = DataIterator(X_train, y_train, nclass=nclass)
test = DataIterator(X_test, y_test, nclass=nclass)
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9)
# set up the model layers
layers = []
layers.append(Affine(nout=200, init=init_uni, activation=Rectlin()))
layers.append(
Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True)))
cost = GeneralizedCost(costfunc=CrossEntropyBinary())
mlp = Model(layers=layers)
batch_size = 128
if args.datatype in [np.float16]:
cost_scale = 10.
num_epochs = args.epochs
# setup backend
be = gen_backend(backend=args.backend,
batch_size=batch_size,
rng_seed=args.rng_seed,
device_id=args.device_id,
default_dtype=args.datatype,
stochastic_round=False)
(X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir)
train = DataIterator(X_train, y_train, nclass=nclass, lshape=(3, 32, 32))
test = DataIterator(X_test, y_test, nclass=nclass, lshape=(3, 32, 32))
init_uni = Uniform(low=-0.1, high=0.1)
if args.datatype in [np.float32, np.float64]:
opt_gdm = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9,
stochastic_round=args.rounding)
elif args.datatype in [np.float16]:
opt_gdm = GradientDescentMomentum(learning_rate=0.01 / cost_scale,
momentum_coef=0.9,
stochastic_round=args.rounding)
layers = []
layers.append(Conv((5, 5, 16), init=init_uni, activation=Rectlin()))
layers.append(Pooling((2, 2)))
from neon.layers import Conv, Pooling, GeneralizedCost, Deconv
from neon.models import Model
from neon.optimizers import GradientDescentMomentum
from neon.transforms import Rectlin, SumSquared
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser
# parse the command line arguments
parser = NeonArgparser(__doc__)
args = parser.parse_args()
# Load dataset
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir)
# Set input and target to X_train
train = DataIterator(X_train, lshape=(1, 28, 28))
# Initialize the weights and the learning rule
init_uni = Uniform(low=-0.1, high=0.1)
opt_gdm = GradientDescentMomentum(learning_rate=0.001, momentum_coef=0.9)
# Define the layers
layers = [
Conv((4, 4, 8), init=init_uni, activation=Rectlin()),
Pooling(2),
Conv((4, 4, 32), init=init_uni, activation=Rectlin()),
Pooling(2),
Deconv(fshape=(3, 3, 8), init=init_uni, strides=2, padding=1),
Deconv(fshape=(3, 3, 8), init=init_uni, strides=2, padding=1),
Deconv(fshape=(4, 4, 1), init=init_uni, strides=2, padding=0)
]
from neon.layers import GeneralizedCost, Affine, Sequential, MergeMultistream
from neon.models import Model
from neon.optimizers import GradientDescentMomentum
from neon.transforms import Rectlin, Logistic, CrossEntropyBinary
from neon.callbacks.callbacks import Callbacks
from neon.util.argparser import NeonArgparser
# parse the command line arguments
parser = NeonArgparser(__doc__)
args = parser.parse_args()
# hyperparameters
num_epochs = args.epochs
(X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir)
train_set = DataIterator([X_train, X_train], y_train, nclass=nclass)
valid_set = DataIterator([X_test, X_test], y_test, nclass=nclass)
# weight initialization
init_norm = Gaussian(loc=0.0, scale=0.01)
# initialize model
path1 = Sequential(layers=[Affine(nout=100, init=init_norm, activation=Rectlin()),
Affine(nout=100, init=init_norm, activation=Rectlin())])
path2 = Sequential(layers=[Affine(nout=100, init=init_norm, activation=Rectlin()),
Affine(nout=100, init=init_norm, activation=Rectlin())])
layers = [MergeMultistream(layers=[path1, path2], merge="stack"),
Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
],
# Sign up for an enterprise account to get more observations
observation_budget=150,
)
DATA_DIR = "/home/ubuntu/data"
(X_train, y_train), (X_test, y_test), nclass = load_cifar10(
path=DATA_DIR,
normalize=False,
contrast_normalize=True,
whiten=False,
)
# get error on this command
train_set = DataIterator(X_train, y_train, nclass=16, lshape=(3, 32, 32))
valid_set = DataIterator(X_test, y_test, nclass=16, lshape=(3, 32, 32))
# run optimization loop
for ir in xrange(experiment.observation_budget):
suggestion = conn.experiments(experiment.id).suggestions().create()
assignments = suggestion.assignments
print assignments
num_epochs = int(assignments.get("epochs"))
init_uni = Gaussian(scale=assignments.get("gaussian_scale"))
step_config = [
int(
assignments.get("momentum_step_schedule_start") +
i * assignments.get("momentum_step_schedule_step_width"))
for i in range(int(assignments.get("momentum_step_schedule_steps")))
from neon.transforms import Rectlin, Logistic, Misclassification, Softmax from neon.transforms import CrossEntropyBinary, CrossEntropyMulti from neon.util.argparser import NeonArgparser # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() # load up the mnist data set # split into train and tests sets (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) # setup a training set iterator train_set = DataIterator(X_train, y_train, nclass=nclass) # setup a validation data set iterator valid_set = DataIterator(X_test, y_test, nclass=nclass) # setup weight initialization function init_norm = Gaussian(loc=0.0, scale=0.01) normrelu = dict(init=init_norm, activation=Rectlin()) normsigm = dict(init=init_norm, activation=Logistic(shortcut=True)) normsoft = dict(init=init_norm, activation=Softmax()) # setup model layers b1 = BranchNode(name="b1") b2 = BranchNode(name="b2")
# hyperparameters
if args.datatype in [np.float16]:
cost_scale = 10.
num_epochs = args.epochs
(X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir)
Nmax = X_train.shape[0] / args.batch_size
Nmax *= args.batch_size
X_train = X_train[0:Nmax, :]
y_train = y_train[0:Nmax, :]
train = DataIterator(X_train,
y_train,
nclass=nclass,
lshape=(3, 32, 32),
name='train')
test = DataIterator(X_test,
y_test,
nclass=nclass,
lshape=(3, 32, 32),
name='test')
init_uni = Uniform(low=-0.1, high=0.1)
if args.datatype in [np.float32, np.float64]:
opt_gdm = GradientDescentMomentum(learning_rate=0.01,
momentum_coef=0.9,
stochastic_round=args.rounding)
elif args.datatype in [np.float16]:
opt_gdm = GradientDescentMomentum(learning_rate=0.01 / cost_scale,
