def test_iterator(): parser = NeonArgparser(__doc__) args = parser.parse_args() (X_train, y_train), (X_test, y_test), nclass = load_cifar10_imgs(path=args.data_dir) train = DataIterator(X_train, y_train, nclass=nclass, lshape=(3, 32, 32)) test = DataIterator(X_test, y_test, nclass=nclass, lshape=(3, 32, 32)) return run(args, train, test)
def test_dataset(backend): (X_train, y_train), (X_test, y_test), nclass = load_mnist() train_set = DataIterator(X_train, y_train, nclass=nclass) train_set.be = NervanaObject.be for i in range(2): for X_batch, y_batch in train_set: print X_batch.shape, y_batch.shape train_set.index = 0
def train(self): """Provide neon data iterator for training purposes.""" return DataIterator( X=self.X_train, y=self.y_train, nclass=self.num_labels, make_onehot=True, lshape=(self.num_features, 1, 1))
def test_model_get_outputs(backend): (X_train, y_train), (X_test, y_test), nclass = load_mnist() train_set = DataIterator(X_train[:backend.bsz * 3]) init_norm = Gaussian(loc=0.0, scale=0.1) layers = [Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()), Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))] mlp = Model(layers=layers) out_list = [] for x, t in train_set: x = mlp.fprop(x) out_list.append(x.get().T.copy()) ref_output = np.vstack(out_list) train_set.reset() output = mlp.get_outputs(train_set) assert np.allclose(output, ref_output)
def test_model_get_outputs(backend_default): (X_train, y_train), (X_test, y_test), nclass = load_mnist() train_set = DataIterator(X_train[:backend_default.bsz * 3]) init_norm = Gaussian(loc=0.0, scale=0.1) layers = [Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()), Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))] mlp = Model(layers=layers) out_list = [] mlp.initialize(train_set) for x, t in train_set: x = mlp.fprop(x) out_list.append(x.get().T.copy()) ref_output = np.vstack(out_list) train_set.reset() output = mlp.get_outputs(train_set) assert np.allclose(output, ref_output)
def test(self): """Provide neon data iterator for testing purposes.""" if self.num_test_examples > 0: return DataIterator( X=self.X_test, y=self.y_test, nclass=self.num_labels, make_onehot=True, lshape=(self.num_features, 1, 1)) else: return None
def visualize(model, data, max_fm, filename): data_shape = data.shape data = data.reshape((data.shape[0], -1)) dataset = DataIterator(data, lshape=data_shape[1:]) deconv_file = h5py.File("no_file", driver='core', backing_store=False) deconv = DeconvCallback(deconv_file, model, dataset, dataset, max_fm=max_fm, dataset_pct=100) deconv.on_train_end() deconv_data = h5_deconv_data(deconv_file) deconv_summary_page(filename, deconv_data, max_fm)
from neon.util.argparser import NeonArgparser # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() logger = logging.getLogger() logger.setLevel(args.log_thresh) # load up the mnist data set # split into train and tests sets (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) # setup a training set iterator train_set = DataIterator(X_train, y_train, nclass=nclass, lshape=(1, 28, 28)) # setup a validation data set iterator valid_set = DataIterator(X_test, y_test, nclass=nclass, lshape=(1, 28, 28)) # setup weight initialization function init_norm = Gaussian(loc=0.0, scale=0.01) # setup model layers layers = [ Affine(nout=100, init=init_norm, activation=Rectlin()), Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True)) ] # setup cost function as CrossEntropy cost = GeneralizedCost(costfunc=CrossEntropyBinary())
def test_model_serialize(backend): (X_train, y_train), (X_test, y_test), nclass = load_mnist() train_set = DataIterator([X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28)) init_norm = Gaussian(loc=0.0, scale=0.01) # initialize model path1 = [ Conv((5, 5, 16), init=init_norm, bias=Constant(0), activation=Rectlin()), Pooling(2), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()) ] path2 = [ Dropout(keep=0.5), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()) ] layers = [ MergeConcat([path1, path2]), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()), BatchNorm(), Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True)) ] tmp_save = 'test_model_serialize_tmp_save.pickle' mlp = Model(layers=layers) mlp.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9) mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary()) n_test = 3 num_epochs = 3 # Train model for num_epochs and n_test batches for epoch in range(num_epochs): for i, (x, t) in enumerate(train_set): x = mlp.fprop(x) delta = mlp.cost.get_errors(x, t) mlp.bprop(delta) mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch) if i > n_test: break # Get expected outputs of n_test batches and states of all layers outputs_exp = [] pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize] for i, (x, t) in enumerate(train_set): outputs_exp.append(mlp.fprop(x, inference=True)) if i > n_test: break # Serialize model save_obj(mlp.serialize(keep_states=True), tmp_save) # Load model mlp = Model(layers=layers) mlp.load_weights(tmp_save) outputs = [] pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize] for i, (x, t) in enumerate(train_set): outputs.append(mlp.fprop(x, inference=True)) if i > n_test: break # Check outputs, states, and params are the same for output, output_exp in zip(outputs, outputs_exp): assert np.allclose(output.get(), output_exp.get()) for pd, pd_exp in zip(pdicts, pdicts_exp): for s, s_e in zip(pd['states'], pd_exp['states']): if isinstance(s, list): # this is the batch norm case for _s, _s_e in zip(s, s_e): assert np.allclose(_s, _s_e) else: assert np.allclose(s, s_e) for p, p_e in zip(pd['params'], pd_exp['params']): if isinstance(p, list): # this is the batch norm case for _p, _p_e in zip(p, p_e): assert np.allclose(_p, _p_e) else: assert np.allclose(p, p_e) os.remove(tmp_save)
def main(): # setup the model and run for num_epochs saving the last state only # this is at the top so that the be is generated mlp = gen_model(args.backend) # setup data iterators (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) if args.backend == 'nervanacpu' or args.backend == 'cpu': # limit data since cpu backend runs slower train = DataIterator(X_train[:1000], y_train[:1000], nclass=nclass, lshape=(1, 28, 28)) valid = DataIterator(X_test[:1000], y_test[:1000], nclass=nclass, lshape=(1, 28, 28)) else: train = DataIterator(X_train, y_train, nclass=nclass, lshape=(1, 28, 28)) valid = DataIterator(X_test, y_test, nclass=nclass, lshape=(1, 28, 28)) # serialization related cost = GeneralizedCost(costfunc=CrossEntropyBinary()) opt_gdm = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9) checkpoint_model_path = os.path.join('./', 'test_oneshot.pkl') checkpoint_schedule = 1 # save at every step callbacks = Callbacks(mlp, train) callbacks.add_serialize_callback(checkpoint_schedule, checkpoint_model_path, history=2) # run the fit all the way through saving a checkpoint e mlp.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks) # setup model with same random seed run epoch by epoch # serializing and deserializing at each step mlp = gen_model(args.backend) cost = GeneralizedCost(costfunc=CrossEntropyBinary()) opt_gdm = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9) # reset data iterators train.reset() valid.reset() checkpoint_model_path = os.path.join('./', 'test_manyshot.pkl') checkpoint_schedule = 1 # save at evey step callbacks = Callbacks(mlp, train) callbacks.add_serialize_callback(checkpoint_schedule, checkpoint_model_path, history=num_epochs) for epoch in range(num_epochs): # _0 points to state at end of epoch 0 mlp.fit(train, optimizer=opt_gdm, num_epochs=epoch+1, cost=cost, callbacks=callbacks) # load saved file prts = os.path.splitext(checkpoint_model_path) fn = prts[0] + '_%d' % epoch + prts[1] mlp.load_weights(fn) # load the saved weights # compare test_oneshot_<num_epochs>.pkl to test_manyshot_<num_epochs>.pkl try: compare_model_pickles('test_oneshot_%d.pkl' % (num_epochs-1), 'test_manyshot_%d.pkl' % (num_epochs-1)) except: print 'test failed....' sys.exit(1)
args = parser.parse_args() # hyperparameters batch_size = 128 num_epochs = args.epochs # setup backend be = gen_backend(backend=args.backend, batch_size=batch_size, rng_seed=args.rng_seed, device_id=args.device_id, default_dtype=args.datatype) (X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir) train = DataIterator(X_train, y_train, nclass=nclass) test = DataIterator(X_test, y_test, nclass=nclass) init_uni = Uniform(low=-0.1, high=0.1) opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9) # set up the model layers layers = [] layers.append(Affine(nout=200, init=init_uni, activation=Rectlin())) layers.append( Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True))) cost = GeneralizedCost(costfunc=CrossEntropyBinary()) mlp = Model(layers=layers)
batch_size = 128 if args.datatype in [np.float16]: cost_scale = 10. num_epochs = args.epochs # setup backend be = gen_backend(backend=args.backend, batch_size=batch_size, rng_seed=args.rng_seed, device_id=args.device_id, default_dtype=args.datatype, stochastic_round=False) (X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir) train = DataIterator(X_train, y_train, nclass=nclass, lshape=(3, 32, 32)) test = DataIterator(X_test, y_test, nclass=nclass, lshape=(3, 32, 32)) init_uni = Uniform(low=-0.1, high=0.1) if args.datatype in [np.float32, np.float64]: opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9, stochastic_round=args.rounding) elif args.datatype in [np.float16]: opt_gdm = GradientDescentMomentum(learning_rate=0.01 / cost_scale, momentum_coef=0.9, stochastic_round=args.rounding) layers = [] layers.append(Conv((5, 5, 16), init=init_uni, activation=Rectlin())) layers.append(Pooling((2, 2)))
from neon.layers import Conv, Pooling, GeneralizedCost, Deconv from neon.models import Model from neon.optimizers import GradientDescentMomentum from neon.transforms import Rectlin, SumSquared from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() # Load dataset (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) # Set input and target to X_train train = DataIterator(X_train, lshape=(1, 28, 28)) # Initialize the weights and the learning rule init_uni = Uniform(low=-0.1, high=0.1) opt_gdm = GradientDescentMomentum(learning_rate=0.001, momentum_coef=0.9) # Define the layers layers = [ Conv((4, 4, 8), init=init_uni, activation=Rectlin()), Pooling(2), Conv((4, 4, 32), init=init_uni, activation=Rectlin()), Pooling(2), Deconv(fshape=(3, 3, 8), init=init_uni, strides=2, padding=1), Deconv(fshape=(3, 3, 8), init=init_uni, strides=2, padding=1), Deconv(fshape=(4, 4, 1), init=init_uni, strides=2, padding=0) ]
from neon.layers import GeneralizedCost, Affine, Sequential, MergeMultistream from neon.models import Model from neon.optimizers import GradientDescentMomentum from neon.transforms import Rectlin, Logistic, CrossEntropyBinary from neon.callbacks.callbacks import Callbacks from neon.util.argparser import NeonArgparser # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() # hyperparameters num_epochs = args.epochs (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) train_set = DataIterator([X_train, X_train], y_train, nclass=nclass) valid_set = DataIterator([X_test, X_test], y_test, nclass=nclass) # weight initialization init_norm = Gaussian(loc=0.0, scale=0.01) # initialize model path1 = Sequential(layers=[Affine(nout=100, init=init_norm, activation=Rectlin()), Affine(nout=100, init=init_norm, activation=Rectlin())]) path2 = Sequential(layers=[Affine(nout=100, init=init_norm, activation=Rectlin()), Affine(nout=100, init=init_norm, activation=Rectlin())]) layers = [MergeMultistream(layers=[path1, path2], merge="stack"), Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))]
], # Sign up for an enterprise account to get more observations observation_budget=150, ) DATA_DIR = "/home/ubuntu/data" (X_train, y_train), (X_test, y_test), nclass = load_cifar10( path=DATA_DIR, normalize=False, contrast_normalize=True, whiten=False, ) # get error on this command train_set = DataIterator(X_train, y_train, nclass=16, lshape=(3, 32, 32)) valid_set = DataIterator(X_test, y_test, nclass=16, lshape=(3, 32, 32)) # run optimization loop for ir in xrange(experiment.observation_budget): suggestion = conn.experiments(experiment.id).suggestions().create() assignments = suggestion.assignments print assignments num_epochs = int(assignments.get("epochs")) init_uni = Gaussian(scale=assignments.get("gaussian_scale")) step_config = [ int( assignments.get("momentum_step_schedule_start") + i * assignments.get("momentum_step_schedule_step_width")) for i in range(int(assignments.get("momentum_step_schedule_steps")))
from neon.transforms import Rectlin, Logistic, Misclassification, Softmax from neon.transforms import CrossEntropyBinary, CrossEntropyMulti from neon.util.argparser import NeonArgparser # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() # load up the mnist data set # split into train and tests sets (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) # setup a training set iterator train_set = DataIterator(X_train, y_train, nclass=nclass) # setup a validation data set iterator valid_set = DataIterator(X_test, y_test, nclass=nclass) # setup weight initialization function init_norm = Gaussian(loc=0.0, scale=0.01) normrelu = dict(init=init_norm, activation=Rectlin()) normsigm = dict(init=init_norm, activation=Logistic(shortcut=True)) normsoft = dict(init=init_norm, activation=Softmax()) # setup model layers b1 = BranchNode(name="b1") b2 = BranchNode(name="b2")
# hyperparameters if args.datatype in [np.float16]: cost_scale = 10. num_epochs = args.epochs (X_train, y_train), (X_test, y_test), nclass = load_cifar10(path=args.data_dir) Nmax = X_train.shape[0] / args.batch_size Nmax *= args.batch_size X_train = X_train[0:Nmax, :] y_train = y_train[0:Nmax, :] train = DataIterator(X_train, y_train, nclass=nclass, lshape=(3, 32, 32), name='train') test = DataIterator(X_test, y_test, nclass=nclass, lshape=(3, 32, 32), name='test') init_uni = Uniform(low=-0.1, high=0.1) if args.datatype in [np.float32, np.float64]: opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9, stochastic_round=args.rounding) elif args.datatype in [np.float16]: opt_gdm = GradientDescentMomentum(learning_rate=0.01 / cost_scale,