strides=dict(str_h=2, str_w=4)), Pooling(2, strides=2), Conv((5, 5, 64), init=init, batch_norm=True, activation=Rectlin(), strides=dict(str_h=1, str_w=2)), DeepBiRNN(128, init=GlorotUniform(), batch_norm=True, activation=Rectlin(), reset_cells=True, depth=3), RecurrentMean(), Affine(nout=common['nclasses'], init=init, activation=Softmax()) ] model = Model(layers=layers) opt = Adagrad(learning_rate=0.01) metric = Misclassification() callbacks = Callbacks(model, eval_set=val, metric=metric, **args.callback_args) cost = GeneralizedCost(costfunc=CrossEntropyMulti()) model.fit(train, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks) print('Misclassification error = %.1f%%' % (model.eval(val, metric=metric) * 100))
layers = [ Affine(nout=50, init=w, bias=b, activation=Rectlin()), Dropout(keep=0.5), Affine(nout=50, init=w, bias=b, activation=Rectlin()), Dropout(keep=0.4), Affine(nout=3, init=w, bias=b, activation=Softmax()), Dropout(keep=0.3) ] # Optimizer optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9, stochastic_round=args.rounding) # Cost cost = GeneralizedCost(costfunc=MeanSquared()) model = Model(layers=layers) callbacks = Callbacks(model, eval_set=val_iter, **args.callback_args) # Training model.fit(train_iter, optimizer=optimizer, num_epochs=1, cost=cost, callbacks=callbacks) # Evluate evaluate(model, val_iter, Metric=Misclassification())
reset_cells=False), Affine(train_set.nfeatures, init, bias=init, activation=Identity()) ] else: layers = [ LSTM(hidden, init, activation=Logistic(), gate_activation=Tanh(), reset_cells=True), RecurrentLast(), Affine(train_set.nfeatures, init, bias=init, activation=Identity()) ] model = Model(layers=layers) cost = GeneralizedCost(MeanSquared()) optimizer = RMSProp(stochastic_round=args.rounding) callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args) # fit model model.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks) # =======visualize how the model does on validation set============== # run the trained model on train and valid dataset and see how the outputs # match train_output = model.get_outputs(train_set).reshape(
def main(): # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() logger = logging.getLogger() logger.setLevel(args.log_thresh) #Set up batch iterator for training images print "Setting up data batch loaders..." train = ImgMaster(repo_dir='dataTmp', set_name='train', inner_size=120, subset_pct=100) val = ImgMaster(repo_dir='dataTmp', set_name='train', inner_size=120, subset_pct=100, do_transforms=False) test = ImgMaster(repo_dir='dataTestTmp', set_name='train', inner_size=120, subset_pct=100, do_transforms=False) train.init_batch_provider() val.init_batch_provider() test.init_batch_provider() print "Constructing network..." #Create AlexNet architecture model = constuct_network() #model.load_weights(args.model_file) # drop weights LR by 1/250**(1/3) at epochs (23, 45, 66), drop bias LR by 1/10 at epoch 45 weight_sched = Schedule([22, 44, 65, 90, 97], (1 / 250.)**(1 / 3.)) opt_gdm = GradientDescentMomentum(0.01, 0.9, wdecay=0.005, schedule=weight_sched) opt_biases = GradientDescentMomentum(0.04, 1.0, schedule=Schedule([130], .1)) opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases}) # configure callbacks valmetric = TopKMisclassification(k=5) callbacks = Callbacks(model, train, eval_set=val, metric=valmetric, **args.callback_args) cost = GeneralizedCost(costfunc=CrossEntropyMulti()) #flag = input("Press Enter if you want to begin training process.") print "Training network..." model.fit(train, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks) mets = model.eval(test, metric=valmetric) print 'Validation set metrics:' print 'LogLoss: %.2f, Accuracy: %.1f %%0 (Top-1), %.1f %% (Top-5)' % ( mets[0], (1.0 - mets[1]) * 100, (1.0 - mets[2]) * 100) test.exit_batch_provider() val.exit_batch_provider() train.exit_batch_provider()
def test_model_serialize(backend_default, data): (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=data) train_set = ArrayIterator([X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28)) init_norm = Gaussian(loc=0.0, scale=0.01) # initialize model path1 = Sequential([ Conv((5, 5, 16), init=init_norm, bias=Constant(0), activation=Rectlin()), Pooling(2), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()) ]) path2 = Sequential([ Affine(nout=100, init=init_norm, bias=Constant(0), activation=Rectlin()), Dropout(keep=0.5), Affine(nout=20, init=init_norm, bias=init_norm, activation=Rectlin()) ]) layers = [ MergeMultistream(layers=[path1, path2], merge="stack"), Affine(nout=20, init=init_norm, batch_norm=True, activation=Rectlin()), Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True)) ] tmp_save = 'test_model_serialize_tmp_save.pickle' mlp = Model(layers=layers) mlp.optimizer = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9) mlp.cost = GeneralizedCost(costfunc=CrossEntropyBinary()) mlp.initialize(train_set, cost=mlp.cost) n_test = 3 num_epochs = 3 # Train model for num_epochs and n_test batches for epoch in range(num_epochs): for i, (x, t) in enumerate(train_set): x = mlp.fprop(x) delta = mlp.cost.get_errors(x, t) mlp.bprop(delta) mlp.optimizer.optimize(mlp.layers_to_optimize, epoch=epoch) if i > n_test: break # Get expected outputs of n_test batches and states of all layers outputs_exp = [] pdicts_exp = [l.get_params_serialize() for l in mlp.layers_to_optimize] for i, (x, t) in enumerate(train_set): outputs_exp.append(mlp.fprop(x, inference=True)) if i > n_test: break # Serialize model mlp.save_params(tmp_save, keep_states=True) # Load model mlp = Model(tmp_save) mlp.initialize(train_set) outputs = [] pdicts = [l.get_params_serialize() for l in mlp.layers_to_optimize] for i, (x, t) in enumerate(train_set): outputs.append(mlp.fprop(x, inference=True)) if i > n_test: break # Check outputs, states, and params are the same for output, output_exp in zip(outputs, outputs_exp): assert np.allclose(output.get(), output_exp.get()) for pd, pd_exp in zip(pdicts, pdicts_exp): for s, s_e in zip(pd['states'], pd_exp['states']): if isinstance(s, list): # this is the batch norm case for _s, _s_e in zip(s, s_e): assert np.allclose(_s, _s_e) else: assert np.allclose(s, s_e) for p, p_e in zip(pd['params'], pd_exp['params']): assert type(p) == type(p_e) if isinstance(p, list): # this is the batch norm case for _p, _p_e in zip(p, p_e): assert np.allclose(_p, _p_e) elif isinstance(p, np.ndarray): assert np.allclose(p, p_e) else: assert p == p_e os.remove(tmp_save)
print D_layers print G_layers layers = myGenerativeAdversarial(generator=G_layers, discriminator=D_layers) #discriminator=Sequential(D_layers, name="Discriminator")) print 'layers defined' print layers # setup optimizer # optimizer = RMSProp(learning_rate=1e-4, decay_rate=0.9, epsilon=1e-8) optimizer = GradientDescentMomentum(learning_rate=1e-3, momentum_coef = 0.9) #optimizer = Adam(learning_rate=1e-3) # setup cost function as Binary CrossEntropy #cost = GeneralizedGANCost(costfunc=GANCost(func="wasserstein")) #cost = GeneralizedGANCost(costfunc=Multicost[GANCost(func="wasserstein"), MeanSquared, MeanSquared]) cost = Multicost(costs=[GeneralizedGANCost(costfunc=GANCost(func="wasserstein")), GeneralizedCost(costfunc=MeanSquared), GeneralizedCost(costfunc=MeanSquared)]) nb_epochs = 15 latent_size = 200 # initialize model noise_dim = (latent_size) gan = myGAN(layers=layers, noise_dim=noise_dim, dataset=train_set, k=5, wgan_param_clamp=0.9) # configure callbacks callbacks = Callbacks(gan, eval_set=valid_set) callbacks.add_callback(GANCostCallback()) #callbacks.add_save_best_state_callback("./best_state.pkl") print 'starting training' # run fit gan.fit(train_set, num_epochs=nb_epochs, optimizer=optimizer, cost=cost, callbacks=callbacks)
# setting model layers for AE1 encoder1 = Affine(nout=config.encoder_size[0], init=init_norm, activation=Logistic(), name='encoder1') decoder1 = Affine(nout=image_size, init=init_norm, activation=Logistic(), name='decoder1') encoder2 = Affine(nout=config.encoder_size[1], init=init_norm, activation=Logistic(), name='encoder2') decoder2 = Affine(nout=config.encoder_size[0], init=init_norm, activation=Logistic(), name='decoder2') encoder3 = Affine(nout=config.encoder_size[2], init=init_norm, activation=Logistic(), name='encoder3') decoder3 = Affine(nout=config.encoder_size[1], init=init_norm, activation=Logistic(), name='decoder3') classifier = Affine(nout=config.ydim, init=init_norm, activation=Softmax()) cost_reconst = GeneralizedCost(costfunc=SumSquared()) cost_classification = GeneralizedCost(costfunc=CrossEntropyMulti()) # Setting model layers for AE1 AE1 = Model([encoder1, decoder1]) AE1.cost = cost_reconst AE1.initialize(data, cost_reconst) # AE1.optimizer = optimizer_default measure_time(data, AE1, config, 'AE1') # Setting model layers for AE2 # It has an extra encoder layer compared to what AE should really be. This is # done to avoid saving the outputs for each AE. AE2_mimic = Model([encoder1, encoder2, decoder2]) AE2_mimic.cost = cost_reconst AE2_mimic.initialize(data, cost_reconst)
def benchmark(self): for d in self.devices: b = d if (self.backends is None) or ( "mkl" not in self.backends) else "mkl" print("Use {} as backend.".format(b)) # Common suffix suffix = "neon_{}_{}_{}by{}_{}".format(b, self.dataset, self.resize_size[0], self.resize_size[1], self.preprocessing) # Set up backend # backend: 'cpu' for single cpu, 'mkl' for cpu using mkl library, and 'gpu' for gpu be = gen_backend(backend=b, batch_size=self.batch_size, rng_seed=542, datatype=np.float32) # Prepare training/validation/testing sets neon_train_set = ArrayIterator(X=np.asarray( [t.flatten().astype('float32') / 255 for t in self.x_train]), y=np.asarray(self.y_train), make_onehot=True, nclass=self.class_num, lshape=(3, self.resize_size[0], self.resize_size[1])) neon_valid_set = ArrayIterator(X=np.asarray( [t.flatten().astype('float32') / 255 for t in self.x_valid]), y=np.asarray(self.y_valid), make_onehot=True, nclass=self.class_num, lshape=(3, self.resize_size[0], self.resize_size[1])) neon_test_set = ArrayIterator(X=np.asarray([ t.flatten().astype('float32') / 255 for t in self.testImages ]), y=np.asarray(self.testLabels), make_onehot=True, nclass=self.class_num, lshape=(3, self.resize_size[0], self.resize_size[1])) # Initialize model object self.neon_model = SelfModel(layers=self.constructCNN()) # Costs neon_cost = GeneralizedCost(costfunc=CrossEntropyMulti()) # Model summary self.neon_model.initialize(neon_train_set, neon_cost) print(self.neon_model) # Learning rules neon_optimizer = SGD(0.01, momentum_coef=0.9, schedule=ExpSchedule(0.2)) # neon_optimizer = RMSProp(learning_rate=0.0001, decay_rate=0.95) # # Benchmark for 20 minibatches # d[b] = self.neon_model.benchmark(neon_train_set, cost=neon_cost, optimizer=neon_optimizer) # Reset model # self.neon_model = None # self.neon_model = Model(layers=layers) # self.neon_model.initialize(neon_train_set, neon_cost) # Callbacks: validate on validation set callbacks = Callbacks( self.neon_model, eval_set=neon_valid_set, metric=Misclassification(3), output_file="./saved_data/{}/{}/callback_data_{}.h5".format( self.network_type, d, suffix)) callbacks.add_callback( SelfCallback(eval_set=neon_valid_set, test_set=neon_test_set, epoch_freq=1)) # Fit start = time.time() self.neon_model.fit(neon_train_set, optimizer=neon_optimizer, num_epochs=self.epoch_num, cost=neon_cost, callbacks=callbacks) print("Neon training finishes in {:.2f} seconds.".format( time.time() - start)) # Result # results = self.neon_model.get_outputs(neon_valid_set) # Print error on validation set start = time.time() neon_error_mis = self.neon_model.eval( neon_valid_set, metric=Misclassification()) * 100 print( 'Misclassification error = {:.1f}%. Finished in {:.2f} seconds.' .format(neon_error_mis[0], time.time() - start)) # start = time.time() # neon_error_top3 = self.neon_model.eval(neon_valid_set, metric=TopKMisclassification(3))*100 # print('Top 3 Misclassification error = {:.1f}%. Finished in {:.2f} seconds.'.format(neon_error_top3[2], time.time() - start)) # start = time.time() # neon_error_top5 = self.neon_model.eval(neon_valid_set, metric=TopKMisclassification(5))*100 # print('Top 5 Misclassification error = {:.1f}%. Finished in {:.2f} seconds.'.format(neon_error_top5[2], time.time() - start)) self.neon_model.save_params("./saved_models/{}/{}/{}.prm".format( self.network_type, d, suffix)) # Print error on test set start = time.time() neon_error_mis_t = self.neon_model.eval( neon_test_set, metric=Misclassification()) * 100 print( 'Misclassification error = {:.1f}% on test set. Finished in {:.2f} seconds.' .format(neon_error_mis_t[0], time.time() - start)) # start = time.time() # neon_error_top3_t = self.neon_model.eval(neon_test_set, metric=TopKMisclassification(3))*100 # print('Top 3 Misclassification error = {:.1f}% on test set. Finished in {:.2f} seconds.'.format(neon_error_top3_t[2], time.time() - start)) # start = time.time() # neon_error_top5_t = self.neon_model.eval(neon_test_set, metric=TopKMisclassification(5))*100 # print('Top 5 Misclassification error = {:.1f}% on test set. Finished in {:.2f} seconds.'.format(neon_error_top5_t[2], time.time() - start)) cleanup_backend() self.neon_model = None
def main(): # Get command-line parameters parser = get_p1b3_parser() args = parser.parse_args() #print('Args:', args) # Get parameters from configuration file fileParameters = p1b3.read_config_file(args.config_file) #print ('Params:', fileParameters) # Correct for arguments set by default by neon parser # (i.e. instead of taking the neon parser default value fall back to the config file, # if effectively the command-line was used, then use the command-line value) # This applies to conflictive parameters: batch_size, epochs and rng_seed if not any("--batch_size" in ag or "-z" in ag for ag in sys.argv): args.batch_size = fileParameters['batch_size'] if not any("--epochs" in ag or "-e" in ag for ag in sys.argv): args.epochs = fileParameters['epochs'] if not any("--rng_seed" in ag or "-r" in ag for ag in sys.argv): args.rng_seed = fileParameters['rng_seed'] # Consolidate parameter set. Command-line parameters overwrite file configuration gParameters = p1_common.args_overwrite_config(args, fileParameters) print('Params:', gParameters) # Determine verbosity level loggingLevel = logging.DEBUG if args.verbose else logging.INFO logging.basicConfig(level=loggingLevel, format='') # Construct extension to save model ext = p1b3.extension_from_parameters(gParameters, '.neon') # Get default parameters for initialization and optimizer functions kerasDefaults = p1_common.keras_default_config() seed = gParameters['rng_seed'] # Build dataset loader object loader = p1b3.DataLoader( seed=seed, dtype=gParameters['datatype'], val_split=gParameters['validation_split'], test_cell_split=gParameters['test_cell_split'], cell_features=gParameters['cell_features'], drug_features=gParameters['drug_features'], feature_subsample=gParameters['feature_subsample'], scaling=gParameters['scaling'], scramble=gParameters['scramble'], min_logconc=gParameters['min_logconc'], max_logconc=gParameters['max_logconc'], subsample=gParameters['subsample'], category_cutoffs=gParameters['category_cutoffs']) # Re-generate the backend after consolidating parsing and file config gen_backend(backend=args.backend, rng_seed=seed, device_id=args.device_id, batch_size=gParameters['batch_size'], datatype=gParameters['datatype'], max_devices=args.max_devices, compat_mode=args.compat_mode) # Initialize weights and learning rule initializer_weights = p1_common_neon.build_initializer( gParameters['initialization'], kerasDefaults, seed) initializer_bias = p1_common_neon.build_initializer( 'constant', kerasDefaults, 0.) activation = p1_common_neon.get_function(gParameters['activation'])() # Define model architecture layers = [] reshape = None if 'dense' in gParameters: # Build dense layers for layer in gParameters['dense']: if layer: layers.append( Affine(nout=layer, init=initializer_weights, bias=initializer_bias, activation=activation)) if gParameters['drop']: layers.append(Dropout(keep=(1 - gParameters['drop']))) else: # Build convolutional layers reshape = (1, loader.input_dim, 1) layer_list = list(range(0, len(gParameters['conv']), 3)) for l, i in enumerate(layer_list): nb_filter = gParameters['conv'][i] filter_len = gParameters['conv'][i + 1] stride = gParameters['conv'][i + 2] # print(nb_filter, filter_len, stride) # fshape: (height, width, num_filters). layers.append( Conv((1, filter_len, nb_filter), strides={ 'str_h': 1, 'str_w': stride }, init=initializer_weights, activation=activation)) if gParameters['pool']: layers.append(Pooling((1, gParameters['pool']))) layers.append( Affine(nout=1, init=initializer_weights, bias=initializer_bias, activation=neon.transforms.Identity())) # Build model model = Model(layers=layers) # Define neon data iterators train_samples = int(loader.n_train) val_samples = int(loader.n_val) if 'train_samples' in gParameters: train_samples = gParameters['train_samples'] if 'val_samples' in gParameters: val_samples = gParameters['val_samples'] train_iter = ConcatDataIter(loader, ndata=train_samples, lshape=reshape, datatype=gParameters['datatype']) val_iter = ConcatDataIter(loader, partition='val', ndata=val_samples, lshape=reshape, datatype=gParameters['datatype']) # Define cost and optimizer cost = GeneralizedCost(p1_common_neon.get_function(gParameters['loss'])()) optimizer = p1_common_neon.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) callbacks = Callbacks(model, eval_set=val_iter, eval_freq=1) #**args.callback_args) model.fit(train_iter, optimizer=optimizer, num_epochs=gParameters['epochs'], cost=cost, callbacks=callbacks)
from neon.backends import gen_backend import bot_params as params import replay_memory as mem from enemydetector1 import model, predict params.batch_size = 64 be = gen_backend(backend='cpu', batch_size=params.batch_size) dataset = mem.load() opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9, stochastic_round=0) cost = GeneralizedCost(costfunc=CrossEntropyMulti(scale=10)) (X_train, y_train), (X_test, y_test) = dataset.get_dataset() print X_train.shape, y_train.shape, X_test.shape, y_test.shape train_set = ArrayIterator(X=X_train, y=y_train, nclass=dataset.nclass, lshape=dataset.shape, make_onehot=False) test = ArrayIterator(X=X_test, y=y_test, nclass=dataset.nclass, lshape=dataset.shape, make_onehot=False) callbacks = Callbacks(model, eval_set=test, eval_freq=1,) model.fit(train_set, optimizer=opt_gdm, num_epochs=2, cost=cost, callbacks=callbacks) model.save_params(params.weigths_path) def test_example(i): val = predict(X_train[i])
def create_network_lrn(): init1 = Gaussian(scale=0.01) init2 = Gaussian(scale=0.005) layers = [ Conv((11, 11, 96), padding=0, strides=4, init=init1, bias=Constant(0), activation=Rectlin(), name='conv1'), Pooling(3, strides=2, name='pool1'), LRN(5, ascale=0.0001, bpower=0.75, name='norm1'), Conv((5, 5, 256), padding=2, init=init1, bias=Constant(1.0), activation=Rectlin(), name='conv2'), Pooling(3, strides=2, name='pool2'), LRN(5, ascale=0.0001, bpower=0.75, name='norm2'), Conv((3, 3, 384), padding=1, init=init1, bias=Constant(0), activation=Rectlin(), name='conv3'), Conv((3, 3, 384), padding=1, init=init1, bias=Constant(1.0), activation=Rectlin(), name='conv4'), Conv((3, 3, 256), padding=1, init=init1, bias=Constant(1.0), activation=Rectlin(), name='conv5'), Pooling(3, strides=2, name='pool5'), Affine(nout=4096, init=init2, bias=Constant(1.0), activation=Rectlin(), name='fc6'), Dropout(keep=0.5, name='drop6'), Affine(nout=4096, init=init2, bias=Constant(1.0), activation=Rectlin(), name='fc7'), Dropout(keep=0.5, name='drop7'), Affine(nout=1000, init=init1, bias=Constant(0.0), activation=Softmax(), name='fc8') ] return Model(layers=layers), GeneralizedCost(costfunc=CrossEntropyMulti())
def main(): # larger batch sizes may not fit on GPU parser = NeonArgparser(__doc__, default_overrides={'batch_size': 4}) parser.add_argument("--bench", action="store_true", help="run benchmark instead of training") parser.add_argument("--num_classes", type=int, default=12, help="number of classes in the annotation") parser.add_argument("--height", type=int, default=256, help="image height") parser.add_argument("--width", type=int, default=512, help="image width") args = parser.parse_args(gen_be=False) # check that image dimensions are powers of 2 if((args.height & (args.height - 1)) != 0): raise TypeError("Height must be a power of 2.") if((args.width & (args.width - 1)) != 0): raise TypeError("Width must be a power of 2.") (c, h, w) = (args.num_classes, args.height, args.width) # need to use the backend with the new upsampling layer implementation be = NervanaGPU_Upsample(rng_seed=args.rng_seed, device_id=args.device_id) # set batch size be.bsz = args.batch_size # couple backend to global neon object NervanaObject.be = be shape = dict(channel_count=3, height=h, width=w, subtract_mean=False) train_params = ImageParams(center=True, flip=False, scale_min=min(h, w), scale_max=min(h, w), aspect_ratio=0, **shape) test_params = ImageParams(center=True, flip=False, scale_min=min(h, w), scale_max=min(h, w), aspect_ratio=0, **shape) common = dict(target_size=h*w, target_conversion='read_contents', onehot=False, target_dtype=np.uint8, nclasses=args.num_classes) train_set = PixelWiseImageLoader(set_name='train', repo_dir=args.data_dir, media_params=train_params, shuffle=False, subset_percent=100, index_file=os.path.join(args.data_dir, 'train_images.csv'), **common) val_set = PixelWiseImageLoader(set_name='val', repo_dir=args.data_dir,media_params=test_params, index_file=os.path.join(args.data_dir, 'val_images.csv'), **common) # initialize model object layers = gen_model(c, h, w) segnet_model = Model(layers=layers) # configure callbacks callbacks = Callbacks(segnet_model, eval_set=val_set, **args.callback_args) opt_gdm = GradientDescentMomentum(1.0e-6, 0.9, wdecay=0.0005, schedule=Schedule()) opt_biases = GradientDescentMomentum(2.0e-6, 0.9, schedule=Schedule()) opt_bn = GradientDescentMomentum(1.0e-6, 0.9, schedule=Schedule()) opt = MultiOptimizer({'default': opt_gdm, 'Bias': opt_biases, 'BatchNorm': opt_bn}) cost = GeneralizedCost(costfunc=CrossEntropyMulti()) if args.bench: segnet_model.initialize(train_set, cost=cost) segnet_model.benchmark(train_set, cost=cost, optimizer=opt) sys.exit(0) else: segnet_model.fit(train_set, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks) # get the trained segnet model outputs for valisation set outs_val = segnet_model.get_outputs(val_set) with open('outputs.pkl', 'w') as fid: pickle.dump(outs_val, fid, -1)
wdecay=args.weight_decay, schedule=weight_sched, stochastic_round=args.rounding) opt_biases = GradientDescentMomentum(args.rate_init[1], args.momentum[1], schedule=weight_sched, stochastic_round=args.rounding) opt_fixed = GradientDescentMomentum(0.0, 1.0, wdecay=0.0) opt = MultiOptimizer({ 'default': opt_gdm, 'Bias': opt_biases, 'DOG': opt_fixed }) # configure cost and test metrics cost = GeneralizedCost(costfunc=(CrossEntropyBinary() \ if train.parser.independent_labels else CrossEntropyMulti())) metric = EMMetric( oshape=test.parser.oshape, use_softmax=not train.parser.independent_labels) if test else None # configure callbacks if not args.neon_progress: args.callback_args['progress_bar'] = False callbacks = Callbacks(model, eval_set=test, metric=metric, **args.callback_args) if not args.neon_progress: callbacks.add_callback(EMEpochCallback( args.callback_args['eval_freq'], train.nmacrobatches), insert_pos=None)
train_set = ArrayIterator(X=X_train, y=y_train, make_onehot=False) val_set = ArrayIterator(X=X_val, y=y_val, make_onehot=False) # setup weight initialization function init = Uniform(-1, 1) # setup layers layers = [ BinaryAffine(nout=4096, init=init, batch_norm=True, activation=Sign()), BinaryAffine(nout=4096, init=init, batch_norm=True, activation=Sign()), BinaryAffine(nout=4096, init=init, batch_norm=True, activation=Sign()), BinaryAffine(nout=2, init=init, batch_norm=True, activation=Identity()) ] # setup cost function as Square Hinge Loss cost = GeneralizedCost(costfunc=SquareHingeLoss()) # setup optimizer LR_start = 1.65e-2 def ShiftAdaMax_with_Scale(LR=1): return ShiftAdaMax(learning_rate=LR_start * LR, schedule=ShiftSchedule(2, shift_size=1)) optimizer = MultiOptimizer({ 'default': ShiftAdaMax_with_Scale(), 'BinaryLinear_0': ShiftAdaMax_with_Scale(57.038), 'BinaryLinear_1': ShiftAdaMax_with_Scale(73.9008), 'BinaryLinear_2': ShiftAdaMax_with_Scale(73.9008),
def main(): # Get command-line parameters parser = get_p1b2_parser() args = parser.parse_args() #print('Args:', args) # Get parameters from configuration file fileParameters = p1b2.read_config_file(args.config_file) #print ('Params:', fileParameters) # Correct for arguments set by default by neon parser # (i.e. instead of taking the neon parser default value fall back to the config file, # if effectively the command-line was used, then use the command-line value) # This applies to conflictive parameters: batch_size, epochs and rng_seed if not any("--batch_size" in ag or "-z" in ag for ag in sys.argv): args.batch_size = fileParameters['batch_size'] if not any("--epochs" in ag or "-e" in ag for ag in sys.argv): args.epochs = fileParameters['epochs'] if not any("--rng_seed" in ag or "-r" in ag for ag in sys.argv): args.rng_seed = fileParameters['rng_seed'] # Consolidate parameter set. Command-line parameters overwrite file configuration gParameters = p1_common.args_overwrite_config(args, fileParameters) print('Params:', gParameters) # Determine verbosity level loggingLevel = logging.DEBUG if args.verbose else logging.INFO logging.basicConfig(level=loggingLevel, format='') # Construct extension to save model ext = p1b2.extension_from_parameters(gParameters, '.neon') # Get default parameters for initialization and optimizer functions kerasDefaults = p1_common.keras_default_config() seed = gParameters['rng_seed'] # Load dataset #(X_train, y_train), (X_test, y_test) = p1b2.load_data(gParameters, seed) (X_train, y_train), (X_val, y_val), (X_test, y_test) = p1b2.load_data(gParameters, seed) print("Shape X_train: ", X_train.shape) print("Shape X_val: ", X_val.shape) print("Shape X_test: ", X_test.shape) print("Shape y_train: ", y_train.shape) print("Shape y_val: ", y_val.shape) print("Shape y_test: ", y_test.shape) print("Range X_train --> Min: ", np.min(X_train), ", max: ", np.max(X_train)) print("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val)) print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test)) print("Range y_train --> Min: ", np.min(y_train), ", max: ", np.max(y_train)) print("Range y_val --> Min: ", np.min(y_val), ", max: ", np.max(y_val)) print("Range y_test --> Min: ", np.min(y_test), ", max: ", np.max(y_test)) input_dim = X_train.shape[1] num_classes = int(np.max(y_train)) + 1 output_dim = num_classes # The backend will represent the classes using one-hot representation (but requires an integer class as input !) # Re-generate the backend after consolidating parsing and file config gen_backend(backend=args.backend, rng_seed=seed, device_id=args.device_id, batch_size=gParameters['batch_size'], datatype=gParameters['data_type'], max_devices=args.max_devices, compat_mode=args.compat_mode) train = ArrayIterator(X=X_train, y=y_train, nclass=num_classes) val = ArrayIterator(X=X_val, y=y_val, nclass=num_classes) test = ArrayIterator(X=X_test, y=y_test, nclass=num_classes) # Initialize weights and learning rule initializer_weights = p1_common_neon.build_initializer( gParameters['initialization'], kerasDefaults, seed) initializer_bias = p1_common_neon.build_initializer( 'constant', kerasDefaults, 0.) activation = p1_common_neon.get_function(gParameters['activation'])() # Define MLP architecture layers = [] reshape = None for layer in gParameters['dense']: if layer: layers.append( Affine(nout=layer, init=initializer_weights, bias=initializer_bias, activation=activation)) if gParameters['dropout']: layers.append(Dropout(keep=(1 - gParameters['dropout']))) layers.append( Affine(nout=output_dim, init=initializer_weights, bias=initializer_bias, activation=activation)) # Build MLP model mlp = Model(layers=layers) # Define cost and optimizer cost = GeneralizedCost(p1_common_neon.get_function(gParameters['loss'])()) optimizer = p1_common_neon.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) callbacks = Callbacks(mlp, eval_set=val, metric=Accuracy(), eval_freq=1) # Seed random generator for training np.random.seed(seed) mlp.fit(train, optimizer=optimizer, num_epochs=gParameters['epochs'], cost=cost, callbacks=callbacks) # model save #save_fname = "model_mlp_W_" + ext #mlp.save_params(save_fname) # Evalute model on test set print('Model evaluation by neon: ', mlp.eval(test, metric=Accuracy())) y_pred = mlp.get_outputs(test) #print ("Shape y_pred: ", y_pred.shape) scores = p1b2.evaluate_accuracy(p1_common.convert_to_class(y_pred), y_test) print('Evaluation on test data:', scores)
momentum_coef=0.9, stochastic_round=args.rounding) elif args.datatype in [np.float16]: opt_gdm = GradientDescentMomentum(learning_rate=0.01 / cost_scale, momentum_coef=0.9, stochastic_round=args.rounding) bn = True layers = [Conv((5, 5, 16), init=init_uni, activation=Rectlin(), batch_norm=bn), Pooling((2, 2)), Conv((5, 5, 32), init=init_uni, activation=Rectlin(), batch_norm=bn), Pooling((2, 2)), Affine(nout=500, init=init_uni, activation=Rectlin(), batch_norm=bn), Affine(nout=10, init=init_uni, activation=Softmax())] if args.datatype in [np.float32, np.float64]: cost = GeneralizedCost(costfunc=CrossEntropyMulti()) elif args.datatype in [np.float16]: cost = GeneralizedCost(costfunc=CrossEntropyMulti(scale=cost_scale)) model = Model(layers=layers) # configure callbacks callbacks = Callbacks(model, eval_set=test, **args.callback_args) model.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks) error_rate = model.eval(test, metric=Misclassification()) neon_logger.display('Misclassification error = %.1f%%' % (error_rate * 100))
def main(): # Get command-line parameters parser = get_p1b1_parser() args = parser.parse_args() #print('Args:', args) # Get parameters from configuration file fileParameters = p1b1.read_config_file(args.config_file) #print ('Params:', fileParameters) # Correct for arguments set by default by neon parser # (i.e. instead of taking the neon parser default value fall back to the config file, # if effectively the command-line was used, then use the command-line value) # This applies to conflictive parameters: batch_size, epochs and rng_seed if not any("--batch_size" in ag or "-z" in ag for ag in sys.argv): args.batch_size = fileParameters['batch_size'] if not any("--epochs" in ag or "-e" in ag for ag in sys.argv): args.epochs = fileParameters['epochs'] if not any("--rng_seed" in ag or "-r" in ag for ag in sys.argv): args.rng_seed = fileParameters['rng_seed'] # Consolidate parameter set. Command-line parameters overwrite file configuration gParameters = p1_common.args_overwrite_config(args, fileParameters) print('Params:', gParameters) # Determine verbosity level loggingLevel = logging.DEBUG if args.verbose else logging.INFO logging.basicConfig(level=loggingLevel, format='') # Construct extension to save model ext = p1b1.extension_from_parameters(gParameters, '.neon') # Get default parameters for initialization and optimizer functions kerasDefaults = p1_common.keras_default_config() seed = gParameters['rng_seed'] # Load dataset X_train, X_val, X_test = p1b1.load_data(gParameters, seed) print("Shape X_train: ", X_train.shape) print("Shape X_val: ", X_val.shape) print("Shape X_test: ", X_test.shape) print("Range X_train --> Min: ", np.min(X_train), ", max: ", np.max(X_train)) print("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val)) print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test)) input_dim = X_train.shape[1] output_dim = input_dim # Re-generate the backend after consolidating parsing and file config gen_backend(backend=args.backend, rng_seed=seed, device_id=args.device_id, batch_size=gParameters['batch_size'], datatype=gParameters['datatype'], max_devices=args.max_devices, compat_mode=args.compat_mode) # Set input and target to X_train train = ArrayIterator(X_train) val = ArrayIterator(X_val) test = ArrayIterator(X_test) # Initialize weights and learning rule initializer_weights = p1_common_neon.build_initializer( gParameters['initialization'], kerasDefaults) initializer_bias = p1_common_neon.build_initializer( 'constant', kerasDefaults, 0.) activation = p1_common_neon.get_function(gParameters['activation'])() # Define Autoencoder architecture layers = [] reshape = None # Autoencoder layers_params = gParameters['dense'] if layers_params != None: if type(layers_params) != list: layers_params = list(layers_params) # Encoder Part for i, l in enumerate(layers_params): layers.append( Affine(nout=l, init=initializer_weights, bias=initializer_bias, activation=activation)) # Decoder Part for i, l in reversed(list(enumerate(layers_params))): if i < len(layers) - 1: layers.append( Affine(nout=l, init=initializer_weights, bias=initializer_bias, activation=activation)) layers.append( Affine(nout=output_dim, init=initializer_weights, bias=initializer_bias, activation=activation)) # Build Autoencoder model ae = Model(layers=layers) # Define cost and optimizer cost = GeneralizedCost(p1_common_neon.get_function(gParameters['loss'])()) optimizer = p1_common_neon.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) callbacks = Callbacks(ae, eval_set=val, eval_freq=1) # Seed random generator for training np.random.seed(seed) ae.fit(train, optimizer=optimizer, num_epochs=gParameters['epochs'], cost=cost, callbacks=callbacks) # model save #save_fname = "model_ae_W" + ext #ae.save_params(save_fname) # Compute errors X_pred = ae.get_outputs(test) scores = p1b1.evaluate_autoencoder(X_pred, X_test) print('Evaluation on test data:', scores) diff = X_pred - X_test # Plot histogram of errors comparing input and output of autoencoder plt.hist(diff.ravel(), bins='auto') plt.title("Histogram of Errors with 'auto' bins") plt.savefig('histogram_neon.png')
DeepBiRNN(hidden_size, init=glorot, activation=Rectlinclip(), batch_norm=True, reset_cells=True, depth=depth), Affine(hidden_size, init=glorot, activation=Rectlinclip()), Affine(nout=nout, init=glorot, activation=Identity()) ] model = Model(layers=layers) opt = GradientDescentMomentumNesterov(learning_rate, momentum, gradient_clip_norm=gradient_clip_norm, stochastic_round=False) callbacks = Callbacks(model, eval_set=dev, **args.callback_args) # Print validation set word error rate at the end of every epoch pcb = WordErrorRateCallback(dev, argmax_decoder, max_tscrpt_len, epoch_freq=1) callbacks.add_callback(pcb) cost = GeneralizedCost(costfunc=CTC(max_tscrpt_len, nout=nout)) # Fit the model model.fit(train, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks)
def main(): parser = NeonArgparser(__doc__) args = parser.parse_args(gen_be=False) #mat_data = sio.loadmat('../data/timeseries/02_timeseries.mat') #ts = V1TimeSeries(mat_data['timeseries'], mat_data['stim'], binning=10) seq_len = 30 hidden = 20 be = gen_backend(**extract_valid_args(args, gen_backend)) kohn = KohnV1Dataset(path='../tmp/') kohn.gen_iterators(seq_len) import pdb; pdb.set_trace() train_spike_set = V1IteratorSequence(ts.train, seq_len, return_sequences=False) valid_spike_set = V1IteratorSequence(ts.test, seq_len, return_sequences=False) init = GlorotUniform() # dataset = MNIST(path=args.data_dir) # (X_train, y_train), (X_test, y_test), nclass = dataset.load_data() # train_set = ArrayIterator([X_train, X_train], y_train, nclass=nclass, lshape=(1, 28, 28)) # valid_set = ArrayIterator([X_test, X_test], y_test, nclass=nclass, lshape=(1, 28, 28)) # # weight initialization # init_norm = Gaussian(loc=0.0, scale=0.01) # # initialize model # path1 = Sequential(layers=[Affine(nout=100, init=init_norm, activation=Rectlin()), # Affine(nout=100, init=init_norm, activation=Rectlin())]) # path2 = Sequential(layers=[Affine(nout=100, init=init_norm, activation=Rectlin()), # Affine(nout=100, init=init_norm, activation=Rectlin())]) # layers = [MergeMultistream(layers=[path1, path2], merge="stack"), # Affine(nout=10, init=init_norm, activation=Logistic(shortcut=True))] spike_rnn_path = Sequential( layers = [ LSTM(hidden, init, activation=Logistic(), gate_activation=Logistic(), reset_cells=False), Dropout(keep=0.5), LSTM(hidden, init, activation=Logistic(), gate_activation=Logistic(), reset_cells=False), #Dropout(keep=0.85), RecurrentLast(), Affine(train_set.nfeatures, init, bias=init, activation=Identity(), name='spike_in')]) stim_rnn_path = Sequential( layers = [ LSTM(hidden, init, activation=Logistic(), gate_activation=Logistic(), reset_cells=False), Dropout(keep=0.5), RecurrentLast(), Affine(1, init, bias=init, activation=Identity(), name='stim')]) layers = [ MergeMultiStream( layers = [ spike_rnn_path, stim_rnn_path], merge="stack"), Affine(train_set.nfeatures, init, bias=init, activation=Identity(), name='spike_out'), Round() ] model = Model(layers=layers) sched = ExpSchedule(decay=0.7) # cost = GeneralizedCost(SumSquared()) cost = GeneralizedCost(MeanSquared()) optimizer_two = RMSProp(stochastic_round=args.rounding) optimizer_one = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9, schedule=sched) opt = MultiOptimizer({'default': optimizer_one, 'Bias': optimizer_two, 'special_linear': optimizer_two}) callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args) callbacks.add_hist_callback(filter_key = ['W']) #callbacks.add_callback(MetricCallback(eval_set=valid_set, metric=FractionExplainedVariance(), epoch_freq=args.eval_freq)) #callbacks.add_callback(MetricCallback(eval_set=valid_set,metric=Accuracy(), epoch_freq=args.eval_freq)) model.fit(train_set, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks) train_output = model.get_outputs( train_set).reshape(-1, train_set.nfeatures) valid_output = model.get_outputs( valid_set).reshape(-1, valid_set.nfeatures) train_target = train_set.y_series valid_target = valid_set.y_series tfev = fev(train_output, train_target, train_set.mean) vfev = fev(valid_output, valid_target, valid_set.mean) neon_logger.display('Train FEV: %g, Valid FEV: %g' % (tfev, vfev)) # neon_logger.display('Train Mean: %g, Valid Mean: %g' % (train_set.mean, valid_set.mean)) plt.figure() plt.plot(train_output[:, 0], train_output[ :, 1], 'bo', label='prediction') plt.plot(train_target[:, 0], train_target[:, 1], 'r.', label='target') plt.legend() plt.title('Neon on training set') plt.savefig('neon_series_training_output.png') plt.figure() plt.plot(valid_output[:, 0], valid_output[ :, 1], 'bo', label='prediction') plt.plot(valid_target[:, 0], valid_target[:, 1], 'r.', label='target') plt.legend() plt.title('Neon on validation set') plt.savefig('neon_series_validation_output.png')
Conv((3, 3, 192), name="D31", **convp1), Conv((1, 1, 16), name="D32", **conv), Conv((7, 7, 1), name="D_out", init=init, batch_norm=False, activation=Logistic(shortcut=False)) ] layers = GenerativeAdversarial(generator=Sequential(G_layers, name="Generator"), discriminator=Sequential(D_layers, name="Discriminator")) # setup cost function as CrossEntropy cost = GeneralizedCost(costfunc=GANCost(func="modified")) # setup optimizer optimizer = Adam(learning_rate=0.0005, beta_1=0.5) # initialize model noise_dim = (2, 7, 7) gan = GAN(layers=layers, noise_dim=noise_dim, k=args.kbatch) # configure callbacks callbacks = Callbacks(gan, eval_set=valid_set, **args.callback_args) callbacks.add_callback(GANPlotCallback(filename=splitext(__file__)[0], hw=27)) # run fit gan.fit(train_set, optimizer=optimizer, num_epochs=args.epochs,
def __init__(self, num_actions, args): # remember parameters self.num_actions = num_actions self.batch_size = args.batch_size self.discount_rate = args.discount_rate self.history_length = args.history_length self.screen_dim = (args.screen_height, args.screen_width) self.clip_error = args.clip_error self.min_reward = args.min_reward self.max_reward = args.max_reward self.batch_norm = args.batch_norm # create Neon backend self.be = gen_backend(backend=args.backend, batch_size=args.batch_size, rng_seed=args.random_seed, device_id=args.device_id, datatype=np.dtype(args.datatype).type, stochastic_round=args.stochastic_round) # prepare tensors once and reuse them self.input_shape = (self.history_length, ) + self.screen_dim + ( self.batch_size, ) self.input = self.be.empty(self.input_shape) self.input.lshape = self.input_shape # HACK: needed for convolutional networks self.targets = self.be.empty((self.num_actions, self.batch_size)) # create model layers = self._createLayers(num_actions) self.model = Model(layers=layers) self.cost = GeneralizedCost(costfunc=SumSquared()) # Bug fix for l in self.model.layers.layers: l.parallelism = 'Disabled' self.model.initialize(self.input_shape[:-1], self.cost) if args.optimizer == 'rmsprop': self.optimizer = RMSProp(learning_rate=args.learning_rate, decay_rate=args.decay_rate, stochastic_round=args.stochastic_round) elif args.optimizer == 'adam': self.optimizer = Adam(learning_rate=args.learning_rate, stochastic_round=args.stochastic_round) elif args.optimizer == 'adadelta': self.optimizer = Adadelta(decay=args.decay_rate, stochastic_round=args.stochastic_round) else: assert false, "Unknown optimizer" # create target model self.target_steps = args.target_steps self.train_iterations = 0 if self.target_steps: self.target_model = Model(layers=self._createLayers(num_actions)) # Bug fix for l in self.target_model.layers.layers: l.parallelism = 'Disabled' self.target_model.initialize(self.input_shape[:-1]) self.save_weights_prefix = args.save_weights_prefix else: self.target_model = self.model self.callback = None
def main(): # setup the model and run for num_epochs saving the last state only # this is at the top so that the be is generated mlp = gen_model(args.backend) # setup data iterators (X_train, y_train), (X_test, y_test), nclass = load_mnist(path=args.data_dir) if args.backend == 'nervanacpu' or args.backend == 'cpu': # limit data since cpu backend runs slower train = DataIterator(X_train[:1000], y_train[:1000], nclass=nclass, lshape=(1, 28, 28)) valid = DataIterator(X_test[:1000], y_test[:1000], nclass=nclass, lshape=(1, 28, 28)) else: train = DataIterator(X_train, y_train, nclass=nclass, lshape=(1, 28, 28)) valid = DataIterator(X_test, y_test, nclass=nclass, lshape=(1, 28, 28)) # serialization related cost = GeneralizedCost(costfunc=CrossEntropyBinary()) opt_gdm = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9) checkpoint_model_path = os.path.join('./', 'test_oneshot.pkl') checkpoint_schedule = 1 # save at every step callbacks = Callbacks(mlp, train) callbacks.add_serialize_callback(checkpoint_schedule, checkpoint_model_path, history=2) # run the fit all the way through saving a checkpoint e mlp.fit(train, optimizer=opt_gdm, num_epochs=num_epochs, cost=cost, callbacks=callbacks) # setup model with same random seed run epoch by epoch # serializing and deserializing at each step mlp = gen_model(args.backend) cost = GeneralizedCost(costfunc=CrossEntropyBinary()) opt_gdm = GradientDescentMomentum(learning_rate=0.1, momentum_coef=0.9) # reset data iterators train.reset() valid.reset() checkpoint_model_path = os.path.join('./', 'test_manyshot.pkl') checkpoint_schedule = 1 # save at evey step callbacks = Callbacks(mlp, train) callbacks.add_serialize_callback(checkpoint_schedule, checkpoint_model_path, history=num_epochs) for epoch in range(num_epochs): # _0 points to state at end of epoch 0 mlp.fit(train, optimizer=opt_gdm, num_epochs=epoch + 1, cost=cost, callbacks=callbacks) # load saved file prts = os.path.splitext(checkpoint_model_path) fn = prts[0] + '_%d' % epoch + prts[1] mlp.load_weights(fn) # load the saved weights # compare test_oneshot_<num_epochs>.pkl to test_manyshot_<num_epochs>.pkl try: compare_model_pickles('test_oneshot_%d.pkl' % (num_epochs - 1), 'test_manyshot_%d.pkl' % (num_epochs - 1)) except: print 'test failed....' sys.exit(1)
# setup optimizer opt_w = GradientDescentMomentum(0.001 * learning_rate_scale, 0.9, wdecay=0.0005) opt_b = GradientDescentMomentum(0.002 * learning_rate_scale, 0.9) optimizer = MultiOptimizer({'default': opt_w, 'Bias': opt_b}) # setup model model = Model(layers=Tree([frcn_layers, bb_layers])) # if training a new model, seed the Alexnet conv layers with pre-trained weights # otherwise, just load the model file if args.model_file is None: load_imagenet_weights(model, args.data_dir) cost = Multicost(costs=[GeneralizedCost(costfunc=CrossEntropyMulti()), GeneralizedCostMask(costfunc=SmoothL1Loss())], weights=[1, 1]) callbacks = Callbacks(model, **args.callback_args) model.fit(train_set, optimizer=optimizer, num_epochs=num_epochs, cost=cost, callbacks=callbacks) print 'running eval on the training set...' metric_train = model.eval(train_set, metric=ObjectDetection()) print 'Train: label accuracy - {}%, object deteciton SmoothL1Loss - {}'.format( metric_train[0]*100, metric_train[1])
import numpy as np import replay_memory import enemydetector1 try: offset_memory = replay_memory.load(bot_params.offset_data_path) print "offsets loaded" except IOError: offset_memory = replay_memory.OffsetMemory() init_norm = Gaussian(loc=-0.1, scale=0.1) layers = [Linear(1, init=init_norm), Bias(init=init_norm)] mlp = Model(layers=layers) cost = GeneralizedCost(costfunc=SumSquared()) optimizer = GradientDescentMomentum(0.5, momentum_coef=0.9) try: mlp.load_params(bot_params.aim_weights_path) except IOError: print "can't load aiming weights" def get_offset_manual(predictions): #enemy_pos = replay_memory.clean_values_toone(predictions)[0, 0] x = 0. c = 0 for i in range(0, len(predictions)): if predictions[i] > 0.97: x += (i + 1) c += 1
def test_conv_rnn(backend_default): train_shape = (1, 17, 142) be = backend_default inp = be.array(be.rng.randn(np.prod(train_shape), be.bsz)) delta = be.array(be.rng.randn(10, be.bsz)) init_norm = Gaussian(loc=0.0, scale=0.01) bilstm = DeepBiLSTM(128, init_norm, activation=Rectlin(), gate_activation=Rectlin(), depth=1, reset_cells=True) birnn_1 = DeepBiRNN(128, init_norm, activation=Rectlin(), depth=1, reset_cells=True, batch_norm=False) birnn_2 = DeepBiRNN(128, init_norm, activation=Rectlin(), depth=2, reset_cells=True, batch_norm=False) bibnrnn = DeepBiRNN(128, init_norm, activation=Rectlin(), depth=1, reset_cells=True, batch_norm=True) birnnsum = DeepBiRNN(128, init_norm, activation=Rectlin(), depth=1, reset_cells=True, batch_norm=False, bi_sum=True) rnn = Recurrent(128, init=init_norm, activation=Rectlin(), reset_cells=True) lstm = LSTM(128, init_norm, activation=Rectlin(), gate_activation=Rectlin(), reset_cells=True) gru = GRU(128, init_norm, activation=Rectlin(), gate_activation=Rectlin(), reset_cells=True) rlayers = [bilstm, birnn_1, birnn_2, bibnrnn, birnnsum, rnn, lstm, gru] for rl in rlayers: layers = [ Conv((2, 2, 4), init=init_norm, activation=Rectlin(), strides=dict(str_h=2, str_w=4)), Pooling(2, strides=2), Conv((3, 3, 4), init=init_norm, batch_norm=True, activation=Rectlin(), strides=dict(str_h=1, str_w=2)), rl, RecurrentMean(), Affine(nout=10, init=init_norm, activation=Rectlin()), ] model = Model(layers=layers) cost = GeneralizedCost(costfunc=CrossEntropyBinary()) model.initialize(train_shape, cost) model.fprop(inp) model.bprop(delta)
normalize=True, contrast_normalize=False, whiten=False) train = dataset.train_iter test = dataset.valid_iter init_uni = Uniform(low=-0.1, high=0.1) opt_gdm = GradientDescentMomentum(learning_rate=0.01, momentum_coef=0.9) # set up the model layers layers = [ Affine(nout=200, init=init_uni, activation=Rectlin()), Affine(nout=10, init=init_uni, activation=Logistic(shortcut=True)) ] cost = GeneralizedCost(costfunc=CrossEntropyBinary()) mlp = Model(layers=layers) # configure callbacks callbacks = Callbacks(mlp, eval_set=test, **args.callback_args) mlp.fit(train, optimizer=opt_gdm, num_epochs=args.epochs, cost=cost, callbacks=callbacks) neon_logger.display('Misclassification error = %.1f%%' % (mlp.eval(test, metric=Misclassification()) * 100))
p2 = [ b1, Affine(nout=16, linear_name="b1_l1", **normrelu), Affine(nout=10, linear_name="b1_l2", **normsigm) ] p3 = [ b2, Affine(nout=16, linear_name="b2_l1", **normrelu), Affine(nout=10, linear_name="b2_l2", **normsigm) ] # setup cost function as CrossEntropy cost = Multicost(costs=[ GeneralizedCost(costfunc=CrossEntropyMulti()), GeneralizedCost(costfunc=CrossEntropyBinary()), GeneralizedCost(costfunc=CrossEntropyBinary()) ], weights=[1, 0., 0.]) # setup optimizer optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9, stochastic_round=args.rounding) # initialize model object alphas = [1, 0.25, 0.25] mlp = Model(layers=Tree([p1, p2, p3], alphas=alphas)) # setup standard fit callbacks
decoder = [LookupTable(vocab_size=len(train_set.t_vocab), embedding_dim=embedding_dim, init=init, name="LUT_de")] decoder_connections = [] # link up recurrent layers for ii in range(num_layers): encoder.append(GRU(hidden_size, init, activation=Tanh(), gate_activation=Logistic(), reset_cells=True, name="GRU1Enc")) decoder.append(GRU(hidden_size, init, activation=Tanh(), gate_activation=Logistic(), reset_cells=True, name="GRU1Dec")) decoder_connections.append(ii) decoder.append(Affine(train_set.nout, init, bias=init, activation=Softmax(), name="Affout")) layers = Seq2Seq([encoder, decoder], decoder_connections=decoder_connections, name="Seq2Seq") cost = GeneralizedCost(costfunc=CrossEntropyMulti(usebits=True)) model = Model(layers=layers) optimizer = RMSProp(gradient_clip_value=gradient_clip_value, stochastic_round=args.rounding) # configure callbacks callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args) # train model model.fit(train_set, optimizer=optimizer, num_epochs=args.epochs, cost=cost, callbacks=callbacks) # obtain predictions
def main(): # parse the command line arguments parser = NeonArgparser(__doc__) args = parser.parse_args() logger = logging.getLogger() logger.setLevel(args.log_thresh) #Set up batch iterator for training images train = ImgMaster(repo_dir='spectroDataTmp', set_name='train', inner_size=400, subset_pct=100) val = ImgMaster(repo_dir='spectroDataTmp', set_name='validation', inner_size=400, subset_pct=100, do_transforms=False) test = ImgMaster(repo_dir='spectroTestDataTmp', set_name='validation', inner_size=400, subset_pct=100, do_transforms=False) train.init_batch_provider() test.init_batch_provider() print "Constructing network..." model = constuct_network() model.load_weights(args.model_file) #Optimizer opt = Adadelta() # configure callbacks valmetric = TopKMisclassification(k=5) callbacks = Callbacks(model, train, eval_set=val, metric=valmetric, **args.callback_args) cost = GeneralizedCost(costfunc=CrossEntropyMulti()) #flag = input("Press Enter if you want to begin training process.") print "Training network..." print args.epochs model.fit(train, optimizer=opt, num_epochs=args.epochs, cost=cost, callbacks=callbacks) mets = model.eval(test, metric=valmetric) print 'Validation set metrics:' print 'LogLoss: %.2f, Accuracy: %.1f %%0 (Top-1), %.1f %% (Top-5)' % ( mets[0], (1.0 - mets[1]) * 100, (1.0 - mets[2]) * 100) test.exit_batch_provider() train.exit_batch_provider()