def cnn_run_dropout_maxout(data_path, num_rows, num_cols, num_channels, input_path, pred_path): t = time.time() sub_window = gen_center_sub_window(76, num_cols) trn = SarDataset(ds[0][0], ds[0][1], sub_window) vld = SarDataset(ds[1][0], ds[1][1], sub_window) tst = SarDataset(ds[2][0], ds[2][1], sub_window) print 'Take {}s to read data'.format(time.time() - t) t = time.time() batch_size = 100 h1 = maxout.Maxout(layer_name='h2', num_units=1, num_pieces=100, irange=.1) hidden_layer = mlp.ConvRectifiedLinear(layer_name='h2', output_channels=8, irange=0.05, kernel_shape=[5, 5], pool_shape=[2, 2], pool_stride=[2, 2], max_kernel_norm=1.9365) hidden_layer2 = mlp.ConvRectifiedLinear(layer_name='h3', output_channels=8, irange=0.05, kernel_shape=[5, 5], pool_shape=[2, 2], pool_stride=[2, 2], max_kernel_norm=1.9365) #output_layer = mlp.Softplus(dim=1,layer_name='output',irange=0.1) output_layer = mlp.Linear(dim=1, layer_name='output', irange=0.05) trainer = sgd.SGD(learning_rate=0.001, batch_size=100, termination_criterion=EpochCounter(2000), cost=dropout.Dropout(), train_iteration_mode='even_shuffled_sequential', monitor_iteration_mode='even_shuffled_sequential', monitoring_dataset={ 'test': tst, 'valid': vld, 'train': trn }) layers = [hidden_layer, hidden_layer2, output_layer] input_space = space.Conv2DSpace(shape=[num_rows, num_cols], num_channels=num_channels) ann = mlp.MLP(layers, input_space=input_space, batch_size=batch_size) watcher = best_params.MonitorBasedSaveBest(channel_name='valid_objective', save_path='sar_cnn_mlp.pkl') experiment = Train(dataset=trn, model=ann, algorithm=trainer, extensions=[watcher]) print 'Take {}s to compile code'.format(time.time() - t) t = time.time() experiment.main_loop() print 'Training time: {}s'.format(time.time() - t) serial.save('cnn_hhv_{0}_{1}.pkl'.format(num_rows, num_cols), ann, on_overwrite='backup') #read hh and hv into a 3D numpy image = read_hhv(input_path) return ann, sar_predict(ann, image, pred_path)
# termination criterion that stops after 50 epochs without # any increase in misclassification on the validation set termination_criterion = MonitorBased(channel_name='objective', N=20, prop_decrease=0.0) # create Stochastic Gradient Descent trainer trainer = sgd.SGD(learning_rate=.001, batch_size=10, monitoring_dataset=ds_valid, termination_criterion=termination_criterion, cost=L1_cost) #learning_rule=momentum_rule, trainer.setup(ann, ds_train) # add monitor for saving the model with best score monitor_save_best = best_params.MonitorBasedSaveBest('objective','./tmp/best.pkl') ##################################### #Train model #################################### # train neural net until the termination criterion is true while True: trainer.train(dataset=ds_train) ann.monitor.report_epoch() ann.monitor() monitor_save_best.on_monitor(ann, ds_valid, trainer) if not trainer.continue_learning(ann): break
def main(): training_data, validation_data, test_data, std_scale = load_training_data() kaggle_test_features = load_test_data(std_scale) ############### # pylearn2 ML hl1 = mlp.Sigmoid(layer_name='hl1', dim=200, irange=.1, init_bias=1.) hl2 = mlp.Sigmoid(layer_name='hl2', dim=100, irange=.1, init_bias=1.) # create Softmax output layer output_layer = mlp.Softmax(9, 'output', irange=.1) # create Stochastic Gradient Descent trainer that runs for 400 epochs trainer = sgd.SGD(learning_rate=.05, batch_size=300, learning_rule=learning_rule.Momentum(.5), termination_criterion=MonitorBased( channel_name='valid_objective', prop_decrease=0., N=10), monitoring_dataset={ 'valid': validation_data, 'train': training_data }) layers = [hl1, hl2, output_layer] # create neural net model = mlp.MLP(layers, nvis=93) watcher = best_params.MonitorBasedSaveBest( channel_name='valid_objective', save_path='pylearn2_results/pylearn2_test.pkl') velocity = learning_rule.MomentumAdjustor(final_momentum=.6, start=1, saturate=250) decay = sgd.LinearDecayOverEpoch(start=1, saturate=250, decay_factor=.01) ###################### experiment = Train(dataset=training_data, model=model, algorithm=trainer, extensions=[watcher, velocity, decay]) experiment.main_loop() #load best model and test ################ model = serial.load('pylearn2_results/pylearn2_test.pkl') # get an prediction of the accuracy from the test_data test_results = model.fprop(theano.shared(test_data[0], name='test_data')).eval() print test_results.shape loss = multiclass_log_loss(test_data[1], test_results) print 'Test multiclass log loss:', loss out_file = 'pylearn2_results/' + str(loss) + 'ann' #exp.save(out_file + '.pkl') #save the kaggle results results = model.fprop( theano.shared(kaggle_test_features, name='kaggle_test_data')).eval() save_results(out_file + '.csv', kaggle_test_features, results)
lr = 0.1 epochs = 400 trainer = sgd.SGD( learning_rate=lr, batch_size=100, learning_rule=learning_rule.Momentum(.05), # Remember, default dropout is .5 #cost=Dropout(input_include_probs={'l1': .5}, # input_scales={'l1': 1.}), termination_criterion=EpochCounter(epochs), monitoring_dataset={ 'train': ds, 'valid': ds_test }) watcher = best_params.MonitorBasedSaveBest(channel_name='valid_roc_auc', save_path='saved_clf.pkl') velocity = learning_rule.MomentumAdjustor(final_momentum=.9, start=1, saturate=250) decay = sgd.LinearDecayOverEpoch(start=1, saturate=250, decay_factor=lr * .05) rocauc = roc_auc.RocAucChannel() experiment = Train(dataset=ds, model=ann, algorithm=trainer, extensions=[watcher, velocity, decay, rocauc]) experiment.main_loop()
termination_criterion=EpochCounter(max_epochs=max_epochs), monitoring_dataset={'train':X_train, 'valid':X_test}, ) input_space = Conv2DSpace(shape=(central_window_shape, central_window_shape), axes = axes, num_channels = 1) ann = mlp.MLP(layers, input_space=input_space) velocity = learning_rule.MomentumAdjustor(final_momentum=momentum_end, start=1, saturate=momentum_saturate) watcher = best_params.MonitorBasedSaveBest(channel_name='valid_y_nll', save_path=save_path) decay = sgd.LinearDecayOverEpoch(start=1, saturate=decay_saturate, decay_factor=decay_factor) ra = RealtimeAugment(window_shape=[img_dim, img_dim], randomize=[X_train, X_test], scale_diff=scale_diff, translation=translation, center_shape=center_shape, center=[X_train, X_test], preprocess=preprocess) train = Train(dataset=X_train, model=ann, algorithm=trainer, extensions=[watcher, velocity, decay, ra]) train.main_loop() print "using model", save_path model = serial.load(save_path)
batch_size=128, learning_rule=learning_rule.Momentum(.5), # Remember, default dropout is .5 cost=Dropout(input_include_probs={'l1': .8}, input_scales={'l1': 1.}), termination_criterion=EpochCounter(max_epochs=475), monitoring_dataset={'valid': tst, 'train': trn}) preprocessor = Pipeline([GlobalContrastNormalization(scale=55.), ZCA()]) trn.apply_preprocessor(preprocessor=preprocessor, can_fit=True) tst.apply_preprocessor(preprocessor=preprocessor, can_fit=False) serial.save('kaggle_cifar10_preprocessor.pkl', preprocessor) watcher = best_params.MonitorBasedSaveBest( channel_name='valid_y_misclass', save_path='kaggle_cifar10_maxout_zca.pkl') velocity = learning_rule.MomentumAdjustor(final_momentum=.65, start=1, saturate=250) decay = sgd.LinearDecayOverEpoch(start=1, saturate=500, decay_factor=.01) win = window_flip.WindowAndFlipC01B(pad_randomized=8, window_shape=(32, 32), randomize=[trn], center=[tst])
def cnn_train_tranformer(train_path, test_path, valid_path, save_path, predict_path, num_rows=28, num_cols=28, num_channels=2, batch_size=128, output_channels=[64, 64], kernel_shape=[[12, 12], [5, 5]], pool_shape=[[4, 4], [2, 2]], pool_stride=[[2, 2], [2, 2]], irange=[0.05, 0.05, 0.05], max_kernel_norm=[1.9365, 1.9365], learning_rate=0.001, init_momentum=0.9, weight_decay=[0.0002, 0.0002, 0.0002], n_epoch=1000, image_path=''): ds = load_data_transformed(train_path, num_cols, batch_size) ds = (np.transpose(ds[0], axes=[0, 3, 1, 2]), ds[1]) trn = SarDataset(np.array(ds[0]), ds[1]) ds = load_data_transformed(valid_path, num_cols, batch_size) ds = (np.transpose(ds[0], axes=[0, 3, 1, 2]), ds[1]) vld = SarDataset(np.array(ds[0]), ds[1]) ds = load_data_transformed(test_path, num_cols, batch_size) ds = (np.transpose(ds[0], axes=[0, 3, 1, 2]), ds[1]) tst = SarDataset(np.array(ds[0]), ds[1]) #setup the network #X = np.random.random([400000,2,41,41]) #y = np.random.random([400000,1]) #trn = SarDataset(X,y) #X = np.random.random([60000,2,41,41]) #y = np.random.random([60000,1]) #tst = SarDataset(X,y) #X = np.random.random([60000,2,41,41]) #y = np.random.random([60000,1]) #vld = SarDataset(X,y) t = time.time() layers = [] for i in range(len(output_channels)): layer_name = 'h{}'.format(i + 1) convlayer = mlp.ConvRectifiedLinear(layer_name=layer_name, output_channels=output_channels[i], irange=irange[i], kernel_shape=kernel_shape[i], pool_shape=pool_shape[i], pool_stride=pool_stride[i], max_kernel_norm=max_kernel_norm[i]) layers.append(convlayer) output_mlp = mlp.Linear(dim=1, layer_name='output', irange=irange[-1]) #output_mlp = mlp.linear_mlp_bayesian_cost(dim=1,layer_name='output',irange=irange[-1]) layers.append(output_mlp) trainer = sgd.SGD( learning_rate=learning_rate, batch_size=batch_size, termination_criterion=EpochCounter(n_epoch), #termination_criterion = termination_criteria.And([termination_criteria.MonitorBased(channel_name = 'train_objective', prop_decrease=0.01,N=10),EpochCounter(n_epoch)]), #cost = dropout.Dropout(), cost=cost.SumOfCosts( [cost.MethodCost('cost_from_X'), WeightDecay(weight_decay)]), init_momentum=init_momentum, train_iteration_mode='even_shuffled_sequential', monitor_iteration_mode='even_shuffled_sequential', monitoring_dataset={ 'test': tst, 'valid': vld, 'train': trn }) input_space = space.Conv2DSpace(shape=[num_rows, num_cols], num_channels=num_channels) #ann = mlp.MLP(layers,input_space=input_space,batch_size=batch_size) watcher = best_params.MonitorBasedSaveBest(channel_name='valid_objective', save_path=predict_path + save_path) #flip = window_flip.WindowAndFlip((num_rows,num_cols),randomize=[tst,vld,trn]) experiment = Train(dataset=trn, model=ann, algorithm=trainer, extensions=[watcher]) print 'Take {}s to compile code'.format(time.time() - t) #train the network t = time.time() experiment.main_loop() print 'Training time: {}h'.format((time.time() - t) / 3600) utils.sms_notice('Training time:{}'.format((time.time() - t) / 3600)) return ann
def cnn_train( train_path, test_path, valid_path, save_path, predict_path, image_path, num_rows=28, num_cols=28, num_channels=2, batch_size=128, output_channels=[64, 64], kernel_shape=[[12, 12], [5, 5]], pool_shape=[[4, 4], [2, 2]], pool_stride=[[2, 2], [2, 2]], irange=[0.05, 0.05, 0.05], max_kernel_norm=[1.9365, 1.9365], learning_rate=0.001, init_momentum=0.9, weight_decay=[0.0002, 0.0002, 0.0002], n_epoch=1000, ): #load data #t = time.time() ds = load_data(valid_path, num_rows, num_cols, num_channels) vld = SarDataset(np.array(ds[0]), ds[1]) ds = load_data(train_path, num_rows, num_cols, num_channels) trn = SarDataset(np.array(ds[0]), ds[1]) ds = load_data(test_path, num_rows, num_cols, num_channels) tst = SarDataset(np.array(ds[0]), ds[1]) #load balanced data #ds = load_data_balance_under_sample(train_path, num_rows,num_cols, num_channels) #trn = SarDataset(np.array(ds[0]),ds[1]) #ds = load_data_balance(valid_path, num_rows,num_cols, num_channels) #vld = SarDataset(np.array(ds[0]),ds[1]) #ds = load_data_balance(test_path, num_rows,num_cols, num_channels) #tst = SarDataset(np.array(ds[0]),ds[1]) #print 'Take {}s to read data'.format( time.time()-t) #use gaussian convlution on the origional image to see if it can concentrate in the center #trn,tst,vld = load_data_lidar() #mytransformer = transformer.TransformationPipeline(input_space=space.Conv2DSpace(shape=[num_rows,num_cols],num_channels=num_channels),transformations=[transformer.Rotation(),transformer.Flipping()]) #trn = contestTransformerDataset.TransformerDataset(trn,mytransformer,space_preserving=True) #tst = contestTransformerDataset.TransformerDataset(tst,mytransformer,space_preserving=True) #vld = contestTransformerDataset.TransformerDataset(vld,mytransformer,space_preserving=True) #trn = transformer_dataset.TransformerDataset(trn,mytransformer,space_preserving=True) #tst = transformer_dataset.TransformerDataset(tst,mytransformer,space_preserving=True) #vld = transformer_dataset.TransformerDataset(vld,mytransformer,space_preserving=True) #setup the network t = time.time() layers = [] for i in range(len(output_channels)): layer_name = 'h{}'.format(i + 1) convlayer = mlp.ConvRectifiedLinear(layer_name=layer_name, output_channels=output_channels[i], irange=irange[i], kernel_shape=kernel_shape[i], pool_shape=pool_shape[i], pool_stride=pool_stride[i], max_kernel_norm=max_kernel_norm[i]) layers.append(convlayer) output_mlp = mlp.Linear(dim=1, layer_name='output', irange=irange[-1], use_abs_loss=True) #output_mlp = mlp.linear_mlp_ace(dim=1,layer_name='output',irange=irange[-1]) layers.append(output_mlp) #ann = cPickle.load(open('../output/train_with_2010_2l_40_64/original_500/f/f0.pkl')) #layers = [] #for layer in ann.layers: # layer.set_mlp_force(None) # layers.append(layer) trainer = sgd.SGD( learning_rate=learning_rate, batch_size=batch_size, termination_criterion=EpochCounter(n_epoch), #termination_criterion = termination_criteria.And([termination_criteria.MonitorBased(channel_name = 'train_objective', prop_decrease=0.01,N=10),EpochCounter(n_epoch)]), #cost = dropout.Dropout(), cost=cost.SumOfCosts( [cost.MethodCost('cost_from_X'), WeightDecay(weight_decay)]), init_momentum=init_momentum, train_iteration_mode='even_shuffled_sequential', monitor_iteration_mode='even_shuffled_sequential', monitoring_dataset={ 'test': tst, 'valid': vld, 'train': trn }) input_space = space.Conv2DSpace(shape=[num_rows, num_cols], num_channels=num_channels) #ann = mlp.MLP(layers,input_space=input_space,batch_size=batch_size) ann = serial.load( '../output/train_with_2010_2l_40_64/original_500/f/f0.pkl') ann = monitor.push_monitor(ann, 'stage_0') watcher = best_params.MonitorBasedSaveBest(channel_name='valid_objective', save_path=predict_path + save_path) flip = window_flip.WindowAndFlip((num_rows, num_cols), randomize=[tst, vld, trn]) experiment = Train(dataset=trn, model=ann, algorithm=trainer, extensions=[watcher, flip]) print 'Take {}s to compile code'.format(time.time() - t) #train the network t = time.time() experiment.main_loop() print 'Training time: {}h'.format((time.time() - t) / 3600) utils.sms_notice('Training time:{}'.format((time.time() - t) / 3600)) return ann
def supervisedLayerwisePRL(trainset, testset): ''' The supervised layerwise training as used in the PRL Paper. Input ------ trainset : A path to an hdf5 file created through h5py. testset : A path to an hdf5 file created through h5py. ''' batch_size = 100 # Both train and test h5py files are expected to have a 'topo_view' and 'y' # datasets side them corresponding to the 'b01c' data format as used in pylearn2 # and 'y' equivalent to the one hot encoded labels trn = HDF5Dataset(filename=trainset, topo_view='topo_view', y='y', load_all=False) tst = HDF5Dataset(filename=testset, topo_view='topo_view', y='y', load_all=False) ''' The 1st Convolution and Pooling Layers are added below. ''' h1 = mlp.ConvRectifiedLinear(layer_name='h1', output_channels=64, irange=0.05, kernel_shape=[4, 4], pool_shape=[4, 4], pool_stride=[2, 2], max_kernel_norm=1.9365) fc = mlp.RectifiedLinear(layer_name='fc', dim=1500, irange=0.05) output = mlp.Softmax(layer_name='y', n_classes=171, irange=.005, max_col_norm=1.9365) layers = [h1, fc, output] mdl = mlp.MLP(layers, input_space=Conv2DSpace(shape=(70, 70), num_channels=1)) trainer = sgd.SGD( learning_rate=0.002, batch_size=batch_size, learning_rule=learning_rule.RMSProp(), cost=SumOfCosts( costs=[Default(), WeightDecay(coeffs=[0.0005, 0.0005, 0.0005])]), train_iteration_mode='shuffled_sequential', monitor_iteration_mode='sequential', termination_criterion=EpochCounter(max_epochs=15), monitoring_dataset={ 'test': tst, 'valid': vld }) watcher = best_params.MonitorBasedSaveBest( channel_name='valid_y_misclass', save_path='./Saved Models/conv_supervised_layerwise_best1.pkl') decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1) experiment = Train( dataset=trn, model=mdl, algorithm=trainer, extensions=[watcher, decay], ) experiment.main_loop() del mdl mdl = serial.load('./Saved Models/conv_supervised_layerwise_best1.pkl') mdl = push_monitor(mdl, 'k') ''' The 2nd Convolution and Pooling Layers are added below. ''' h2 = mlp.ConvRectifiedLinear(layer_name='h2', output_channels=64, irange=0.05, kernel_shape=[4, 4], pool_shape=[4, 4], pool_stride=[2, 2], max_kernel_norm=1.9365) fc = mlp.RectifiedLinear(layer_name='fc', dim=1500, irange=0.05) output = mlp.Softmax(layer_name='y', n_classes=171, irange=.005, max_col_norm=1.9365) del mdl.layers[-1] mdl.layer_names.remove('y') del mdl.layers[-1] mdl.layer_names.remove('fc') mdl.add_layers([h2, fc, output]) trainer = sgd.SGD(learning_rate=0.002, batch_size=batch_size, learning_rule=learning_rule.RMSProp(), cost=SumOfCosts(costs=[ Default(), WeightDecay(coeffs=[0.0005, 0.0005, 0.0005, 0.0005]) ]), train_iteration_mode='shuffled_sequential', monitor_iteration_mode='sequential', termination_criterion=EpochCounter(max_epochs=15), monitoring_dataset={ 'test': tst, 'valid': vld }) watcher = best_params.MonitorBasedSaveBest( channel_name='valid_y_misclass', save_path='./Saved Models/conv_supervised_layerwise_best2.pkl') decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1) experiment = Train( dataset=trn, model=mdl, algorithm=trainer, extensions=[watcher, decay], ) experiment.main_loop() del mdl mdl = serial.load('./Saved Models/conv_supervised_layerwise_best2.pkl') mdl = push_monitor(mdl, 'l') ''' The 3rd Convolution and Pooling Layers are added below. ''' h3 = mlp.ConvRectifiedLinear(layer_name='h2', output_channels=64, irange=0.05, kernel_shape=[4, 4], pool_shape=[4, 4], pool_stride=[2, 2], max_kernel_norm=1.9365) fc = mlp.RectifiedLinear(layer_name='h3', dim=1500, irange=0.05) output = mlp.Softmax(layer_name='y', n_classes=10, irange=.005, max_col_norm=1.9365) del mdl.layers[-1] mdl.layer_names.remove('y') del mdl.layers[-1] mdl.layer_names.remove('fc') mdl.add_layers([h3, output]) trainer = sgd.SGD( learning_rate=.002, batch_size=batch_size, learning_rule=learning_rule.RMSProp(), cost=SumOfCosts(costs=[ Default(), WeightDecay(coeffs=[0.0005, 0.0005, 0.0005, 0.0005, 0.0005]) ]), train_iteration_mode='shuffled_sequential', monitor_iteration_mode='sequential', termination_criterion=EpochCounter(max_epochs=15), monitoring_dataset={ 'test': tst, 'valid': vld }) watcher = best_params.MonitorBasedSaveBest( channel_name='valid_y_misclass', save_path='./Saved Models/conv_supervised_layerwise_best3.pkl') decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1) experiment = Train( dataset=trn, model=mdl, algorithm=trainer, extensions=[watcher, decay], ) experiment.main_loop()
batch_size=32, learning_rule=learning_rule.Momentum(.5), # Remember, default dropout is .5 cost=Dropout(input_include_probs={'h2': .8}, input_scales={'h2': 1.}), termination_criterion=EpochCounter(max_epochs=475), monitoring_dataset={ 'valid': tst, 'train': trn }) preprocessor = Pipeline([GlobalContrastNormalization(scale=55.), ZCA()]) trn.apply_preprocessor(preprocessor=preprocessor, can_fit=True) tst.apply_preprocessor(preprocessor=preprocessor, can_fit=False) serial.save('deepcuts_preprocessor.pkl', preprocessor) watcher = best_params.MonitorBasedSaveBest(channel_name='valid_y_misclass', save_path='deepcuts_maxout_zca.pkl') velocity = learning_rule.MomentumAdjustor(final_momentum=.65, start=1, saturate=250) decay = sgd.LinearDecayOverEpoch(start=1, saturate=500, decay_factor=.01) ''' win = window_flip.WindowAndFlipC01B(pad_randomized=8, window_shape=(32, 32), randomize=[trn], center=[tst]) ''' #Define experiment experiment = Train(dataset=trn, model=mdl,
def main( x ): l1_dim = x[0] l2_dim = x[1] learning_rate = x[2] momentum = x[3] l1_dropout = x[4] decay_factor = x[5] min_lr = 1e-7 # train = np.loadtxt( train_file, delimiter = ',' ) x_train = train[:,0:-1] y_train = train[:,-1] y_train.shape = ( y_train.shape[0], 1 ) # validation = np.loadtxt( validation_file, delimiter = ',' ) x_valid = validation[:,0:-1] y_valid = validation[:,-1] y_valid.shape = ( y_valid.shape[0], 1 ) # #input_space = VectorSpace( dim = x.shape[1] ) full = DenseDesignMatrix( X = x_train, y = y_train ) valid = DenseDesignMatrix( X = x_valid, y = y_valid ) l1 = mlp.RectifiedLinear( layer_name='l1', irange=.001, dim = l1_dim, # "Rather than using weight decay, we constrain the norms of the weight vectors" max_col_norm=1. ) l2 = mlp.RectifiedLinear( layer_name='l2', irange=.001, dim = l2_dim, max_col_norm=1. ) output = mlp.Linear( dim = 1, layer_name='y', irange=.0001 ) layers = [l1, l2, output] nvis = x_train.shape[1] mdl = mlp.MLP( layers, nvis = nvis ) # input_space = input_space #lr = .001 #epochs = 100 decay = sgd.ExponentialDecay( decay_factor = decay_factor, min_lr = min_lr ) trainer = sgd.SGD( learning_rate = learning_rate, batch_size=128, learning_rule=learning_rule.Momentum( momentum ), update_callbacks = [ decay ], # Remember, default dropout is .5 cost = Dropout( input_include_probs = {'l1': l1_dropout}, input_scales={'l1': 1.}), #termination_criterion = EpochCounter(epochs), termination_criterion = MonitorBased( channel_name = "valid_objective", prop_decrease = 0.001, # 0.1% of objective N = 10 ), # valid_objective is MSE monitoring_dataset = { 'train': full, 'valid': valid } ) watcher = best_params.MonitorBasedSaveBest( channel_name = 'valid_objective', save_path = output_model_file ) experiment = Train( dataset = full, model = mdl, algorithm = trainer, extensions = [ watcher ] ) experiment.main_loop() ### error = get_error_from_model( output_model_file ) print "*** error: {} ***".format( error ) return error
delimiter=',') # ------------------------------------------Simple ANN h0 = mlp.Sigmoid(layer_name="h0", dim=73, sparse_init=0) y0 = mlp.Softmax(n_classes=5, layer_name="y0", irange=0) layers = [h0, y0] nn = mlp.MLP(layers, nvis=train.X.shape[1]) algo = sgd.SGD(learning_rate=0.05, batch_size=100, monitoring_dataset=valid, termination_criterion=EpochCounter(100)) algo.setup(nn, train) save_best = best_params.MonitorBasedSaveBest(channel_name="objective", save_path='best_params.pkl') while True: algo.train(dataset=train) nn.monitor.report_epoch() nn.monitor() save_best.on_monitor(nn, train, algo) if not algo.continue_learning(nn): break # SoftPlus with Dropout h0 = mlp.Softplus(layer_name='h0', dim=60, sparse_init=0) h1 = mlp.Softplus(layer_name='h1', dim=60, sparse_init=0) y0 = mlp.Softmax(layer_name='y0', n_classes=5, irange=0) layers = [h0, h1, y0]
mdl = mlp.MLP(layers, input_space=in_space) lr = .0001 epochs = 100 trainer = sgd.SGD(learning_rate=lr, batch_size=128, learning_rule=learning_rule.Momentum(.5), # Remember, default dropout is .5 cost=Dropout(input_include_probs={'l1': .8}, input_scales={'l1': 1.}), termination_criterion=EpochCounter(epochs), monitoring_dataset={'train': full}) watcher = best_params.MonitorBasedSaveBest( channel_name='train_y_misclass', save_path='saved_clf.pkl') velocity = learning_rule.MomentumAdjustor(final_momentum=.6, start=1, saturate=250) decay = sgd.LinearDecayOverEpoch(start=1, saturate=250, decay_factor=lr*.05) experiment = Train(dataset=full, model=mdl, algorithm=trainer, extensions=[watcher, velocity, decay])
# learning rate start = 1 saturate = 50 decay_factor = .1 learning_rate_adjustor = sgd.LinearDecayOverEpoch(start, saturate, decay_factor) # create neural net ann = mlp.MLP(layers, nvis=ds_train.nr_inputs) # create Stochastic Gradient Descent trainer trainer = sgd.SGD(learning_rate=.05, batch_size=10, monitoring_dataset=ds_valid, termination_criterion=termination_criterion, learning_rule=momentum_rule) trainer.setup(ann, ds_train) # add monitor for saving the model with best score monitor_save_best = best_params.MonitorBasedSaveBest('output_misclass', '/tmp/best.pkl') # train neural net until the termination criterion is true while True: trainer.train(dataset=ds_train) ann.monitor.report_epoch() ann.monitor() monitor_save_best.on_monitor(ann, ds_valid, trainer) if not trainer.continue_learning(ann): break momentum_adjustor.on_monitor(ann, ds_valid, trainer) learning_rate_adjustor.on_monitor(ann, ds_valid, trainer) # load the best model ann = serial.load('/tmp/best.pkl')