Пример #1
0
def cnn_run_dropout_maxout(data_path, num_rows, num_cols, num_channels,
                           input_path, pred_path):
    t = time.time()
    sub_window = gen_center_sub_window(76, num_cols)
    trn = SarDataset(ds[0][0], ds[0][1], sub_window)
    vld = SarDataset(ds[1][0], ds[1][1], sub_window)
    tst = SarDataset(ds[2][0], ds[2][1], sub_window)
    print 'Take {}s to read data'.format(time.time() - t)
    t = time.time()
    batch_size = 100
    h1 = maxout.Maxout(layer_name='h2', num_units=1, num_pieces=100, irange=.1)
    hidden_layer = mlp.ConvRectifiedLinear(layer_name='h2',
                                           output_channels=8,
                                           irange=0.05,
                                           kernel_shape=[5, 5],
                                           pool_shape=[2, 2],
                                           pool_stride=[2, 2],
                                           max_kernel_norm=1.9365)
    hidden_layer2 = mlp.ConvRectifiedLinear(layer_name='h3',
                                            output_channels=8,
                                            irange=0.05,
                                            kernel_shape=[5, 5],
                                            pool_shape=[2, 2],
                                            pool_stride=[2, 2],
                                            max_kernel_norm=1.9365)
    #output_layer = mlp.Softplus(dim=1,layer_name='output',irange=0.1)
    output_layer = mlp.Linear(dim=1, layer_name='output', irange=0.05)
    trainer = sgd.SGD(learning_rate=0.001,
                      batch_size=100,
                      termination_criterion=EpochCounter(2000),
                      cost=dropout.Dropout(),
                      train_iteration_mode='even_shuffled_sequential',
                      monitor_iteration_mode='even_shuffled_sequential',
                      monitoring_dataset={
                          'test': tst,
                          'valid': vld,
                          'train': trn
                      })
    layers = [hidden_layer, hidden_layer2, output_layer]
    input_space = space.Conv2DSpace(shape=[num_rows, num_cols],
                                    num_channels=num_channels)

    ann = mlp.MLP(layers, input_space=input_space, batch_size=batch_size)
    watcher = best_params.MonitorBasedSaveBest(channel_name='valid_objective',
                                               save_path='sar_cnn_mlp.pkl')
    experiment = Train(dataset=trn,
                       model=ann,
                       algorithm=trainer,
                       extensions=[watcher])
    print 'Take {}s to compile code'.format(time.time() - t)
    t = time.time()
    experiment.main_loop()
    print 'Training time: {}s'.format(time.time() - t)
    serial.save('cnn_hhv_{0}_{1}.pkl'.format(num_rows, num_cols),
                ann,
                on_overwrite='backup')

    #read hh and hv into a 3D numpy
    image = read_hhv(input_path)
    return ann, sar_predict(ann, image, pred_path)
Пример #2
0
# termination criterion that stops after 50 epochs without
# any increase in misclassification on the validation set
termination_criterion = MonitorBased(channel_name='objective', N=20, prop_decrease=0.0)
 
# create Stochastic Gradient Descent trainer 
trainer = sgd.SGD(learning_rate=.001,
                    batch_size=10,
                    monitoring_dataset=ds_valid, 
                    termination_criterion=termination_criterion, 
                    cost=L1_cost)
#learning_rule=momentum_rule,
trainer.setup(ann, ds_train) 

# add monitor for saving the model with best score
monitor_save_best = best_params.MonitorBasedSaveBest('objective','./tmp/best.pkl')
 

#####################################
#Train model
####################################

# train neural net until the termination criterion is true
while True:
    trainer.train(dataset=ds_train)
    ann.monitor.report_epoch()
    ann.monitor()
    monitor_save_best.on_monitor(ann, ds_valid, trainer)
    if not trainer.continue_learning(ann):
        break
Пример #3
0
def main():
    training_data, validation_data, test_data, std_scale = load_training_data()
    kaggle_test_features = load_test_data(std_scale)

    ###############
    # pylearn2 ML
    hl1 = mlp.Sigmoid(layer_name='hl1', dim=200, irange=.1, init_bias=1.)
    hl2 = mlp.Sigmoid(layer_name='hl2', dim=100, irange=.1, init_bias=1.)

    # create Softmax output layer
    output_layer = mlp.Softmax(9, 'output', irange=.1)
    # create Stochastic Gradient Descent trainer that runs for 400 epochs
    trainer = sgd.SGD(learning_rate=.05,
                      batch_size=300,
                      learning_rule=learning_rule.Momentum(.5),
                      termination_criterion=MonitorBased(
                          channel_name='valid_objective',
                          prop_decrease=0.,
                          N=10),
                      monitoring_dataset={
                          'valid': validation_data,
                          'train': training_data
                      })

    layers = [hl1, hl2, output_layer]
    # create neural net
    model = mlp.MLP(layers, nvis=93)

    watcher = best_params.MonitorBasedSaveBest(
        channel_name='valid_objective',
        save_path='pylearn2_results/pylearn2_test.pkl')

    velocity = learning_rule.MomentumAdjustor(final_momentum=.6,
                                              start=1,
                                              saturate=250)
    decay = sgd.LinearDecayOverEpoch(start=1, saturate=250, decay_factor=.01)
    ######################

    experiment = Train(dataset=training_data,
                       model=model,
                       algorithm=trainer,
                       extensions=[watcher, velocity, decay])

    experiment.main_loop()

    #load best model and test
    ################
    model = serial.load('pylearn2_results/pylearn2_test.pkl')
    # get an prediction of the accuracy from the test_data
    test_results = model.fprop(theano.shared(test_data[0],
                                             name='test_data')).eval()

    print test_results.shape
    loss = multiclass_log_loss(test_data[1], test_results)

    print 'Test multiclass log loss:', loss

    out_file = 'pylearn2_results/' + str(loss) + 'ann'
    #exp.save(out_file + '.pkl')

    #save the kaggle results

    results = model.fprop(
        theano.shared(kaggle_test_features, name='kaggle_test_data')).eval()
    save_results(out_file + '.csv', kaggle_test_features, results)
Пример #4
0
lr = 0.1
epochs = 400
trainer = sgd.SGD(
    learning_rate=lr,
    batch_size=100,
    learning_rule=learning_rule.Momentum(.05),
    # Remember, default dropout is .5
    #cost=Dropout(input_include_probs={'l1': .5},
    #             input_scales={'l1': 1.}),
    termination_criterion=EpochCounter(epochs),
    monitoring_dataset={
        'train': ds,
        'valid': ds_test
    })

watcher = best_params.MonitorBasedSaveBest(channel_name='valid_roc_auc',
                                           save_path='saved_clf.pkl')

velocity = learning_rule.MomentumAdjustor(final_momentum=.9,
                                          start=1,
                                          saturate=250)

decay = sgd.LinearDecayOverEpoch(start=1, saturate=250, decay_factor=lr * .05)
rocauc = roc_auc.RocAucChannel()
experiment = Train(dataset=ds,
                   model=ann,
                   algorithm=trainer,
                   extensions=[watcher, velocity, decay, rocauc])

experiment.main_loop()
Пример #5
0
                          termination_criterion=EpochCounter(max_epochs=max_epochs),
                          monitoring_dataset={'train':X_train, 'valid':X_test},
                          )
        
        
        input_space = Conv2DSpace(shape=(central_window_shape, central_window_shape),
                    axes = axes,
                    num_channels = 1)
                    
        ann = mlp.MLP(layers, input_space=input_space)

        velocity = learning_rule.MomentumAdjustor(final_momentum=momentum_end,
                                          start=1,
                                          saturate=momentum_saturate)

        watcher = best_params.MonitorBasedSaveBest(channel_name='valid_y_nll',
                                                   save_path=save_path)

        decay = sgd.LinearDecayOverEpoch(start=1, saturate=decay_saturate, decay_factor=decay_factor)

        ra = RealtimeAugment(window_shape=[img_dim, img_dim], randomize=[X_train, X_test], 
                scale_diff=scale_diff, translation=translation, center_shape=center_shape, center=[X_train, X_test],
                preprocess=preprocess)
                
        train = Train(dataset=X_train, model=ann, algorithm=trainer,
                      extensions=[watcher, velocity, decay, ra])
                  
        train.main_loop()

    print "using model", save_path
    model = serial.load(save_path)
    
Пример #6
0
                  batch_size=128,
                  learning_rule=learning_rule.Momentum(.5),
                  # Remember, default dropout is .5
                  cost=Dropout(input_include_probs={'l1': .8},
                               input_scales={'l1': 1.}),
                  termination_criterion=EpochCounter(max_epochs=475),
                  monitoring_dataset={'valid': tst,
                                      'train': trn})

preprocessor = Pipeline([GlobalContrastNormalization(scale=55.), ZCA()])
trn.apply_preprocessor(preprocessor=preprocessor, can_fit=True)
tst.apply_preprocessor(preprocessor=preprocessor, can_fit=False)
serial.save('kaggle_cifar10_preprocessor.pkl', preprocessor)

watcher = best_params.MonitorBasedSaveBest(
    channel_name='valid_y_misclass',
    save_path='kaggle_cifar10_maxout_zca.pkl')

velocity = learning_rule.MomentumAdjustor(final_momentum=.65,
                                          start=1,
                                          saturate=250)

decay = sgd.LinearDecayOverEpoch(start=1,
                                 saturate=500,
                                 decay_factor=.01)

win = window_flip.WindowAndFlipC01B(pad_randomized=8,
                                    window_shape=(32, 32),
                                    randomize=[trn],
                                    center=[tst])
Пример #7
0
def cnn_train_tranformer(train_path,
                         test_path,
                         valid_path,
                         save_path,
                         predict_path,
                         num_rows=28,
                         num_cols=28,
                         num_channels=2,
                         batch_size=128,
                         output_channels=[64, 64],
                         kernel_shape=[[12, 12], [5, 5]],
                         pool_shape=[[4, 4], [2, 2]],
                         pool_stride=[[2, 2], [2, 2]],
                         irange=[0.05, 0.05, 0.05],
                         max_kernel_norm=[1.9365, 1.9365],
                         learning_rate=0.001,
                         init_momentum=0.9,
                         weight_decay=[0.0002, 0.0002, 0.0002],
                         n_epoch=1000,
                         image_path=''):

    ds = load_data_transformed(train_path, num_cols, batch_size)
    ds = (np.transpose(ds[0], axes=[0, 3, 1, 2]), ds[1])
    trn = SarDataset(np.array(ds[0]), ds[1])
    ds = load_data_transformed(valid_path, num_cols, batch_size)
    ds = (np.transpose(ds[0], axes=[0, 3, 1, 2]), ds[1])
    vld = SarDataset(np.array(ds[0]), ds[1])
    ds = load_data_transformed(test_path, num_cols, batch_size)
    ds = (np.transpose(ds[0], axes=[0, 3, 1, 2]), ds[1])
    tst = SarDataset(np.array(ds[0]), ds[1])
    #setup the network
    #X = np.random.random([400000,2,41,41])
    #y = np.random.random([400000,1])
    #trn = SarDataset(X,y)
    #X = np.random.random([60000,2,41,41])
    #y = np.random.random([60000,1])
    #tst = SarDataset(X,y)
    #X = np.random.random([60000,2,41,41])
    #y = np.random.random([60000,1])
    #vld = SarDataset(X,y)
    t = time.time()
    layers = []
    for i in range(len(output_channels)):
        layer_name = 'h{}'.format(i + 1)
        convlayer = mlp.ConvRectifiedLinear(layer_name=layer_name,
                                            output_channels=output_channels[i],
                                            irange=irange[i],
                                            kernel_shape=kernel_shape[i],
                                            pool_shape=pool_shape[i],
                                            pool_stride=pool_stride[i],
                                            max_kernel_norm=max_kernel_norm[i])
        layers.append(convlayer)

    output_mlp = mlp.Linear(dim=1, layer_name='output', irange=irange[-1])
    #output_mlp = mlp.linear_mlp_bayesian_cost(dim=1,layer_name='output',irange=irange[-1])
    layers.append(output_mlp)

    trainer = sgd.SGD(
        learning_rate=learning_rate,
        batch_size=batch_size,
        termination_criterion=EpochCounter(n_epoch),
        #termination_criterion = termination_criteria.And([termination_criteria.MonitorBased(channel_name = 'train_objective', prop_decrease=0.01,N=10),EpochCounter(n_epoch)]),
        #cost = dropout.Dropout(),
        cost=cost.SumOfCosts(
            [cost.MethodCost('cost_from_X'),
             WeightDecay(weight_decay)]),
        init_momentum=init_momentum,
        train_iteration_mode='even_shuffled_sequential',
        monitor_iteration_mode='even_shuffled_sequential',
        monitoring_dataset={
            'test': tst,
            'valid': vld,
            'train': trn
        })

    input_space = space.Conv2DSpace(shape=[num_rows, num_cols],
                                    num_channels=num_channels)
    #ann = mlp.MLP(layers,input_space=input_space,batch_size=batch_size)
    watcher = best_params.MonitorBasedSaveBest(channel_name='valid_objective',
                                               save_path=predict_path +
                                               save_path)
    #flip = window_flip.WindowAndFlip((num_rows,num_cols),randomize=[tst,vld,trn])
    experiment = Train(dataset=trn,
                       model=ann,
                       algorithm=trainer,
                       extensions=[watcher])
    print 'Take {}s to compile code'.format(time.time() - t)

    #train the network
    t = time.time()
    experiment.main_loop()
    print 'Training time: {}h'.format((time.time() - t) / 3600)
    utils.sms_notice('Training time:{}'.format((time.time() - t) / 3600))

    return ann
Пример #8
0
def cnn_train(
    train_path,
    test_path,
    valid_path,
    save_path,
    predict_path,
    image_path,
    num_rows=28,
    num_cols=28,
    num_channels=2,
    batch_size=128,
    output_channels=[64, 64],
    kernel_shape=[[12, 12], [5, 5]],
    pool_shape=[[4, 4], [2, 2]],
    pool_stride=[[2, 2], [2, 2]],
    irange=[0.05, 0.05, 0.05],
    max_kernel_norm=[1.9365, 1.9365],
    learning_rate=0.001,
    init_momentum=0.9,
    weight_decay=[0.0002, 0.0002, 0.0002],
    n_epoch=1000,
):
    #load data
    #t = time.time()
    ds = load_data(valid_path, num_rows, num_cols, num_channels)
    vld = SarDataset(np.array(ds[0]), ds[1])
    ds = load_data(train_path, num_rows, num_cols, num_channels)
    trn = SarDataset(np.array(ds[0]), ds[1])
    ds = load_data(test_path, num_rows, num_cols, num_channels)
    tst = SarDataset(np.array(ds[0]), ds[1])
    #load balanced data
    #ds = load_data_balance_under_sample(train_path, num_rows,num_cols, num_channels)
    #trn = SarDataset(np.array(ds[0]),ds[1])
    #ds = load_data_balance(valid_path, num_rows,num_cols, num_channels)
    #vld = SarDataset(np.array(ds[0]),ds[1])
    #ds = load_data_balance(test_path, num_rows,num_cols, num_channels)
    #tst = SarDataset(np.array(ds[0]),ds[1])
    #print 'Take {}s to read data'.format( time.time()-t)
    #use gaussian convlution on the origional image to see if it can concentrate in the center
    #trn,tst,vld = load_data_lidar()

    #mytransformer = transformer.TransformationPipeline(input_space=space.Conv2DSpace(shape=[num_rows,num_cols],num_channels=num_channels),transformations=[transformer.Rotation(),transformer.Flipping()])
    #trn = contestTransformerDataset.TransformerDataset(trn,mytransformer,space_preserving=True)
    #tst = contestTransformerDataset.TransformerDataset(tst,mytransformer,space_preserving=True)
    #vld = contestTransformerDataset.TransformerDataset(vld,mytransformer,space_preserving=True)

    #trn = transformer_dataset.TransformerDataset(trn,mytransformer,space_preserving=True)
    #tst = transformer_dataset.TransformerDataset(tst,mytransformer,space_preserving=True)
    #vld = transformer_dataset.TransformerDataset(vld,mytransformer,space_preserving=True)

    #setup the network
    t = time.time()
    layers = []
    for i in range(len(output_channels)):
        layer_name = 'h{}'.format(i + 1)
        convlayer = mlp.ConvRectifiedLinear(layer_name=layer_name,
                                            output_channels=output_channels[i],
                                            irange=irange[i],
                                            kernel_shape=kernel_shape[i],
                                            pool_shape=pool_shape[i],
                                            pool_stride=pool_stride[i],
                                            max_kernel_norm=max_kernel_norm[i])
        layers.append(convlayer)

    output_mlp = mlp.Linear(dim=1,
                            layer_name='output',
                            irange=irange[-1],
                            use_abs_loss=True)
    #output_mlp = mlp.linear_mlp_ace(dim=1,layer_name='output',irange=irange[-1])
    layers.append(output_mlp)

    #ann = cPickle.load(open('../output/train_with_2010_2l_40_64/original_500/f/f0.pkl'))
    #layers = []
    #for layer in ann.layers:
    #    layer.set_mlp_force(None)
    #    layers.append(layer)

    trainer = sgd.SGD(
        learning_rate=learning_rate,
        batch_size=batch_size,
        termination_criterion=EpochCounter(n_epoch),
        #termination_criterion = termination_criteria.And([termination_criteria.MonitorBased(channel_name = 'train_objective', prop_decrease=0.01,N=10),EpochCounter(n_epoch)]),
        #cost = dropout.Dropout(),
        cost=cost.SumOfCosts(
            [cost.MethodCost('cost_from_X'),
             WeightDecay(weight_decay)]),
        init_momentum=init_momentum,
        train_iteration_mode='even_shuffled_sequential',
        monitor_iteration_mode='even_shuffled_sequential',
        monitoring_dataset={
            'test': tst,
            'valid': vld,
            'train': trn
        })

    input_space = space.Conv2DSpace(shape=[num_rows, num_cols],
                                    num_channels=num_channels)
    #ann = mlp.MLP(layers,input_space=input_space,batch_size=batch_size)
    ann = serial.load(
        '../output/train_with_2010_2l_40_64/original_500/f/f0.pkl')
    ann = monitor.push_monitor(ann, 'stage_0')
    watcher = best_params.MonitorBasedSaveBest(channel_name='valid_objective',
                                               save_path=predict_path +
                                               save_path)
    flip = window_flip.WindowAndFlip((num_rows, num_cols),
                                     randomize=[tst, vld, trn])
    experiment = Train(dataset=trn,
                       model=ann,
                       algorithm=trainer,
                       extensions=[watcher, flip])
    print 'Take {}s to compile code'.format(time.time() - t)

    #train the network
    t = time.time()
    experiment.main_loop()
    print 'Training time: {}h'.format((time.time() - t) / 3600)
    utils.sms_notice('Training time:{}'.format((time.time() - t) / 3600))

    return ann
Пример #9
0
def supervisedLayerwisePRL(trainset, testset):
    '''
	The supervised layerwise training as used in the PRL Paper.
	
	Input
	------
	trainset : A path to an hdf5 file created through h5py.
	testset  : A path to an hdf5 file created through h5py.
	'''
    batch_size = 100

    # Both train and test h5py files are expected to have a 'topo_view' and 'y'
    # datasets side them corresponding to the 'b01c' data format as used in pylearn2
    # and 'y' equivalent to the one hot encoded labels
    trn = HDF5Dataset(filename=trainset,
                      topo_view='topo_view',
                      y='y',
                      load_all=False)
    tst = HDF5Dataset(filename=testset,
                      topo_view='topo_view',
                      y='y',
                      load_all=False)
    '''
	The 1st Convolution and Pooling Layers are added below.
	'''
    h1 = mlp.ConvRectifiedLinear(layer_name='h1',
                                 output_channels=64,
                                 irange=0.05,
                                 kernel_shape=[4, 4],
                                 pool_shape=[4, 4],
                                 pool_stride=[2, 2],
                                 max_kernel_norm=1.9365)

    fc = mlp.RectifiedLinear(layer_name='fc', dim=1500, irange=0.05)
    output = mlp.Softmax(layer_name='y',
                         n_classes=171,
                         irange=.005,
                         max_col_norm=1.9365)

    layers = [h1, fc, output]

    mdl = mlp.MLP(layers,
                  input_space=Conv2DSpace(shape=(70, 70), num_channels=1))

    trainer = sgd.SGD(
        learning_rate=0.002,
        batch_size=batch_size,
        learning_rule=learning_rule.RMSProp(),
        cost=SumOfCosts(
            costs=[Default(),
                   WeightDecay(coeffs=[0.0005, 0.0005, 0.0005])]),
        train_iteration_mode='shuffled_sequential',
        monitor_iteration_mode='sequential',
        termination_criterion=EpochCounter(max_epochs=15),
        monitoring_dataset={
            'test': tst,
            'valid': vld
        })

    watcher = best_params.MonitorBasedSaveBest(
        channel_name='valid_y_misclass',
        save_path='./Saved Models/conv_supervised_layerwise_best1.pkl')

    decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1)

    experiment = Train(
        dataset=trn,
        model=mdl,
        algorithm=trainer,
        extensions=[watcher, decay],
    )

    experiment.main_loop()

    del mdl
    mdl = serial.load('./Saved Models/conv_supervised_layerwise_best1.pkl')
    mdl = push_monitor(mdl, 'k')
    '''
	The 2nd Convolution and Pooling Layers are added below.
	'''
    h2 = mlp.ConvRectifiedLinear(layer_name='h2',
                                 output_channels=64,
                                 irange=0.05,
                                 kernel_shape=[4, 4],
                                 pool_shape=[4, 4],
                                 pool_stride=[2, 2],
                                 max_kernel_norm=1.9365)

    fc = mlp.RectifiedLinear(layer_name='fc', dim=1500, irange=0.05)
    output = mlp.Softmax(layer_name='y',
                         n_classes=171,
                         irange=.005,
                         max_col_norm=1.9365)

    del mdl.layers[-1]
    mdl.layer_names.remove('y')
    del mdl.layers[-1]
    mdl.layer_names.remove('fc')
    mdl.add_layers([h2, fc, output])

    trainer = sgd.SGD(learning_rate=0.002,
                      batch_size=batch_size,
                      learning_rule=learning_rule.RMSProp(),
                      cost=SumOfCosts(costs=[
                          Default(),
                          WeightDecay(coeffs=[0.0005, 0.0005, 0.0005, 0.0005])
                      ]),
                      train_iteration_mode='shuffled_sequential',
                      monitor_iteration_mode='sequential',
                      termination_criterion=EpochCounter(max_epochs=15),
                      monitoring_dataset={
                          'test': tst,
                          'valid': vld
                      })

    watcher = best_params.MonitorBasedSaveBest(
        channel_name='valid_y_misclass',
        save_path='./Saved Models/conv_supervised_layerwise_best2.pkl')

    decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1)

    experiment = Train(
        dataset=trn,
        model=mdl,
        algorithm=trainer,
        extensions=[watcher, decay],
    )

    experiment.main_loop()

    del mdl
    mdl = serial.load('./Saved Models/conv_supervised_layerwise_best2.pkl')
    mdl = push_monitor(mdl, 'l')
    '''
	The 3rd Convolution and Pooling Layers are added below.
	'''
    h3 = mlp.ConvRectifiedLinear(layer_name='h2',
                                 output_channels=64,
                                 irange=0.05,
                                 kernel_shape=[4, 4],
                                 pool_shape=[4, 4],
                                 pool_stride=[2, 2],
                                 max_kernel_norm=1.9365)

    fc = mlp.RectifiedLinear(layer_name='h3', dim=1500, irange=0.05)
    output = mlp.Softmax(layer_name='y',
                         n_classes=10,
                         irange=.005,
                         max_col_norm=1.9365)

    del mdl.layers[-1]
    mdl.layer_names.remove('y')
    del mdl.layers[-1]
    mdl.layer_names.remove('fc')
    mdl.add_layers([h3, output])

    trainer = sgd.SGD(
        learning_rate=.002,
        batch_size=batch_size,
        learning_rule=learning_rule.RMSProp(),
        cost=SumOfCosts(costs=[
            Default(),
            WeightDecay(coeffs=[0.0005, 0.0005, 0.0005, 0.0005, 0.0005])
        ]),
        train_iteration_mode='shuffled_sequential',
        monitor_iteration_mode='sequential',
        termination_criterion=EpochCounter(max_epochs=15),
        monitoring_dataset={
            'test': tst,
            'valid': vld
        })

    watcher = best_params.MonitorBasedSaveBest(
        channel_name='valid_y_misclass',
        save_path='./Saved Models/conv_supervised_layerwise_best3.pkl')

    decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1)

    experiment = Train(
        dataset=trn,
        model=mdl,
        algorithm=trainer,
        extensions=[watcher, decay],
    )

    experiment.main_loop()
Пример #10
0
    batch_size=32,
    learning_rule=learning_rule.Momentum(.5),
    # Remember, default dropout is .5
    cost=Dropout(input_include_probs={'h2': .8}, input_scales={'h2': 1.}),
    termination_criterion=EpochCounter(max_epochs=475),
    monitoring_dataset={
        'valid': tst,
        'train': trn
    })

preprocessor = Pipeline([GlobalContrastNormalization(scale=55.), ZCA()])
trn.apply_preprocessor(preprocessor=preprocessor, can_fit=True)
tst.apply_preprocessor(preprocessor=preprocessor, can_fit=False)
serial.save('deepcuts_preprocessor.pkl', preprocessor)

watcher = best_params.MonitorBasedSaveBest(channel_name='valid_y_misclass',
                                           save_path='deepcuts_maxout_zca.pkl')

velocity = learning_rule.MomentumAdjustor(final_momentum=.65,
                                          start=1,
                                          saturate=250)

decay = sgd.LinearDecayOverEpoch(start=1, saturate=500, decay_factor=.01)
'''
win = window_flip.WindowAndFlipC01B(pad_randomized=8,
                                    window_shape=(32, 32),
                                    randomize=[trn],
                                    center=[tst])
'''
#Define experiment
experiment = Train(dataset=trn,
                   model=mdl,
Пример #11
0
def main( x ):

	l1_dim = x[0]
	l2_dim = x[1]
	learning_rate = x[2]
	momentum = x[3]
	l1_dropout = x[4]
	decay_factor = x[5]
	
	min_lr = 1e-7

	#

	train = np.loadtxt( train_file, delimiter = ',' )
	x_train = train[:,0:-1]
	y_train = train[:,-1]
	y_train.shape = ( y_train.shape[0], 1 )

	# 

	validation = np.loadtxt( validation_file, delimiter = ',' )
	x_valid = validation[:,0:-1]
	y_valid = validation[:,-1]
	y_valid.shape = ( y_valid.shape[0], 1 )

	#

	#input_space = VectorSpace( dim = x.shape[1] )
	full = DenseDesignMatrix( X = x_train, y = y_train )
	valid = DenseDesignMatrix( X = x_valid, y = y_valid )

	l1 = mlp.RectifiedLinear( 
		layer_name='l1',
		irange=.001,
		dim = l1_dim,
		# "Rather than using weight decay, we constrain the norms of the weight vectors"
		max_col_norm=1.
	)

	l2 = mlp.RectifiedLinear(
		layer_name='l2',
		irange=.001,
		dim = l2_dim,
		max_col_norm=1.
	)

	output = mlp.Linear( dim = 1, layer_name='y', irange=.0001 )

	layers = [l1, l2, output]
	nvis = x_train.shape[1]

	mdl = mlp.MLP( layers, nvis = nvis )	# input_space = input_space

	#lr = .001
	#epochs = 100
	
	decay = sgd.ExponentialDecay( decay_factor = decay_factor, min_lr = min_lr )

	trainer = sgd.SGD(
		learning_rate = learning_rate,
		batch_size=128,
		learning_rule=learning_rule.Momentum( momentum ),
		
		update_callbacks = [ decay ],

		# Remember, default dropout is .5
		cost = Dropout( input_include_probs = {'l1': l1_dropout},
				   input_scales={'l1': 1.}),

		#termination_criterion = EpochCounter(epochs),
		termination_criterion = MonitorBased(
			channel_name = "valid_objective",
			prop_decrease = 0.001,				# 0.1% of objective
			N = 10	
		),

		# valid_objective is MSE

		monitoring_dataset = { 'train': full, 'valid': valid }
	)

	watcher = best_params.MonitorBasedSaveBest( channel_name = 'valid_objective', save_path = output_model_file )
	
	experiment = Train( dataset = full, model = mdl, algorithm = trainer, extensions = [ watcher ] )
	experiment.main_loop()

	###

	error = get_error_from_model( output_model_file )
	print "*** error: {} ***".format( error )
	return error
	
	
Пример #12
0
                              delimiter=',')

# ------------------------------------------Simple ANN

h0 = mlp.Sigmoid(layer_name="h0", dim=73, sparse_init=0)
y0 = mlp.Softmax(n_classes=5, layer_name="y0", irange=0)
layers = [h0, y0]

nn = mlp.MLP(layers, nvis=train.X.shape[1])
algo = sgd.SGD(learning_rate=0.05,
               batch_size=100,
               monitoring_dataset=valid,
               termination_criterion=EpochCounter(100))
algo.setup(nn, train)

save_best = best_params.MonitorBasedSaveBest(channel_name="objective",
                                             save_path='best_params.pkl')
while True:
    algo.train(dataset=train)
    nn.monitor.report_epoch()
    nn.monitor()
    save_best.on_monitor(nn, train, algo)
    if not algo.continue_learning(nn):
        break

# SoftPlus with Dropout

h0 = mlp.Softplus(layer_name='h0', dim=60, sparse_init=0)
h1 = mlp.Softplus(layer_name='h1', dim=60, sparse_init=0)
y0 = mlp.Softmax(layer_name='y0', n_classes=5, irange=0)
layers = [h0, h1, y0]
Пример #13
0
mdl = mlp.MLP(layers,
              input_space=in_space)

lr = .0001
epochs = 100
trainer = sgd.SGD(learning_rate=lr,
                  batch_size=128,
                  learning_rule=learning_rule.Momentum(.5),
                  # Remember, default dropout is .5
                  cost=Dropout(input_include_probs={'l1': .8},
                               input_scales={'l1': 1.}),
                  termination_criterion=EpochCounter(epochs),
                  monitoring_dataset={'train': full})

watcher = best_params.MonitorBasedSaveBest(
    channel_name='train_y_misclass',
    save_path='saved_clf.pkl')

velocity = learning_rule.MomentumAdjustor(final_momentum=.6,
                                          start=1,
                                          saturate=250)

decay = sgd.LinearDecayOverEpoch(start=1,
                                 saturate=250,
                                 decay_factor=lr*.05)

experiment = Train(dataset=full,
                   model=mdl,
                   algorithm=trainer,
                   extensions=[watcher, velocity, decay])
Пример #14
0
# learning rate
start = 1
saturate = 50
decay_factor = .1
learning_rate_adjustor = sgd.LinearDecayOverEpoch(start, saturate, decay_factor)

# create neural net
ann = mlp.MLP(layers, nvis=ds_train.nr_inputs)

# create Stochastic Gradient Descent trainer 
trainer = sgd.SGD(learning_rate=.05, batch_size=10, monitoring_dataset=ds_valid,
                  termination_criterion=termination_criterion, learning_rule=momentum_rule)
trainer.setup(ann, ds_train)

# add monitor for saving the model with best score
monitor_save_best = best_params.MonitorBasedSaveBest('output_misclass',
                                                     '/tmp/best.pkl')

# train neural net until the termination criterion is true
while True:
    trainer.train(dataset=ds_train)
    ann.monitor.report_epoch()
    ann.monitor()
    monitor_save_best.on_monitor(ann, ds_valid, trainer)
    if not trainer.continue_learning(ann):
        break
    momentum_adjustor.on_monitor(ann, ds_valid, trainer)
    learning_rate_adjustor.on_monitor(ann, ds_valid, trainer)

# load the best model
ann = serial.load('/tmp/best.pkl')