def runSP(): ds = StockPrice() # create hidden layer with 2 nodes, init weights in range -0.1 to 0.1 and add # a bias with value 1 hidden_layer = mlp.Sigmoid(layer_name='hidden', dim=10000, irange=.1, init_bias=1.) # create Softmax output layer output_layer = mlp.Linear(layer_name='output', dim=1, irange=.1, init_bias=1.) # create Stochastic Gradient Descent trainer that runs for 400 epochs trainer = sgd.SGD(learning_rate=.005, batch_size=500, termination_criterion=EpochCounter(10)) layers = [hidden_layer, output_layer] # create neural net that takes two inputs ann = mlp.MLP(layers, nvis=1000) trainer.setup(ann, ds) # train neural net until the termination criterion is true while True: trainer.train(dataset=ds) ann.monitor.report_epoch() ann.monitor() if not trainer.continue_learning(ann): break #accuracy = Accuracy() acc = Accuracy() for i, predict in enumerate(ann.fprop(theano.shared(ds.valid[0], name='inputs')).eval()): print predict, ds.valid[1][i] acc.evaluatePN(predict[0], ds.valid[1][i][0]) acc.printResult()
def construct_dbn_from_stack(stack): # some settings irange = 0.05 layers = [] for ii, layer in enumerate(stack.layers()): lr_scale = 0.25 if ii == 0 else 0.25 layers.append( mlp.Sigmoid(dim=layer.nhid, layer_name='h' + str(ii), irange=irange, W_lr_scale=lr_scale, max_col_norm=2.)) # softmax layer at then end for classification layers.append( mlp.Softmax(n_classes=9, layer_name='y', irange=irange, W_lr_scale=0.25)) dbn = mlp.MLP(layers=layers, nvis=stack.layers()[0].get_input_space().dim) # copy weigths to DBN for ii, layer in enumerate(stack.layers()): dbn.layers[ii].set_weights(layer.get_weights()) dbn.layers[ii].set_biases(layer.hidbias.get_value(borrow=False)) return dbn
def _create_layer(self, name, layer, irange): if isinstance(layer, Convolution): return self._create_convolution_layer(name, layer, irange) if layer.type == "Rectifier": self._check_layer(layer, ['units']) return mlp.RectifiedLinear(layer_name=name, dim=layer.units, irange=irange) if layer.type == "Sigmoid": self._check_layer(layer, ['units']) return mlp.Sigmoid(layer_name=name, dim=layer.units, irange=irange) if layer.type == "Tanh": self._check_layer(layer, ['units']) return mlp.Tanh(layer_name=name, dim=layer.units, irange=irange) if layer.type == "Maxout": self._check_layer(layer, ['units', 'pieces']) return maxout.Maxout(layer_name=name, num_units=layer.units, num_pieces=layer.pieces, irange=irange) if layer.type == "Linear": self._check_layer(layer, ['units']) return mlp.Linear(layer_name=layer.name, dim=layer.units, irange=irange) if layer.type == "Gaussian": self._check_layer(layer, ['units']) return mlp.LinearGaussian(layer_name=layer.name, init_beta=0.1, min_beta=0.001, max_beta=1000, beta_lr_scale=None, dim=layer.units, irange=irange) if layer.type == "Softmax": self._check_layer(layer, ['units']) return mlp.Softmax(layer_name=layer.name, n_classes=layer.units, irange=irange)
def __init__(self, data): self.N = 5 * 5 self.predictionLength = 2 # create hidden layer with 2 nodes, init weights in range -0.1 to 0.1 and add # a bias with value 1 hidden_layer = mlp.Sigmoid(layer_name='hidden', dim=25, irange=.1, init_bias=1.) # create Linear output layer output_layer = mlp.Linear(1, 'output', irange=.1, init_bias=1.) # create Stochastic Gradient Descent trainer that runs for 400 epochs trainer = sgd.SGD(learning_rate=.005, batch_size=10, termination_criterion=EpochCounter(100)) layers = [hidden_layer, output_layer] # create neural net that takes two inputs nn = mlp.MLP(layers, nvis=self.N) NeuralNetwork.__init__(self, data, nn, trainer)
def runXOR(): ds = XOR() hidden_layer = mlp.Sigmoid(layer_name='hidden', dim=10, irange=.1, init_bias=1.) output_layer = mlp.Linear(layer_name='output', dim=1, irange=.1, init_bias=1.) trainer = sgd.SGD(learning_rate=.05, batch_size=1, termination_criterion=EpochCounter(1000)) layers = [hidden_layer, output_layer] # create neural net that takes two inputs ann = mlp.MLP(layers, nvis=4) trainer.setup(ann, ds) # train neural net until the termination criterion is true while True: trainer.train(dataset=ds) #ann.monitor.report_epoch() #ann.monitor() if not trainer.continue_learning(ann): break inputs= np.array([[0, 0, 0, 1]]) print ann.fprop(theano.shared(inputs, name='inputs')).eval() inputs = np.array([[0, 1, 0, 1]]) print ann.fprop(theano.shared(inputs, name='inputs')).eval() inputs = np.array([[1, 1, 1, 1]]) print ann.fprop(theano.shared(inputs, name='inputs')).eval() inputs = np.array([[1, 1, 0, 0]]) print ann.fprop(theano.shared(inputs, name='inputs')).eval()
def init_train(): # Initialize train object. idpath = os.path.splitext( os.path.abspath(__file__))[0] # ID for output files. save_path = idpath + '.pkl' # Dataset #seed = 42 benchmark = 1 #derived_feat, nvis = False, 21 derived_feat, nvis = True, 28 #derived_feat, nvis = 'only', 7 dataset_train = pylearn2.datasets.physics.PHYSICS( which_set='train', benchmark=benchmark, derived_feat=derived_feat) dataset_train_monitor = pylearn2.datasets.physics.PHYSICS( which_set='train', benchmark=benchmark, derived_feat=derived_feat, start=0, stop=100000) dataset_valid = pylearn2.datasets.physics.PHYSICS( which_set='valid', benchmark=benchmark, derived_feat=derived_feat) dataset_test = pylearn2.datasets.physics.PHYSICS(which_set='test', benchmark=benchmark, derived_feat=derived_feat) # Parameters momentum_saturate = 200 # Model model = pylearn2.models.mlp.MLP( layers=[ mlp.Tanh(layer_name='h0', dim=300, istdev=.1), #istdev=.05 for any intermediates mlp.Sigmoid(layer_name='y', dim=1, istdev=.001) ], nvis=nvis) # Algorithm algorithm = pylearn2.training_algorithms.sgd.SGD( batch_size=100, # If changed, change learning rate! learning_rate= .05, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize. init_momentum=.9, monitoring_dataset={ 'train': dataset_train_monitor, 'valid': dataset_valid, 'test': dataset_test }, termination_criterion=pylearn2.termination_criteria.Or(criteria=[ pylearn2.termination_criteria.MonitorBased( channel_name="valid_objective", prop_decrease=0.00001, N=10), pylearn2.termination_criteria.EpochCounter( max_epochs=momentum_saturate) ]), cost=pylearn2.costs.cost.SumOfCosts(costs=[ pylearn2.costs.mlp.Default(), pylearn2.costs.mlp.WeightDecay(coeffs=[.00001, .00001]) ]), update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay( decay_factor= 1.0000002, # Decreases by this factor every batch. (1/(1.000001^8000)^100 min_lr=.000001)) # Extensions extensions = [ #pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path) pylearn2.training_algorithms.sgd.MomentumAdjustor( start=0, saturate=momentum_saturate, final_momentum=.99 # Dropout=.5->.99 over 500 epochs. ) ] # Train train = pylearn2.train.Train(dataset=dataset_train, model=model, algorithm=algorithm, extensions=extensions, save_path=save_path, save_freq=100) return train
def __linit(self, X, y): if (self.verbose > 0): print "Lazy initialisation" layers = self.layers pylearn2mlp_layers = [] self.units_per_layer = [] #input layer units self.units_per_layer += [X.shape[1]] for layer in layers[:-1]: self.units_per_layer += [layer[1]] #Output layer units self.units_per_layer += [y.shape[1]] if (self.verbose > 0): print "Units per layer", str(self.units_per_layer) for i, layer in enumerate(layers[:-1]): fan_in = self.units_per_layer[i] + 1 fan_out = self.units_per_layer[i + 1] lim = np.sqrt(6) / (np.sqrt(fan_in + fan_out)) layer_name = "Hidden_%i_%s" % (i, layer[0]) activate_type = layer[0] if activate_type == "RectifiedLinear": hidden_layer = mlp.RectifiedLinear(dim=layer[1], layer_name=layer_name, irange=lim) elif activate_type == "Sigmoid": hidden_layer = mlp.Sigmoid(dim=layer[1], layer_name=layer_name, irange=lim) elif activate_type == "Tanh": hidden_layer = mlp.Tanh(dim=layer[1], layer_name=layer_name, irange=lim) elif activate_type == "Maxout": hidden_layer = maxout.Maxout(num_units=layer[1], num_pieces=layer[2], layer_name=layer_name, irange=lim) else: raise NotImplementedError( "Layer of type %s are not implemented yet" % layer[0]) pylearn2mlp_layers += [hidden_layer] output_layer_info = layers[-1] output_layer_name = "Output_%s" % output_layer_info[0] fan_in = self.units_per_layer[-2] + 1 fan_out = self.units_per_layer[-1] lim = np.sqrt(6) / (np.sqrt(fan_in + fan_out)) if (output_layer_info[0] == "Linear"): output_layer = mlp.Linear(dim=self.units_per_layer[-1], layer_name=output_layer_name, irange=lim) pylearn2mlp_layers += [output_layer] self.mlp = mlp.MLP(pylearn2mlp_layers, nvis=self.units_per_layer[0]) self.ds = DenseDesignMatrix(X=X, y=y) self.trainer.setup(self.mlp, self.ds) inputs = self.mlp.get_input_space().make_theano_batch() self.f = theano.function([inputs], self.mlp.fprop(inputs))
def train(d): print 'Creating dataset' # load mnist here # X = d.train_X # y = d.train_Y # test_X = d.test_X # test_Y = d.test_Y # nb_classes = len(np.unique(y)) # train_y = convert_one_hot(y) # train_set = DenseDesignMatrix(X=X, y=y) train = DenseDesignMatrix(X=d.train_X, y=convert_one_hot(d.train_Y)) valid = DenseDesignMatrix(X=d.valid_X, y=convert_one_hot(d.valid_Y)) test = DenseDesignMatrix(X=d.test_X, y=convert_one_hot(d.test_Y)) print 'Setting up' batch_size = 1000 conv = mlp.ConvRectifiedLinear( layer_name='c0', output_channels=20, irange=.05, kernel_shape=[5, 5], pool_shape=[4, 4], pool_stride=[2, 2], # W_lr_scale=0.25, max_kernel_norm=1.9365) mout = MaxoutConvC01B(layer_name='m0', num_pieces=4, num_channels=96, irange=.05, kernel_shape=[5, 5], pool_shape=[4, 4], pool_stride=[2, 2], W_lr_scale=0.25, max_kernel_norm=1.9365) mout2 = MaxoutConvC01B(layer_name='m1', num_pieces=4, num_channels=96, irange=.05, kernel_shape=[5, 5], pool_shape=[4, 4], pool_stride=[2, 2], W_lr_scale=0.25, max_kernel_norm=1.9365) sigmoid = mlp.Sigmoid( layer_name='Sigmoid', dim=500, sparse_init=15, ) smax = mlp.Softmax(layer_name='y', n_classes=10, irange=0.) in_space = Conv2DSpace(shape=[28, 28], num_channels=1, axes=['c', 0, 1, 'b']) net = mlp.MLP( layers=[mout, mout2, smax], input_space=in_space, # nvis=784, ) trainer = bgd.BGD(batch_size=batch_size, line_search_mode='exhaustive', conjugate=1, updates_per_batch=10, monitoring_dataset={ 'train': train, 'valid': valid, 'test': test }, termination_criterion=termination_criteria.MonitorBased( channel_name='valid_y_misclass')) trainer = sgd.SGD(learning_rate=0.15, cost=dropout.Dropout(), batch_size=batch_size, monitoring_dataset={ 'train': train, 'valid': valid, 'test': test }, termination_criterion=termination_criteria.MonitorBased( channel_name='valid_y_misclass')) trainer.setup(net, train) epoch = 0 while True: print 'Training...', epoch trainer.train(dataset=train) net.monitor() epoch += 1
def main(): training_data, validation_data, test_data, std_scale = load_training_data() kaggle_test_features = load_test_data(std_scale) ############### # pylearn2 ML hl1 = mlp.Sigmoid(layer_name='hl1', dim=200, irange=.1, init_bias=1.) hl2 = mlp.Sigmoid(layer_name='hl2', dim=100, irange=.1, init_bias=1.) # create Softmax output layer output_layer = mlp.Softmax(9, 'output', irange=.1) # create Stochastic Gradient Descent trainer that runs for 400 epochs trainer = sgd.SGD(learning_rate=.05, batch_size=300, learning_rule=learning_rule.Momentum(.5), termination_criterion=MonitorBased( channel_name='valid_objective', prop_decrease=0., N=10), monitoring_dataset={ 'valid': validation_data, 'train': training_data }) layers = [hl1, hl2, output_layer] # create neural net model = mlp.MLP(layers, nvis=93) watcher = best_params.MonitorBasedSaveBest( channel_name='valid_objective', save_path='pylearn2_results/pylearn2_test.pkl') velocity = learning_rule.MomentumAdjustor(final_momentum=.6, start=1, saturate=250) decay = sgd.LinearDecayOverEpoch(start=1, saturate=250, decay_factor=.01) ###################### experiment = Train(dataset=training_data, model=model, algorithm=trainer, extensions=[watcher, velocity, decay]) experiment.main_loop() #load best model and test ################ model = serial.load('pylearn2_results/pylearn2_test.pkl') # get an prediction of the accuracy from the test_data test_results = model.fprop(theano.shared(test_data[0], name='test_data')).eval() print test_results.shape loss = multiclass_log_loss(test_data[1], test_results) print 'Test multiclass log loss:', loss out_file = 'pylearn2_results/' + str(loss) + 'ann' #exp.save(out_file + '.pkl') #save the kaggle results results = model.fprop( theano.shared(kaggle_test_features, name='kaggle_test_data')).eval() save_results(out_file + '.csv', kaggle_test_features, results)
from pylearn2.training_algorithms.sgd import ExponentialDecay; from deepmath.intfactor_dataset import IFDataset; train_data=IFDataset(path="${PYLEARN2_DATA_PATH}/deepmath/factorize_data.pkl"); valid_data=IFDataset(path="${PYLEARN2_DATA_PATH}/deepmath/factorize_data.pkl", which_set="valid"); test_data=IFDataset(path="${PYLEARN2_DATA_PATH}/deepmath/factorize_data.pkl", which_set="test"); print "[MESSAGE] The datasets are loaded"; ### build model model=MLP(layers=[mlp.Sigmoid(layer_name="hidden_0", dim=300, istdev=0.1), mlp.Sigmoid(layer_name="hidden_1", dim=300, istdev=0.1), mlp.Sigmoid(layer_name="y", dim=13, istdev=0.01, monitor_style="bit_vector_class")], nvis=26); print "[MESSAGE] The model is built"; ### build algorithm algorithm=SGD(batch_size=100,
train = csv_dataset.CSVDataset("../data/train.csv", expect_labels=True, expect_headers=False, delimiter=',') valid = csv_dataset.CSVDataset("../data/valid.csv", expect_labels=True, expect_headers=False, delimiter=',') test = csv_dataset.CSVDataset("../data/test.csv", expect_labels=True, expect_headers=False, delimiter=',') # ------------------------------------------Simple ANN h0 = mlp.Sigmoid(layer_name="h0", dim=73, sparse_init=0) y0 = mlp.Softmax(n_classes=5, layer_name="y0", irange=0) layers = [h0, y0] nn = mlp.MLP(layers, nvis=train.X.shape[1]) algo = sgd.SGD(learning_rate=0.05, batch_size=100, monitoring_dataset=valid, termination_criterion=EpochCounter(100)) algo.setup(nn, train) save_best = best_params.MonitorBasedSaveBest(channel_name="objective", save_path='best_params.pkl') while True: algo.train(dataset=train) nn.monitor.report_epoch()
def init_train(args): # Interpret arguments. derived_feat, seed, nhid, width, lrinit, lrdecay, momentum_init, momentum_saturate, wdecay, dropout_include, = args derived_feat = derived_feat # string seed = int(seed) nhid = int(nhid) width = int(width) lrinit = float(lrinit) lrdecay = float(lrdecay) momentum_init = float(momentum_init) momentum_saturate = int(momentum_saturate) wdecay = float(wdecay) dropout_include = float( dropout_include) # dropout include probability in top layer. # Specify output files: log and saved model pkl. #idpath = os.path.splitext(os.path.abspath(__file__))[0] # ID for output files. idpath = '' idpath = idpath + '%s_%d_%d_%d_%0.5f_%0.9f_%0.2f_%d_%f_%0.1f' % ( derived_feat, seed, nhid, width, lrinit, lrdecay, momentum_init, momentum_saturate, wdecay, dropout_include) save_path = idpath + '.pkl' logfile = idpath + '.log' print 'Using=%s' % theano.config.device # Can use gpus. print 'Writing to %s' % logfile print 'Writing to %s' % save_path sys.stdout = open(logfile, 'w') # Dataset benchmark = 1 dataset_train = pylearn2.datasets.physics.PHYSICS( which_set='train', benchmark=benchmark, derived_feat=derived_feat) #dataset_train = pylearn2.datasets.physics.PHYSICS(which_set='train', benchmark=benchmark, derived_feat=derived_feat, start=0, stop=2600000) # Smaller set for choosing hyperparameters. dataset_train_monitor = pylearn2.datasets.physics.PHYSICS( which_set='train', benchmark=benchmark, derived_feat=derived_feat, start=0, stop=100000) dataset_valid = pylearn2.datasets.physics.PHYSICS( which_set='valid', benchmark=benchmark, derived_feat=derived_feat) dataset_test = pylearn2.datasets.physics.PHYSICS(which_set='test', benchmark=benchmark, derived_feat=derived_feat) # Model nvis = dataset_train.X.shape[1] istdev = 1.0 / np.sqrt(width) layers = [] for i in range(nhid): # Hidden layer i layer = mlp.Tanh( layer_name='h%d' % i, dim=width, istdev=(istdev if i > 0 else 0.1), # First layer should have higher stdev. ) layers.append(layer) #layers.append(mlp.Sigmoid(layer_name='y', dim=1, istdev=istdev/100.0)) layers.append(mlp.Sigmoid(layer_name='y', dim=1, istdev=0.001)) model = pylearn2.models.mlp.MLP(layers, nvis=nvis, seed=seed) # Cost cost = pylearn2.costs.mlp.Default() # Default cost. if dropout_include != 1.0: # Use dropout cost if specified. cost = pylearn2.costs.mlp.dropout.Dropout( input_include_probs={'y': dropout_include}, input_scales={'y': 1.0 / dropout_include}, default_input_include_prob=1.0, default_input_scale=1.0) if wdecay != 0.0: # Add weight decay term if specified. cost2 = pylearn2.costs.mlp.WeightDecay( coeffs=[wdecay] * (nhid + 1)) # wdecay if specified. cost = pylearn2.costs.cost.SumOfCosts(costs=[cost, cost2]) # Algorithm algorithm = pylearn2.training_algorithms.sgd.SGD( batch_size=100, # If changed, change learning rate! learning_rate= lrinit, #.05, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize. learning_rule=pylearn2.training_algorithms.learning_rule.Momentum( init_momentum=momentum_init, ), monitoring_dataset={ 'train': dataset_train_monitor, 'valid': dataset_valid, 'test': dataset_test }, termination_criterion=pylearn2.termination_criteria.Or(criteria=[ pylearn2.termination_criteria.MonitorBased( channel_name="valid_y_kl", prop_decrease=0.00001, N=10), pylearn2.termination_criteria.EpochCounter( max_epochs=momentum_saturate) ]), cost=cost, update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay( decay_factor= lrdecay, #1.0000002 # Decreases by this factor every batch. (1/(1.000001^8000)^100 min_lr=.000001)) # Extensions extensions = [ #pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path) pylearn2.training_algorithms.learning_rule.MomentumAdjustor( start=0, saturate=momentum_saturate, # 200,500 final_momentum=0.99, # Dropout=.5->.99 over 500 epochs. ) ] # Train train = pylearn2.train.Train(dataset=dataset_train, model=model, algorithm=algorithm, extensions=extensions, save_path=save_path, save_freq=100) return train
def train(d=None): train_X = np.array(d.train_X) train_y = np.array(d.train_Y) valid_X = np.array(d.valid_X) valid_y = np.array(d.valid_Y) test_X = np.array(d.test_X) test_y = np.array(d.test_Y) nb_classes = len(np.unique(train_y)) train_y = convert_one_hot(train_y) valid_y = convert_one_hot(valid_y) # train_set = RotationalDDM(X=train_X, y=train_y) train_set = DenseDesignMatrix(X=train_X, y=train_y) valid_set = DenseDesignMatrix(X=valid_X, y=valid_y) print 'Setting up' batch_size = 100 c0 = mlp.ConvRectifiedLinear( layer_name='c0', output_channels=64, irange=.05, kernel_shape=[5, 5], pool_shape=[4, 4], pool_stride=[2, 2], # W_lr_scale=0.25, max_kernel_norm=1.9365) c1 = mlp.ConvRectifiedLinear( layer_name='c1', output_channels=64, irange=.05, kernel_shape=[5, 5], pool_shape=[4, 4], pool_stride=[2, 2], # W_lr_scale=0.25, max_kernel_norm=1.9365) c2 = mlp.ConvRectifiedLinear( layer_name='c2', output_channels=64, irange=.05, kernel_shape=[5, 5], pool_shape=[4, 4], pool_stride=[5, 4], W_lr_scale=0.25, # max_kernel_norm=1.9365 ) sp0 = mlp.SoftmaxPool( detector_layer_dim=16, layer_name='sp0', pool_size=4, sparse_init=512, ) sp1 = mlp.SoftmaxPool( detector_layer_dim=16, layer_name='sp1', pool_size=4, sparse_init=512, ) r0 = mlp.RectifiedLinear( layer_name='r0', dim=512, sparse_init=512, ) r1 = mlp.RectifiedLinear( layer_name='r1', dim=512, sparse_init=512, ) s0 = mlp.Sigmoid( layer_name='s0', dim=500, # max_col_norm=1.9365, sparse_init=15, ) out = mlp.Softmax( n_classes=nb_classes, layer_name='output', irange=.0, # max_col_norm=1.9365, # sparse_init=nb_classes, ) epochs = EpochCounter(100) layers = [s0, out] decay_coeffs = [.00005, .00005, .00005] in_space = Conv2DSpace( shape=[d.size, d.size], num_channels=1, ) vec_space = VectorSpace(d.size**2) nn = mlp.MLP( layers=layers, # input_space=in_space, nvis=d.size**2, # batch_size=batch_size, ) trainer = sgd.SGD( learning_rate=0.01, # cost=SumOfCosts(costs=[ # dropout.Dropout(), # MethodCost(method='cost_from_X'), # WeightDecay(decay_coeffs), # ]), # cost=MethodCost(method='cost_from_X'), batch_size=batch_size, # train_iteration_mode='even_shuffled_sequential', termination_criterion=epochs, # learning_rule=learning_rule.Momentum(init_momentum=0.5), ) trainer = bgd.BGD( batch_size=10000, line_search_mode='exhaustive', conjugate=1, updates_per_batch=10, termination_criterion=epochs, ) lr_adjustor = LinearDecayOverEpoch( start=1, saturate=10, decay_factor=.1, ) momentum_adjustor = learning_rule.MomentumAdjustor( final_momentum=.99, start=1, saturate=10, ) trainer.setup(nn, train_set) print 'Learning' test_X = vec_space.np_format_as(test_X, nn.get_input_space()) train_X = vec_space.np_format_as(train_X, nn.get_input_space()) i = 0 X = nn.get_input_space().make_theano_batch() Y = nn.fprop(X) predict = theano.function([X], Y) best = -40 best_iter = -1 while trainer.continue_learning(nn): print '--------------' print 'Training Epoch ' + str(i) trainer.train(dataset=train_set) nn.monitor() print 'Evaluating...' predictions = convert_categorical(predict(train_X[:2000])) score = accuracy_score(convert_categorical(train_y[:2000]), predictions) print 'Score on train: ' + str(score) predictions = convert_categorical(predict(test_X)) score = accuracy_score(test_y, predictions) print 'Score on test: ' + str(score) best, best_iter = (best, best_iter) if best > score else (score, i) print 'Current best: ' + str(best) + ' at iter ' + str(best_iter) print classification_report(test_y, predictions) print 'Adjusting parameters...' # momentum_adjustor.on_monitor(nn, valid_set, trainer) # lr_adjustor.on_monitor(nn, valid_set, trainer) i += 1 print ' '
import theano from pylearn2.models import mlp from pylearn2.training_algorithms import sgd from pylearn2.termination_criteria import EpochCounter from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix from csv_data import CSVData import numpy as np class MLPData(DenseDesignMatrix): def __init__(self, X, y): super(MLPData, self).__init__(X=X, y=y.astype(int), y_labels=2) threshold = 0.95 hidden_layer = mlp.Sigmoid(layer_name='h0', dim=10, sparse_init=10) output_layer = mlp.Softmax(layer_name='y', n_classes=2, irange=0.05) layers = [hidden_layer, output_layer] neural_net = mlp.MLP(layers, nvis=10) trainer = sgd.SGD(batch_size=5, learning_rate=.1, termination_criterion=EpochCounter(100)) first = True learning = True correct = 0 incorrect = 0 total = 0 data = CSVData("results2.csv") while True: X, y = data.get_data()
y = [] for a, b in X: if a + b == 1: y.append([0, 1]) else: y.append([1, 0]) X = np.array(X) y = np.array(y) super(XOR, self).__init__(X=X, y=y) # create XOR dataset ds = XOR() # create hidden layer with 2 nodes, init weights in range -0.1 to 0.1 and add # a bias with value 1 hidden_layer = mlp.Sigmoid(layer_name='hidden', dim=2, irange=.1, init_bias=1.) # create Softmax output layer output_layer = mlp.Softmax(2, 'output', irange=.1) # create Stochastic Gradient Descent trainer that runs for 400 epochs trainer = sgd.SGD(learning_rate=.05, batch_size=10, termination_criterion=EpochCounter(400)) layers = [hidden_layer, output_layer] # create neural net that takes two inputs ann = mlp.MLP(layers, nvis=2) trainer.setup(ann, ds) # train neural net until the termination criterion is true while True: trainer.train(dataset=ds) ann.monitor.report_epoch() ann.monitor()
return itertools.izip_longest(self.X, self.y) # create datasets ds_train = the_data() ds_train, ds_valid = ds_train.split(0.7) ds_valid, ds_test = ds_valid.split(0.7) ##################################### #Define Model ##################################### # create sigmoid hidden layer with 20 nodes, init weights in range -0.05 to 0.05 and add # a bias with value 1 hidden_layer = mlp.Sigmoid(layer_name='h0', dim=1, irange=.05, init_bias=1.) # softmax output layer output_layer = mlp.Softmax(2, 'softmax', irange=.05) layers = [hidden_layer, output_layer] # create neural net ann = mlp.MLP(layers, nvis=ds_train.nr_inputs) ##################################### #Define Training ##################################### #L1 Weight Decay L1_cost = PL.costs.cost.SumOfCosts([PL.costs.cost.MethodCost(method='cost_from_X'), PL.costs.mlp.L1WeightDecay(coeffs=[0.1, 0.01])]) # momentum
def init_train(): # Initialize train object. idpath = os.path.splitext( os.path.abspath(__file__))[0] # ID for output files. save_path = idpath + '.pkl' # Dataset #seed = 42 nvis = 32 dataset_train = pylearn2.datasets.improver2013.IMPROVER2013( which_set='A_human_phospho_from_rat_phospho_replicate') # Parameters momentum_saturate = 500 # Model model = pylearn2.models.mlp.MLP(layers=[ mlp.Sigmoid(layer_name='h0', dim=1000, istdev=.01), mlp.Sigmoid(layer_name='y', dim=32, istdev=.001) ], nvis=nvis) # Algorithm algorithm = pylearn2.training_algorithms.sgd.SGD( batch_size=1000, # If changed, change learning rate! learning_rate= .1, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize. init_momentum=.5, monitoring_dataset={ 'train': dataset_train, }, termination_criterion=pylearn2.termination_criteria.EpochCounter( max_epochs=6000), #termination_criterion=pylearn2.termination_criteria.Or(criteria=[ # pylearn2.termination_criteria.MonitorBased( # channel_name="valid_objective", # prop_decrease=0.00001, # N=40), # pylearn2.termination_criteria.EpochCounter( # max_epochs=momentum_saturate) # ]), #cost=pylearn2.costs.cost.SumOfCosts( # costs=[pylearn2.costs.mlp.Default() # ] #), #cost = pylearn2.costs.mlp.dropout.Dropout( # input_include_probs={'h0':1., 'h1':1., 'h2':1., 'h3':1., 'y':0.5}, # input_scales={ 'h0': 1., 'h1':1., 'h2':1., 'h3':1., 'y':2.}), cost=pylearn2.costs.mlp.addgauss.AddGauss(input_noise_stdev={ 'h0': 0.2, 'y': 0.2 }), update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay( decay_factor= 1.000004, # Decreases by this factor every batch. (1/(1.000001^8000)^100 min_lr=.000001)) # Extensions extensions = [ #pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path) pylearn2.training_algorithms.sgd.MomentumAdjustor( start=0, saturate=momentum_saturate, final_momentum=.99 # Dropout=.5->.99 over 500 epochs. ) ] # Train train = pylearn2.train.Train(dataset=dataset_train, model=model, algorithm=algorithm, extensions=extensions, save_path=save_path, save_freq=100) return train