def get_init(features, initializer_class, nhid=5): initializer = initializer_class.build({}, graph=features.graph) if hasattr(initializer, "Input"): initializer.Input.resize(2) initializer.Input[0].connect(features.Output) nvis = features.Output.meta.shape[1] layer = Sigmoid(layer_name="a", irange=0, dim=nhid) # layer needs to be initialized by MLP first MLP(layers=[layer], nvis=nvis) initializer.init_layer(layer, nvis=nvis, nhid=nhid) weights = layer.get_weights() biases = layer.get_biases() return weights, biases
def test_kl(): """ Test whether function kl() has properly processed the input. """ init_mode = theano.config.compute_test_value theano.config.compute_test_value = 'raise' try: mlp = MLP(layers=[Sigmoid(dim=10, layer_name='Y', irange=0.1)], nvis=10) X = mlp.get_input_space().make_theano_batch() Y = mlp.get_output_space().make_theano_batch() X.tag.test_value = np.random.random( get_debug_values(X)[0].shape).astype(theano.config.floatX) Y_hat = mlp.fprop(X) # This call should not raise any error: ave = kl(Y, Y_hat, 1) # The following calls should raise ValueError exceptions: Y.tag.test_value[2][3] = 1.1 np.testing.assert_raises(ValueError, kl, Y, Y_hat, 1) Y.tag.test_value[2][3] = -0.1 np.testing.assert_raises(ValueError, kl, Y, Y_hat, 1) finally: theano.config.compute_test_value = init_mode
def test_convnet(): layers = [] dataset = get_dataset() input_space = Conv2DSpace(shape=[256, 256], num_channels=1) conv_layer = ConvRectifiedLinear(output_channels=12, irange=.005, layer_name="h0", kernel_shape=[88, 88], kernel_stride=[8, 8], pool_shape=[1, 1], pool_stride=[1, 1], max_kernel_norm=1.932) layers.append(conv_layer) maxout_layer = Maxout(layer_name="h1", irange=.005, num_units=600, num_pieces=4, max_col_norm=1.932) layers.append(maxout_layer) sigmoid_layer = Sigmoid(layer_name="y", dim=484, monitor_style="detection", irange=.005) layers.append(sigmoid_layer) model = MLP(batch_size=100, layers=layers, input_space=input_space) trainer = get_layer_trainer_sgd(model, dataset) trainer.main_loop()
def testLSF(self): init = LeastSquaresWeightInitializer.build({}, graph=Graph()) init.Input.resize(2) init.Input[0].setValue(self.X) init.Input[1].setValue(self.y) for k in (2, 3, 5, 7): layer = Sigmoid(layer_name="a", irange=0, dim=k) # layer needs to be initialized by MLP first MLP(layers=[layer], nvis=self.d) np.random.seed(123) init.init_layer(layer, nvis=self.d, nhid=k) weights = layer.get_weights() np.testing.assert_array_equal(weights.shape, (self.d, k)) assert np.abs(weights).sum() > 0
def get_layer_sigmoid(self, layer_id, layer_name): row = self.db.executeSQL( """ SELECT dim,irange,istdev,sparse_init,sparse_stdev,include_prob,init_bias, W_lr_scale,b_lr_scale,max_col_norm,max_row_norm FROM hps3.layer_sigmoid WHERE layer_id = %s """, (layer_id, ), self.db.FETCH_ONE) if not row or row is None: raise HPSData("No sigmoid layer for layer_id=" \ +str(layer_id)) (dim, irange, istdev, sparse_init, sparse_stdev, include_prob, init_bias, W_lr_scale, b_lr_scale, max_col_norm, max_row_norm) = row return Sigmoid(layer_name=layer_name, dim=dim, irange=irange, istdev=istdev, sparse_init=sparse_init, sparse_stdev=sparse_stdev, include_prob=include_prob, init_bias=init_bias, W_lr_scale=W_lr_scale, b_lr_scale=b_lr_scale, max_col_norm=max_col_norm, max_row_norm=max_row_norm)
def get_layer_sigmoid(self, layer): return Sigmoid(layer_name=layer.layer_name,dim=layer.dim,irange=layer.irange, istdev=layer.istdev,sparse_init=layer.sparse_init, sparse_stdev=layer.sparse_stdev, include_prob=layer.include_prob, init_bias=layer.init_bias,W_lr_scale=layer.W_lr_scale, b_lr_scale=layer.b_lr_scale,max_col_norm=layer.max_col_norm, max_row_norm=layer.max_row_norm)
def get_Sigmoid_Layer(self, structure, i=0): n_input, n_output = structure config = { 'dim': n_output, 'layer_name': ("s%d" % i), 'irange': 0.05, } return Sigmoid(**config)
def test_sigmoid_detection_cost(): # This is only a smoke test: verifies that it compiles and runs, # not any particular value. rng = np.random.RandomState(0) y = (rng.uniform(size=(4, 3)) > 0.5).astype('uint8') X = theano.shared(rng.uniform(size=(4, 2))) model = MLP(nvis=2, layers=[Sigmoid(monitor_style='detection', dim=3, layer_name='y', irange=0.8)]) y_hat = model.fprop(X) model.cost(y, y_hat).eval()
def test_correctness(): """ Test that the cost function works with float64 """ x_train, y_train, x_valid, y_valid = create_dataset() trainset = DenseDesignMatrix(X=np.array(x_train), y=y_train) validset = DenseDesignMatrix(X=np.array(x_valid), y=y_valid) n_inputs = trainset.X.shape[1] n_outputs = 1 n_hidden = 10 hidden_istdev = 4 * (6 / float(n_inputs + n_hidden)) ** 0.5 output_istdev = 4 * (6 / float(n_hidden + n_outputs)) ** 0.5 model = MLP(layers=[Sigmoid(dim=n_hidden, layer_name='hidden', istdev=hidden_istdev), Sigmoid(dim=n_outputs, layer_name='output', istdev=output_istdev)], nvis=n_inputs, seed=[2013, 9, 16]) termination_criterion = And([EpochCounter(max_epochs=1), MonitorBased(prop_decrease=1e-7, N=2)]) cost = SumOfCosts([(0.99, Default()), (0.01, L1WeightDecay({}))]) algo = SGD(1e-1, update_callbacks=[ExponentialDecay(decay_factor=1.00001, min_lr=1e-10)], cost=cost, monitoring_dataset=validset, termination_criterion=termination_criterion, monitor_iteration_mode='even_shuffled_sequential', batch_size=2) train = Train(model=model, dataset=trainset, algorithm=algo) train.main_loop()
def testPCA(self): init = PCAWeightInitializer.build({}, graph=Graph()) init.Input.resize(2) init.Input[0].setValue(self.X) init.Input[1].setValue(self.y) for k in (2, 4, 6, 8): layer = Sigmoid(layer_name="a", irange=0, dim=k) # layer needs to be initialized by MLP first MLP(layers=[layer], nvis=self.d) np.random.seed(123) init.init_layer(layer, nvis=self.d, nhid=k) weights = layer.get_weights() np.testing.assert_array_equal(weights.shape, (self.d, k)) assert np.abs(weights).sum() > 0 if k <= 2*self.d: for i in range(k): for j in range(i+2, k, 2): dot = np.dot(weights[:, i], weights[:, j]) np.testing.assert_almost_equal(dot, 0, decimal=4)
def test_sigmoid_layer_misclass_reporting(): mlp = MLP(nvis=3, layers=[Sigmoid(layer_name='h0', dim=1, irange=0.005, monitor_style='classification')]) target = theano.tensor.matrix(dtype=theano.config.floatX) batch = theano.tensor.matrix(dtype=theano.config.floatX) rval = mlp.layers[0].get_monitoring_channels_from_state(mlp.fprop(batch), target) f = theano.function([batch, target], [tensor.gt(mlp.fprop(batch), 0.5), rval['misclass']], allow_input_downcast=True) rng = np.random.RandomState(0) for _ in range(10): # repeat a few times for statistical strength targets = (rng.uniform(size=(30, 1)) > 0.5).astype('uint8') out, misclass = f(rng.normal(size=(30, 3)), targets) np.testing.assert_allclose((targets != out).mean(), misclass)
def generateNonConvRegressor(teacher_hintlayer, student_output_space): dim = teacher_hintlayer.output_space.get_total_dimension() layer_name = 'hint_regressor' irng = 0.05 mcn = 0.9 if isinstance(teacher_hintlayer, MaxoutConvC01B): hint_reg_layer = Maxout(layer_name, dim, teacher_hintlayer.num_pieces, irange=irng, max_col_norm=mcn) elif isinstance(teacher_hintlayer, ConvRectifiedLinear): hint_reg_layer = RectifiedLinear(dim=dim, layer_name=layer_name, irange=irng, max_col_norm=mcn) elif isinstance(teacher_hintlayer, ConvElemwise) or isinstance( teacher_hintlayer, ConvElemwisePL2): if isinstance(teacher_hintlayer.nonlinearity, RectifierConvNonlinearity): hint_reg_layer = RectifiedLinear(dim=dim, layer_name=layer_name, irange=irng, max_col_norm=mcn) elif isinstance(teacher_hintlayer.nonlinearity, SigmoidConvNonlinearity): hint_reg_layer = Sigmoid(dim=dim, layer_name=layer_name, irange=irng, max_col_norm=mcn) elif isinstance(teacher_hintlayer.nonlinearity, TanhConvNonlinearity): hint_reg_layer = Tanh(dim=dim, layer_name=layer_name, irange=irng, max_col_norm=mcn) else: raise AssertionError("Unknown layer type") else: raise AssertionError("Unknown fully-connected layer type") return hint_reg_layer
def construct_dbn_from_stack(stack): # some settings irange = 0.05 layers = [] for ii, layer in enumerate(stack.layers()): layers.append( Sigmoid(dim=layer.nhid, layer_name='h' + str(ii), irange=irange, max_col_norm=2.)) nc = 159 if SUBMODEL == 1 else 8 # softmax layer at then end for classification layers.append(Softmax(n_classes=nc, layer_name='y', irange=irange)) dbn = MLP(layers=layers, nvis=stack.layers()[0].get_input_space().dim) # copy weigths to DBN for ii, layer in enumerate(stack.layers()): dbn.layers[ii].set_weights(layer.get_weights()) dbn.layers[ii].set_biases(layer.hidbias.get_value(borrow=False)) return dbn
def get_convnet(img_shape=[256, 256], output_channels=16, kernel_shape=[88, 88], kernel_stride=[8, 8]): layers = [] dataset = get_dataset() input_space = Conv2DSpace(shape=img_shape, num_channels=1) conv_layer = ConvRectifiedLinear(output_channels=output_channels, irange=.005, layer_name="h0", kernel_shape=kernel_shape, kernel_stride=kernel_stride, pool_shape=[1, 1], pool_stride=[1, 1], max_kernel_norm=1.932) layers.append(conv_layer) maxout_layer = Maxout(layer_name="h1", irange=.005, num_units=600, num_pieces=4, max_col_norm=1.932) layers.append(maxout_layer) conv_out_dim = ((img_shape[0] - kernel_shape[0]) / kernel_stride[0] + 1)**2 sigmoid_layer = Sigmoid(layer_name="y", dim=conv_out_dim, monitor_style="detection", irange=.005) layers.append(sigmoid_layer) model = MLP(batch_size=100, layers=layers, input_space=input_space) return model
from pylearn2.costs.cost import MethodCost from pylearn2.datasets.mnist import MNIST from pylearn2.models.mlp import MLP, Sigmoid, Softmax from pylearn2.train import Train from pylearn2.training_algorithms.sgd import SGD from pylearn2.training_algorithms.learning_rule import Momentum, MomentumAdjustor from pylearn2.termination_criteria import EpochCounter train_set = MNIST(which_set='train', start=0, stop=50000) valid_set = MNIST(which_set='train', start=50000, stop=60000) test_set = MNIST(which_set='test') model = MLP(nvis=784, layers=[Sigmoid(layer_name='h', dim=500, irange=0.01), Softmax(layer_name='y', n_classes=10, irange=0.01)]) algorithm = SGD(batch_size=100, learning_rate=0.01, learning_rule=Momentum(init_momentum=0.5), monitoring_dataset={'train': train_set, 'valid': valid_set, 'test': test_set}, cost=MethodCost('cost_from_X'), termination_criterion=EpochCounter(10)) train = Train(dataset=train_set, model=model, algorithm=algorithm, save_path="mnist_example.pkl", save_freq=1, extensions=[MomentumAdjustor(start=5, saturate=6, final_momentum=0.95)]) train.main_loop()
kernel_shape=[5, 5], pool_shape=[2, 2], pool_stride=[2, 2], layer_name="h1", irange=0.1, border_mode="full") h2 = ConvRectifiedLinear(output_channels=32, kernel_shape=[5, 5], pool_shape=[2, 2], pool_stride=[2, 2], layer_name="h2", irange=0.1, border_mode="full") h3 = Sigmoid(layer_name="h3", dim=100, irange=0.01) h4 = Sigmoid(layer_name="h4", dim=100, irange=0.01) y = Softmax(n_classes=2, layer_name="y", irange=0.1) inputSpace = Conv2DSpace(shape=[cropSize, cropSize], num_channels=3) model = MLP(layers=[h0, h1, h2, h3, h4, y], batch_size=batchSize, input_space=inputSpace) algorithm = SGD(learning_rate=1e-3, cost=MethodCost("cost_from_X"), batch_size=batchSize, monitoring_batch_size=batchSize,