def test_convolutional(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, image_size=(17, 13), weights_init=Constant(1.), biases_init=Constant(5.)) conv.initialize() y = conv.apply(x) func = function([x], y) x_val = numpy.ones((batch_size, num_channels, 17, 13), dtype=theano.config.floatX) assert_allclose( func(x_val), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 15, 11)) + 5) conv.image_size = (17, 13) conv.batch_size = 2 # This should have effect on get_dim assert conv.get_dim('output') == (num_filters, 15, 11)
def test_tied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.), tied_biases=True) conv.initialize() y = conv.apply(x) func = function([x], y) # Tied biases allows to pass images of different sizes x_val_1 = numpy.ones((batch_size, num_channels, 10, 12), dtype=theano.config.floatX) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) assert_allclose(func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 8, 10)) + 2) assert_allclose(func(x_val_2), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 21, 17)) + 2)
def test_convolutional(): x = tensor.tensor4("x") num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional( filter_size, num_filters, num_channels, image_size=(17, 13), weights_init=Constant(1.0), biases_init=Constant(5.0), ) conv.initialize() y = conv.apply(x) func = function([x], y) x_val = numpy.ones((batch_size, num_channels, 17, 13), dtype=theano.config.floatX) assert_allclose( func(x_val), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 15, 11)) + 5 ) conv.image_size = (17, 13) conv.batch_size = 2 # This should have effect on get_dim assert conv.get_dim("output") == (num_filters, 15, 11)
def test_tied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) # Tied biases are the default conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.)) conv.initialize() y = conv.apply(x) func = function([x], y) # Tied biases only provide one bias for each filter assert_allclose(conv.b.eval().shape, (3, )) # Tied biases allows to pass images of different sizes x_val_1 = numpy.ones((batch_size, num_channels, 10, 12), dtype=theano.config.floatX) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) assert_allclose( func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 8, 10)) + 2) assert_allclose( func(x_val_2), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 21, 17)) + 2)
def conv_layer(self, name, wt, bias, image_size): """Creates a Convolutional brick with the given name, weights, bias, and image_size.""" layer = Convolutional( name=name, filter_size=wt.shape[0:2], num_channels=wt.shape[2], # in num_filters=wt.shape[3], # out weights_init=Constant(0), # does not matter biases_init=Constant(0), # does not matter tied_biases=True, border_mode='valid', ) if image_size: layer.image_size = image_size layer.initialize() weights = self.to_bc01(wt) layer.parameters[0].set_value(weights.astype("float32")) # W layer.parameters[1].set_value(bias.squeeze().astype("float32")) # b return (layer, layer.get_dim("output")[1:3])
def test_no_input_size(): # suppose x is outputted by some RNN x = tensor.tensor4('x') filter_size = (1, 3) num_filters = 2 num_channels = 5 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=True, weights_init=Constant(1.), biases_init=Constant(1.)) c.initialize() out = c.apply(x) assert c.get_dim('output') == (2, None, None) assert out.ndim == 4 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=False, weights_init=Constant(1.), biases_init=Constant(1.)) assert_raises_regexp(ValueError, 'Cannot infer bias size \S+', c.initialize)
def test_untied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.), image_size=(28, 30), tied_biases=False) conv.initialize() y = conv.apply(x) func = function([x], y) # Untied biases provide a bias for every individual output assert_allclose(conv.b.eval().shape, (3, 26, 28)) # Untied biases require images of a specific size x_val_1 = numpy.ones((batch_size, num_channels, 28, 30), dtype=theano.config.floatX) assert_allclose(func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 26, 28)) + 2) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) def wrongsize(): func(x_val_2) assert_raises_regexp(AssertionError, 'AbstractConv shape mismatch', wrongsize)
def conv_layer(self, name, wt, bias, image_size): """Creates a Convolutional brick with the given name, weights, bias, and image_size.""" layer = Convolutional( name=name, filter_size=wt.shape[0:2], num_channels=wt.shape[2], # in num_filters=wt.shape[3], # out weights_init=Constant(0), # does not matter biases_init=Constant(0), # does not matter tied_biases=True, border_mode="valid", ) if image_size: layer.image_size = image_size layer.initialize() weights = self.to_bc01(wt) layer.parameters[0].set_value(weights.astype("float32")) # W layer.parameters[1].set_value(bias.squeeze().astype("float32")) # b return (layer, layer.get_dim("output")[1:3])
def test_no_input_size(): # suppose x is outputted by some RNN x = tensor.tensor4('x') filter_size = (1, 3) num_filters = 2 num_channels = 5 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=True, weights_init=Constant(1.), biases_init=Constant(1.)) c.initialize() out = c.apply(x) assert c.get_dim('output') == (2, None, None) assert out.ndim == 4 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=False, weights_init=Constant(1.), biases_init=Constant(1.)) assert_raises_regexp(ValueError, 'Cannot infer bias size \S+', c.initialize)
def build_model(Zs): # first block z1 = conv_block(Zs['Z1'], 8, (3, 3), (3, 256, 256), '1') z2 = conv_block(Zs['Z2'], 8, (3, 3), (3, 128, 128), '2') z3 = conv_block(Zs['Z3'], 8, (3, 3), (3, 64, 64), '3') z4 = conv_block(Zs['Z4'], 8, (3, 3), (3, 32, 32), '4') z5 = conv_block(Zs['Z5'], 8, (3, 3), (3, 16, 16), '5') # merge 4 & 5 m45 = join(z5, z4, 8, 8, (32, 32), '45') # merge 3 & 45 c45 = conv_block(m45, 16, (3, 3), (16, 32, 32), '45') m345 = join(c45, z3, 16, 8, (64, 64)) # merge 2 & 345 c345 = conv_block(m345, 24, (3, 3), (24, 64, 64), '345') m2345 = join(c345, z2, 24, 8, (128, 128), '345') # merge 1 & 2345 c2345 = conv_block(m2345, 32, (3, 3), (32, 128, 128), '2345') m12345 = join(c2345, z1, 32, 8, (256, 256), '2345') last_conv_block = conv_block(m12345, 40, (3, 3), (40, 256, 256), '12345') conv_out = Convolutional((1, 1), 3, 40, image_size=(256, 256), biases_init=Uniform(width=.1), weights_init=Uniform(width=.3), use_bias=True, name='conv_out') conv_out.initialize() conv_out_output = conv_out.apply(last_conv_block) mean = tensor.addbroadcast( theano.shared(MEAN_VALUES).astype('float32'), 0, 2, 3) return 128. * (1 + conv_out_output) - mean
from blocks.graph import ComputationGraph X = T.matrix("features") o = X.reshape((X.shape[0], 1, 28, 28)) l = Convolutional(filter_size=(5, 5), num_filters=32, num_channels=1, image_size=(28, 28), weights_init=IsotropicGaussian(std=0.01), biases_init=IsotropicGaussian(std=0.01, mean=1.0), use_bias=True, border_mode="valid", step=(1, 1)) l.initialize() o = l.apply(o) l = BatchNormalizationConv(input_shape=l.get_dim("output"), B_init=IsotropicGaussian(std=0.01), Y_init=IsotropicGaussian(std=0.01)) l.initialize() o = l.apply(o) o = Rectifier().apply(o) l = MaxPooling(pooling_size=(2, 2), step=(2, 2), input_dim=l.get_dim("output")) l.initialize() o = l.apply(o) #ll = Dropout(p_drop=0.5)
from blocks.graph import ComputationGraph X = T.matrix("features") o = X.reshape((X.shape[0], 1, 28, 28)) l = Convolutional(filter_size=(5, 5), num_filters=32, num_channels=1, image_size=(28,28), weights_init=IsotropicGaussian(std=0.01), biases_init=IsotropicGaussian(std=0.01, mean=1.0), use_bias=True, border_mode="valid", step=(1,1)) l.initialize() o = l.apply(o) l = BatchNormalizationConv(input_shape=l.get_dim("output"), B_init=IsotropicGaussian(std=0.01), Y_init=IsotropicGaussian(std=0.01)) l.initialize() o = l.apply(o) o = Rectifier().apply(o) l = MaxPooling(pooling_size=(2, 2), step=(2, 2), input_dim=l.get_dim("output")) l.initialize() o = l.apply(o)
from blocks.bricks import WEIGHT from blocks.graph import ComputationGraph from blocks.filter import VariableFilter cg = ComputationGraph(cost) W1, W2 = VariableFilter(roles=[WEIGHT])(cg.variables) cost = cost + 0.005 * (W1 ** 2).sum() + 0.005 * (W2 ** 2).sum() cost.name = 'cost_with_regularization' from blocks.bricks import MLP mlp = MLP(activations=[Rectifier(), Softmax()], dims=[784, 100, 10]).apply(x) from blocks.initialization import IsotropicGaussian, Constant input_to_hidden.weights_init = hidden_to_output.weights_init = IsotropicGaussian(0.01) input_to_hidden.biases_init = hidden_to_output.biases_init = Constant(0) input_to_hidden.initialize() hidden_to_output.initialize() from fuel.datasets import MNIST mnist = MNIST("train",) from fuel.streams import DataStream from fuel.schemes import SequentialScheme from fuel.transformers import Flatten data_stream = Flatten(DataStream.default_stream( mnist, iteration_scheme=SequentialScheme(mnist.num_examples, batch_size=256))) from blocks.algorithms import GradientDescent, Scale algorithm = GradientDescent(step_rule=None,cost=cost,params=cg.parameters)