def test_convolutional_sequence(): x = tensor.tensor4('x') num_channels = 4 pooling_size = 3 batch_size = 5 act = Rectifier() conv = Convolutional((3, 3), 5, weights_init=Constant(1.), biases_init=Constant(5.)) pooling = MaxPooling(pooling_size=(pooling_size, pooling_size)) conv2 = Convolutional((2, 2), 4, weights_init=Constant(1.)) seq = ConvolutionalSequence([conv, act, pooling.apply, conv2.apply, act], num_channels, image_size=(17, 13)) seq.push_allocation_config() assert conv.num_channels == 4 assert conv2.num_channels == 5 conv2.use_bias = False y = seq.apply(x) seq.initialize() func = function([x], y) x_val = numpy.ones((batch_size, 4, 17, 13), dtype=theano.config.floatX) y_val = (numpy.ones((batch_size, 4, 4, 2)) * (9 * 4 + 5) * 4 * 5) assert_allclose(func(x_val), y_val)
def test_tied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.), tied_biases=True) conv.initialize() y = conv.apply(x) func = function([x], y) # Tied biases allows to pass images of different sizes x_val_1 = numpy.ones((batch_size, num_channels, 10, 12), dtype=theano.config.floatX) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) assert_allclose(func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 8, 10)) + 2) assert_allclose(func(x_val_2), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 21, 17)) + 2)
def create_KmaxPooling_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' k = int(config[pref + '_kpool']) filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] print filter_width_list num_filters = int(config[pref+'_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 print input_len, embedding_size, num_filters for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional( image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1 ) totfilters += conv.num_filters * k # initialize2(conv, num_feature_map) initialize([conv]) conv.name = pref + 'conv_' + str(fw) layer0_input = debug_print(layer0_input, 'inp', False) convout = conv.apply(layer0_input) convout = debug_print(convout, 'convout', False) kpoolout = KmaxPooling(convout, k).apply().flatten(2) kpoolout = debug_print(kpoolout, 'poolout', False) if i == 0: outpools = kpoolout else: outpools = T.concatenate([outpools, kpoolout], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def __init__(self, filter_size, num_filters, num_channels, noise_batch_size, image_size=(None, None), step=(1, 1), border_mode='valid', tied_biases=True, prior_mean=0, prior_noise_level=0, **kwargs): self.convolution = Convolutional() self.mask = Convolutional(name='mask') children = [self.convolution, self.mask] kwargs.setdefault('children', []).extend(children) super(NoisyConvolutional, self).__init__(**kwargs) self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.noise_batch_size = noise_batch_size self.image_size = image_size self.step = step self.border_mode = border_mode self.tied_biases = tied_biases self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] print filter_width_list num_filters = int(config[pref+'_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional( image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1 ) totfilters += conv.num_filters initialize2(conv, num_feature_map) conv.name = pref + 'conv_' + str(fw) convout = conv.apply(layer0_input) pool_layer = MaxPooling( pooling_size=(num_feature_map,1) ) pool_layer.name = pref + 'pool_' + str(fw) act = Rectifier() act.name = pref + 'act_' + str(fw) outpool = act.apply(pool_layer.apply(convout)).flatten(2) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def build_conv_layers(self, image=None): if image is None: image = T.ftensor4('spectrogram') else: image = image conv_list = [] for layer in range(self.layers): layer_param = self.params[layer] conv_layer = Convolutional(layer_param[0], layer_param[1], layer_param[2]) pool_layer = MaxPooling(layer_param[3]) conv_layer.name = "convolution" + str(layer) pool_layer.name = "maxpooling" + str(layer) conv_list.append(conv_layer) conv_list.append(pool_layer) conv_list.append(Rectifier()) conv_seq = ConvolutionalSequence(conv_list, self.params[0][2], image_size=self.image_size, weights_init=IsotropicGaussian( std=0.5, mean=0), biases_init=Constant(0)) conv_seq._push_allocation_config() conv_seq.initialize() out = conv_seq.apply(image) return out, conv_seq.get_dim('output')
def test_tied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) # Tied biases are the default conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.)) conv.initialize() y = conv.apply(x) func = function([x], y) # Tied biases only provide one bias for each filter assert_allclose(conv.b.eval().shape, (3, )) # Tied biases allows to pass images of different sizes x_val_1 = numpy.ones((batch_size, num_channels, 10, 12), dtype=theano.config.floatX) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) assert_allclose( func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 8, 10)) + 2) assert_allclose( func(x_val_2), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 21, 17)) + 2)
def build_conv_layers(self, image=None) : if image is None : image = T.ftensor4('spectrogram') else : image = image conv_list = [] for layer in range(self.layers) : layer_param = self.params[layer] conv_layer = Convolutional(layer_param[0], layer_param[1], layer_param[2]) pool_layer = MaxPooling(layer_param[3]) conv_layer.name = "convolution"+str(layer) pool_layer.name = "maxpooling"+str(layer) conv_list.append(conv_layer) conv_list.append(pool_layer) conv_list.append(Rectifier()) conv_seq = ConvolutionalSequence( conv_list, self.params[0][2], image_size=self.image_size, weights_init=IsotropicGaussian(std=0.5, mean=0), biases_init=Constant(0)) conv_seq._push_allocation_config() conv_seq.initialize() out = conv_seq.apply(image) return out, conv_seq.get_dim('output')
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] print filter_width_list num_filters = int(config[pref + '_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional(image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1) totfilters += conv.num_filters initialize2(conv, num_feature_map) conv.name = pref + 'conv_' + str(fw) convout = conv.apply(layer0_input) pool_layer = MaxPooling(pooling_size=(num_feature_map, 1)) pool_layer.name = pref + 'pool_' + str(fw) act = Rectifier() act.name = pref + 'act_' + str(fw) outpool = act.apply(pool_layer.apply(convout)).flatten(2) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def test_convolutional_sequence_tied_biases_pushed_if_explicitly_set(): cnn = ConvolutionalSequence(sum([[ Convolutional(filter_size=(1, 1), num_filters=1, tied_biases=True), Rectifier() ] for _ in range(3)], []), num_channels=1, image_size=(1, 1), tied_biases=False) cnn.allocate() assert [ not child.tied_biases for child in cnn.children if isinstance(child, Convolutional) ] cnn = ConvolutionalSequence(sum( [[Convolutional(filter_size=(1, 1), num_filters=1), Rectifier()] for _ in range(3)], []), num_channels=1, image_size=(1, 1), tied_biases=True) cnn.allocate() assert [ child.tied_biases for child in cnn.children if isinstance(child, Convolutional) ]
def test_untied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.), image_size=(28, 30), tied_biases=False) conv.initialize() y = conv.apply(x) func = function([x], y) # Untied biases provide a bias for every individual output assert_allclose(conv.b.eval().shape, (3, 26, 28)) # Untied biases require images of a specific size x_val_1 = numpy.ones((batch_size, num_channels, 28, 30), dtype=theano.config.floatX) assert_allclose(func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 26, 28)) + 2) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) def wrongsize(): func(x_val_2) assert_raises_regexp(AssertionError, 'AbstractConv shape mismatch', wrongsize)
def conv_layer(self, name, wt, bias, image_size): """Creates a Convolutional brick with the given name, weights, bias, and image_size.""" layer = Convolutional( name=name, filter_size=wt.shape[0:2], num_channels=wt.shape[2], # in num_filters=wt.shape[3], # out weights_init=Constant(0), # does not matter biases_init=Constant(0), # does not matter tied_biases=True, border_mode="valid", ) if image_size: layer.image_size = image_size layer.initialize() weights = self.to_bc01(wt) layer.parameters[0].set_value(weights.astype("float32")) # W layer.parameters[1].set_value(bias.squeeze().astype("float32")) # b return (layer, layer.get_dim("output")[1:3])
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with the same filtersize ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] print filter_width_list num_filters = int(config[pref + '_num_filters']) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional(filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(input_len, embedding_size), name="conv" + str(fw)) pooling = MaxPooling((num_feature_map, 1), name="pool" + str(fw)) initialize([conv]) totfilters += num_filters outpool = Flattener(name="flat" + str(fw)).apply( Rectifier(name=pref + 'act_' + str(fw)).apply( pooling.apply(conv.apply(layer0_input)))) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def test_batch_normalization_inside_convolutional_sequence(): """Test that BN bricks work in ConvolutionalSequences.""" conv_seq = ConvolutionalSequence( [Convolutional(filter_size=(3, 3), num_filters=4), BatchNormalization(broadcastable=(False, True, True)), AveragePooling(pooling_size=(2, 2)), BatchNormalization(broadcastable=(False, False, False)), MaxPooling(pooling_size=(2, 2), step=(1, 1))], weights_init=Constant(1.), biases_init=Constant(2.), image_size=(10, 8), num_channels=9) conv_seq_no_bn = ConvolutionalSequence( [Convolutional(filter_size=(3, 3), num_filters=4), AveragePooling(pooling_size=(2, 2)), MaxPooling(pooling_size=(2, 2), step=(1, 1))], weights_init=Constant(1.), biases_init=Constant(2.), image_size=(10, 8), num_channels=9) conv_seq.initialize() conv_seq_no_bn.initialize() rng = numpy.random.RandomState((2015, 12, 17)) input_ = random_unif(rng, (2, 9, 10, 8)) x = theano.tensor.tensor4() ybn = conv_seq.apply(x) y = conv_seq_no_bn.apply(x) yield (assert_equal, ybn.eval({x: input_}), y.eval({x: input_})) std = conv_seq.children[-2].population_stdev std.set_value(3 * std.get_value(borrow=True)) yield (assert_equal, ybn.eval({x: input_}), y.eval({x: input_}) / 3.)
def generate_elementary_block4(self, index): number_of_channels = 512 name_conv_0 = 'fconv7_' + str(index) name_relu_0 = 'relu7_' + str(index) name_conv_3 = 'fconv7_output_' + str(index) return [Convolutional(filter_size=(1,1), num_filters = 128, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_0, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = number_of_channels), \ ParallelSum2(), Rectifier(name=name_relu_0), \ Convolutional(filter_size=(1,1), num_filters = 1, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_3, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 128)]
def __init__(self, filter_size, num_filters, num_channels, batch_size=None, mid_noise=False, out_noise=False, tied_noise=False, tied_sigma=False, noise_rate=None, noise_batch_size=None, prior_noise_level=None, image_size=(None, None), step=(1, 1), **kwargs): self.filter_size = filter_size self.num_filters = num_filters self.batch_size = batch_size self.num_channels = num_channels self.image_size = image_size self.mid_noise = mid_noise self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate self.step = step self.border_mode = 'half' self.tied_biases = True depth = 2 self.b0 = SpatialBatchNormalization(name='b0') self.r0 = Rectifier(name='r0') self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if mid_noise else None) self.c0 = Convolutional(name='c0') self.b1 = SpatialBatchNormalization(name='b1') self.r1 = Rectifier(name='r1') self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if out_noise else None) self.c1 = Convolutional(name='c1') kwargs.setdefault('children', []).extend([ c for c in [ self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1, self.n1 ] if c is not None ]) super(ResidualConvolutional, self).__init__(**kwargs)
def __init__(self, image_dimension, **kwargs): layers = [] ############################################# # a first block with 2 convolutions of 32 (3, 3) filters layers.append(Convolutional((3, 3), 32, border_mode='half')) layers.append(Rectifier()) layers.append(Convolutional((3, 3), 32, border_mode='half')) layers.append(Rectifier()) # maxpool with size=(2, 2) layers.append(MaxPooling((2, 2))) ############################################# # a 2nd block with 3 convolutions of 64 (3, 3) filters layers.append(Convolutional((3, 3), 64, border_mode='half')) layers.append(Rectifier()) layers.append(Convolutional((3, 3), 64, border_mode='half')) layers.append(Rectifier()) layers.append(Convolutional((3, 3), 64, border_mode='half')) layers.append(Rectifier()) # maxpool with size=(2, 2) layers.append(MaxPooling((2, 2))) ############################################# # a 3rd block with 4 convolutions of 128 (3, 3) filters layers.append(Convolutional((3, 3), 128, border_mode='half')) layers.append(Rectifier()) layers.append(Convolutional((3, 3), 128, border_mode='half')) layers.append(Rectifier()) layers.append(Convolutional((3, 3), 128, border_mode='half')) layers.append(Rectifier()) layers.append(Convolutional((3, 3), 128, border_mode='half')) layers.append(Rectifier()) # maxpool with size=(2, 2) layers.append(MaxPooling((2, 2))) self.conv_sequence = ConvolutionalSequence(layers, 3, image_size=image_dimension) flattener = Flattener() self.top_mlp = MLP(activations=[Rectifier(), Logistic()], dims=[500, 1]) application_methods = [ self.conv_sequence.apply, flattener.apply, self.top_mlp.apply ] super(VGGNet, self).__init__(application_methods, biases_init=Constant(0), weights_init=Uniform(width=.1), **kwargs)
class ConvolutionalActivation(Initializable): """A convolution followed by an activation function. Parameters ---------- activation : :class:`.BoundApplication` The application method to apply after convolution (i.e. the nonlinear activation function) See Also -------- :class:`Convolutional` : For the documentation of other parameters. """ @lazy(allocation=['filter_size', 'num_filters', 'num_channels']) def __init__(self, activation, filter_size, num_filters, num_channels, batch_size=None, image_size=None, step=(1, 1), border_mode='valid', tied_biases=False, **kwargs): self.convolution = Convolutional(name='conv'+ kwargs['name']) self.bn = BatchNorm(name='bn'+ kwargs['name']) self.activation = activation self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.batch_size = batch_size self.image_size = image_size self.step = step self.border_mode = border_mode self.tied_biases = tied_biases super(ConvolutionalActivation, self).__init__(**kwargs) self.children = [self.convolution, self.bn, self.activation] def _push_allocation_config(self): for attr in ['filter_size', 'num_filters', 'step', 'border_mode', 'batch_size', 'num_channels', 'image_size', 'tied_biases']: setattr(self.convolution, attr, getattr(self, attr)) setattr(self.bn, 'input_dim', self.num_filters) def get_dim(self, name): # TODO The name of the activation output doesn't need to be `output` return self.convolution.get_dim(name) def apply(self, input_): out = self.convolution.apply(input_) out = self.bn.apply(out) out = self.activation.apply(out) return out def inference(self, input_): out = self.convolution.apply(input_) out = self.bn.inference(out) out = self.activation.apply(out) return out
def test_convolutional_sequence_with_convolutions_raw_activation(): seq = ConvolutionalSequence( [Convolutional(filter_size=(3, 3), num_filters=4), Rectifier(), Convolutional(filter_size=(5, 5), num_filters=3, step=(2, 2)), Tanh()], num_channels=2, image_size=(21, 39)) seq.allocate() x = theano.tensor.tensor4() out = seq.apply(x).eval({x: numpy.ones((10, 2, 21, 39), dtype=theano.config.floatX)}) assert out.shape == (10, 3, 8, 17)
def test_convolutional(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, image_size=(17, 13), weights_init=Constant(1.), biases_init=Constant(5.)) conv.initialize() y = conv.apply(x) func = function([x], y) x_val = numpy.ones((batch_size, num_channels, 17, 13), dtype=theano.config.floatX) assert_allclose( func(x_val), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 15, 11)) + 5) conv.image_size = (17, 13) conv.batch_size = 2 # This should have effect on get_dim assert conv.get_dim('output') == (num_filters, 15, 11)
def generate_elementary_block1(self, index, to_index): number_of_channels = 512 name_conv_0 = 'fconv7_' + str(index) name_relu_0 = 'relu7_' + str(index) name_conv_1 = 'fconv7_' + str(index) + 'to' + str(to_index) + '_step1' name_relu_1 = 'relu7_' + str(index) + 'to' + str(to_index) + '_step1' name_conv_2 = 'fconv7_' + str(index) + 'to' + str(to_index) + '_step2' name_conv_3 = 'fconv7_output_' + str(index) return [Convolutional(filter_size=(1,1), num_filters = 128, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_0, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = number_of_channels), \ Rectifier(name=name_relu_0), \ Convolutional(filter_size=(7,7), num_filters = 64, border_mode = (3,3), use_bias=True, tied_biases=True, name=name_conv_1, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 128), \ Rectifier(name=name_relu_1), \ Convolutional(filter_size=(7,7), num_filters = 128, border_mode = (3,3), use_bias=True, tied_biases=True, name=name_conv_2, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 64), \ Convolutional(filter_size=(1,1), num_filters = 1, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_3, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 128)]
def test_no_input_size(): # suppose x is outputted by some RNN x = tensor.tensor4('x') filter_size = (1, 3) num_filters = 2 num_channels = 5 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=True, weights_init=Constant(1.), biases_init=Constant(1.)) c.initialize() out = c.apply(x) assert c.get_dim('output') == (2, None, None) assert out.ndim == 4 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=False, weights_init=Constant(1.), biases_init=Constant(1.)) assert_raises_regexp(ValueError, 'Cannot infer bias size \S+', c.initialize)
def conv_block(input_img, n_filter, filter_size, input_featuremap_size, ordering=''): # found in torch spatialconvolution std0 = 2. / (filter_size[0] * filter_size[1] * input_featuremap_size[0])**.5 std1 = 2. / (input_featuremap_size[0])**.5 layers = [] layers.append( Convolutional(filter_size=filter_size, num_filters=n_filter, border_mode='half', name='conv%s_1' % (ordering, ), use_bias=True, weights_init=Uniform(width=std0))) layers.append(BatchNormalization(name='bn%s_1' % (ordering, ))) layers.append(LeakyReLU()) layers.append( Convolutional(filter_size=filter_size, num_filters=n_filter, border_mode='half', name='conv%s_2' % (ordering, ), use_bias=True, weights_init=Uniform(width=std0))) layers.append(BatchNormalization(name='bn%s_2' % (ordering, ))) layers.append(LeakyReLU()) layers.append( Convolutional(filter_size=(1, 1), num_filters=n_filter, border_mode='valid', name='conv%s_3b' % (ordering, ), use_bias=True, weights_init=Uniform(width=std1))) layers.append(BatchNormalization(name='bn%s_3' % (ordering, ))) layers.append(LeakyReLU()) conv_sequence = ConvolutionalSequence( layers, num_channels=input_featuremap_size[0], image_size=(input_featuremap_size[1], input_featuremap_size[2]), biases_init=Uniform(width=.1), name='convsequence%s' % (ordering, )) conv_sequence.initialize() return conv_sequence.apply(input_img)
def __init__(self, activation, filter_size, num_filters, num_channels, batch_size=None, image_size=None, step=(1, 1), border_mode='valid', tied_biases=False, **kwargs): self.convolution = Convolutional(name='conv'+ kwargs['name']) self.bn = BatchNorm(name='bn'+ kwargs['name']) self.activation = activation self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.batch_size = batch_size self.image_size = image_size self.step = step self.border_mode = border_mode self.tied_biases = tied_biases super(ConvolutionalActivation, self).__init__(**kwargs) self.children = [self.convolution, self.bn, self.activation]
def conv_brick(filter_size, step, num_filters, border_mode='valid'): """Instantiates a ConvolutionalBrick.""" return Convolutional(filter_size=(filter_size, filter_size), step=(step, step), border_mode=border_mode, num_filters=num_filters, name=name_generator())
def __init__(self, conv_activations, num_channels, image_shape, filter_sizes, feature_maps, pooling_sizes, top_mlp_activations, top_mlp_dims, conv_step=None, border_mode='valid', **kwargs): if conv_step is None: self.conv_step = (1, 1) else: self.conv_step = conv_step self.num_channels = num_channels self.image_shape = image_shape self.top_mlp_activations = top_mlp_activations self.top_mlp_dims = top_mlp_dims self.border_mode = border_mode conv_parameters = zip(filter_sizes, feature_maps) # Construct convolutional, activation, and pooling layers with corresponding parameters self.convolution_layer = ( Convolutional(filter_size=filter_size, num_filters=num_filter, step=self.conv_step, border_mode=self.border_mode, name='conv_{}'.format(i)) for i, (filter_size, num_filter) in enumerate(conv_parameters)) self.BN_layer = (BatchNormalization(name='bn_conv_{}'.format(i)) for i in enumerate(conv_parameters)) self.pooling_layer = (MaxPooling(size, name='pool_{}'.format(i)) for i, size in enumerate(pooling_sizes)) self.layers = list( interleave([ self.convolution_layer, self.BN_layer, conv_activations, self.pooling_layer ])) self.conv_sequence = ConvolutionalSequence(self.layers, num_channels, image_size=image_shape) # Construct a top MLP self.top_mlp = MLP(top_mlp_activations, top_mlp_dims) # We need to flatten the output of the last convolutional layer. # This brick accepts a tensor of dimension (batch_size, ...) and # returns a matrix (batch_size, features) self.flattener = Flattener() application_methods = [ self.conv_sequence.apply, self.flattener.apply, self.top_mlp.apply ] super(LeNet, self).__init__(application_methods, **kwargs)
def test_convolutional_sequence_with_no_input_size(): # suppose x is outputted by some RNN x = tensor.tensor4('x') filter_size = (1, 1) num_filters = 2 num_channels = 1 pooling_size = (1, 1) conv = Convolutional(filter_size, num_filters, tied_biases=False, weights_init=Constant(1.), biases_init=Constant(1.)) act = Rectifier() pool = MaxPooling(pooling_size) bad_seq = ConvolutionalSequence([conv, act, pool], num_channels, tied_biases=False) assert_raises_regexp(ValueError, 'Cannot infer bias size \S+', bad_seq.initialize) seq = ConvolutionalSequence([conv, act, pool], num_channels, tied_biases=True) try: seq.initialize() out = seq.apply(x) except TypeError: assert False, "This should have succeeded" assert out.ndim == 4
def __init__(self, **kwargs): children = [] self.list_simple_joints = [19, 20, 8, 7] + range(16,19)[::-1] + range(4, 7)[::-1] + [1] + [15, 3, 2] + [0] self.simple_joints = {} for simple_joint_idx in self.list_simple_joints: self.simple_joints[simple_joint_idx] = [Convolutional(filter_size=(1,1), num_filters = 512, border_mode = (0,0), use_bias=True, tied_biases=True, name='fconv_' + str(simple_joint_idx), biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 512), \ Rectifier(name='fconv_relu' + str(simple_joint_idx)), \ Convolutional(filter_size=(1,1), num_filters = 1, border_mode = (0,0), use_bias=True, tied_biases=True, name='fconv1_' + str(simple_joint_idx), biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 512), \ Logistic(name = 'flogistic_' + str(simple_joint_idx))] children += self.simple_joints[simple_joint_idx] kwargs.setdefault('children', []).extend(children) super(top_direction_block, self).__init__(**kwargs)
def build_pipeline(self, input_shape, params): from blocks.bricks import Tanh, Sequence from blocks.bricks.conv import Convolutional, MaxPooling from blocks.initialization import Uniform _, num_channels, input_len, num_freqs = input_shape # bc01 # Note: this layer is linear conv = Convolutional( name='conv', filter_size=(params['filter_width_time'], params['filter_width_freq']), num_filters=params['num_components'], # out num_channels=num_channels, # in image_size=(input_len, num_freqs), weights_init=Uniform(mean=0, std=0.01), use_bias=params['use_bias']) tanh = Tanh() # optional pooling if params['pool_width_time'] > 1 or params['pool_width_freq'] > 1: pool = MaxPooling( (params['pool_width_time'], params['pool_width_freq']), step=(params['pool_stride_time'], params['pool_stride_freq'])) pipeline = Sequence([conv.apply, tanh.apply, pool.apply], name='pipeline') else: pipeline = Sequence([conv.apply, tanh.apply], name='pipeline') pipeline.initialize() return pipeline
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with the same filtersize ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] print filter_width_list num_filters = int(config[pref+'_num_filters']) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional( filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(input_len, embedding_size), name="conv" + str(fw)) pooling = MaxPooling((num_feature_map,1), name="pool"+str(fw)) initialize([conv]) totfilters += num_filters outpool = Flattener(name="flat"+str(fw)).apply(Rectifier(name=pref+'act_'+str(fw)).apply(pooling.apply(conv.apply(layer0_input)))) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def test_convolutional(): x = tensor.tensor4("x") num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional( filter_size, num_filters, num_channels, image_size=(17, 13), weights_init=Constant(1.0), biases_init=Constant(5.0), ) conv.initialize() y = conv.apply(x) func = function([x], y) x_val = numpy.ones((batch_size, num_channels, 17, 13), dtype=theano.config.floatX) assert_allclose( func(x_val), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 15, 11)) + 5 ) conv.image_size = (17, 13) conv.batch_size = 2 # This should have effect on get_dim assert conv.get_dim("output") == (num_filters, 15, 11)
def test_fully_layer(): batch_size=2 x = T.tensor4(); y = T.ivector() V = 200 layer_conv = Convolutional(filter_size=(5,5),num_filters=V, name="toto", weights_init=IsotropicGaussian(0.01), biases_init=Constant(0.0)) # try with no bias activation = Rectifier() pool = MaxPooling(pooling_size=(2,2)) convnet = ConvolutionalSequence([layer_conv, activation, pool], num_channels=15, image_size=(10,10), name="conv_section") convnet.push_allocation_config() convnet.initialize() output=convnet.apply(x) batch_size=output.shape[0] output_dim=np.prod(convnet.get_dim('output')) result_conv = output.reshape((batch_size, output_dim)) mlp=MLP(activations=[Rectifier().apply], dims=[output_dim, 10], weights_init=IsotropicGaussian(0.01), biases_init=Constant(0.0)) mlp.initialize() output=mlp.apply(result_conv) cost = T.mean(Softmax().categorical_cross_entropy(y.flatten(), output)) cg = ComputationGraph(cost) W = VariableFilter(roles=[WEIGHT])(cg.variables) B = VariableFilter(roles=[BIAS])(cg.variables) W = W[0]; b = B[0] inputs_fully = VariableFilter(roles=[INPUT], bricks=[Linear])(cg) outputs_fully = VariableFilter(roles=[OUTPUT], bricks=[Linear])(cg) var_input=inputs_fully[0] var_output=outputs_fully[0] [d_W,d_S,d_b] = T.grad(cost, [W, var_output, b]) d_b = d_b.dimshuffle(('x',0)) d_p = T.concatenate([d_W, d_b], axis=0) x_value = 1e3*np.random.ranf((2,15, 10, 10)) f = theano.function([x,y], [var_input, d_S, d_p], allow_input_downcast=True, on_unused_input='ignore') A, B, C= f(x_value, [5, 0]) A = np.concatenate([A, np.ones((2,1))], axis=1) print 'A', A.shape print 'B', B.shape print 'C', C.shape print lin.norm(C - np.dot(np.transpose(A), B), 'fro') return """
def test_convolutional_sequence_activation_get_dim(): seq = ConvolutionalSequence([Tanh()], num_channels=9, image_size=(4, 6)) seq.allocate() assert seq.get_dim('output') == (9, 4, 6) seq = ConvolutionalSequence([Convolutional(filter_size=(7, 7), num_filters=5, border_mode=(1, 1)), Tanh()], num_channels=8, image_size=(8, 11)) seq.allocate() assert seq.get_dim('output') == (5, 4, 7)
def test_convolutional_sequence_use_bias(): cnn = ConvolutionalSequence( sum([[Convolutional(filter_size=(1, 1), num_filters=1), Rectifier()] for _ in range(3)], []), num_channels=1, image_size=(1, 1), use_bias=False) cnn.allocate() x = tensor.tensor4() y = cnn.apply(x) params = ComputationGraph(y).parameters assert len(params) == 3 and all(param.name == 'W' for param in params)
def test_border_mode_not_pushed(): layers = [Convolutional(border_mode='full'), Convolutional(), Rectifier(), Convolutional(border_mode='valid'), Rectifier(), Convolutional(border_mode='full'), Rectifier()] stack = ConvolutionalSequence(layers) stack.push_allocation_config() assert stack.children[0].border_mode == 'full' assert stack.children[1].border_mode == 'valid' assert stack.children[3].border_mode == 'valid' assert stack.children[5].border_mode == 'full' stack2 = ConvolutionalSequence(layers, border_mode='full') stack2.push_allocation_config() assert stack2.children[0].border_mode == 'full' assert stack2.children[1].border_mode == 'full' assert stack2.children[3].border_mode == 'full' assert stack2.children[5].border_mode == 'full'
def create_yy_cnn(numConvLayer, conv_input, embedding_size, input_len, config, pref): ''' CNN with several layers of convolution, each with specific filter size. Maxpooling at the end. ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] base_num_filters = int(config[pref + '_num_filters']) assert len(filter_width_list) == numConvLayer convs = [] fmlist = [] last_fm = input_len for i in range(numConvLayer): fw = filter_width_list[i] num_feature_map = last_fm - fw + 1 #39 conv = Convolutional(image_size=(last_fm, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), base_num_filters * fw), num_channels=1) fmlist.append(num_feature_map) last_fm = num_feature_map embedding_size = conv.num_filters convs.append(conv) initialize(convs) for i, conv in enumerate(convs): conv.name = pref + '_conv' + str(i) conv_input = conv.apply(conv_input) conv_input = conv_input.flatten().reshape( (conv_input.shape[0], 1, fmlist[i], conv.num_filters)) lastconv = conv lastconv_out = conv_input pool_layer = MaxPooling(pooling_size=(last_fm, 1)) pool_layer.name = pref + '_pool_' + str(fw) act = Rectifier() act.name = 'act_' + str(fw) outpool = act.apply(pool_layer.apply(lastconv_out).flatten(2)) return outpool, lastconv.num_filters
def build_model(Zs): # first block z1 = conv_block(Zs['Z1'], 8, (3, 3), (3, 256, 256), '1') z2 = conv_block(Zs['Z2'], 8, (3, 3), (3, 128, 128), '2') z3 = conv_block(Zs['Z3'], 8, (3, 3), (3, 64, 64), '3') z4 = conv_block(Zs['Z4'], 8, (3, 3), (3, 32, 32), '4') z5 = conv_block(Zs['Z5'], 8, (3, 3), (3, 16, 16), '5') # merge 4 & 5 m45 = join(z5, z4, 8, 8, (32, 32), '45') # merge 3 & 45 c45 = conv_block(m45, 16, (3, 3), (16, 32, 32), '45') m345 = join(c45, z3, 16, 8, (64, 64)) # merge 2 & 345 c345 = conv_block(m345, 24, (3, 3), (24, 64, 64), '345') m2345 = join(c345, z2, 24, 8, (128, 128), '345') # merge 1 & 2345 c2345 = conv_block(m2345, 32, (3, 3), (32, 128, 128), '2345') m12345 = join(c2345, z1, 32, 8, (256, 256), '2345') last_conv_block = conv_block(m12345, 40, (3, 3), (40, 256, 256), '12345') conv_out = Convolutional((1, 1), 3, 40, image_size=(256, 256), biases_init=Uniform(width=.1), weights_init=Uniform(width=.3), use_bias=True, name='conv_out') conv_out.initialize() conv_out_output = conv_out.apply(last_conv_block) mean = tensor.addbroadcast( theano.shared(MEAN_VALUES).astype('float32'), 0, 2, 3) return 128. * (1 + conv_out_output) - mean
def __init__(self, filter_size, num_filters, num_channels, batch_size=None, mid_noise=False, out_noise=False, tied_noise=False, tied_sigma=False, noise_rate=None, noise_batch_size=None, prior_noise_level=None, image_size=(None, None), step=(1, 1), **kwargs): self.filter_size = filter_size self.num_filters = num_filters self.batch_size = batch_size self.num_channels = num_channels self.image_size = image_size self.mid_noise = mid_noise self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate self.step = step self.border_mode = 'half' self.tied_biases = True depth = 2 self.b0 = SpatialBatchNormalization(name='b0') self.r0 = Rectifier(name='r0') self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if mid_noise else None) self.c0 = Convolutional(name='c0') self.b1 = SpatialBatchNormalization(name='b1') self.r1 = Rectifier(name='r1') self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if out_noise else None) self.c1 = Convolutional(name='c1') kwargs.setdefault('children', []).extend([c for c in [ self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1, self.n1] if c is not None]) super(ResidualConvolutional, self).__init__(**kwargs)
def create_yy_cnn(numConvLayer, conv_input, embedding_size, input_len, config, pref): ''' CNN with several layers of convolution, each with specific filter size. Maxpooling at the end. ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] base_num_filters = int(config[pref + '_num_filters']) assert len(filter_width_list) == numConvLayer convs = []; fmlist = [] last_fm = input_len for i in range(numConvLayer): fw = filter_width_list[i] num_feature_map = last_fm - fw + 1 #39 conv = Convolutional( image_size=(last_fm, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), base_num_filters * fw), num_channels=1 ) fmlist.append(num_feature_map) last_fm = num_feature_map embedding_size = conv.num_filters convs.append(conv) initialize(convs) for i, conv in enumerate(convs): conv.name = pref+'_conv' + str(i) conv_input = conv.apply(conv_input) conv_input = conv_input.flatten().reshape((conv_input.shape[0], 1, fmlist[i], conv.num_filters)) lastconv = conv lastconv_out = conv_input pool_layer = MaxPooling( pooling_size=(last_fm,1) ) pool_layer.name = pref+'_pool_' + str(fw) act = Rectifier(); act.name = 'act_' + str(fw) outpool = act.apply(pool_layer.apply(lastconv_out).flatten(2)) return outpool, lastconv.num_filters
def __init__(self, input_dim, noise_batch_size=None, noise_rate=None, tied_noise=False, tied_sigma=False, prior_mean=0, prior_noise_level=0, **kwargs): self.mask = Convolutional(name='mask') self.flatten = GlobalAverageFlattener() if tied_sigma else None children = list(p for p in [self.mask, self.flatten] if p is not None) kwargs.setdefault('children', []).extend(children) super(SpatialNoise, self).__init__(**kwargs) self.input_dim = input_dim self.tied_noise = tied_noise self.tied_sigma = tied_sigma self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate if noise_rate is not None else 1.0 self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level self._training_mode = []
def __init__(self, activation, filter_size, num_filters, num_channels, batch_size=None, image_size=None, step=(1, 1), border_mode='valid', **kwargs): self.convolution = Convolutional() self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.batch_size = batch_size self.image_size = image_size self.step = step self.border_mode = border_mode super(ConvolutionalActivation, self).__init__( application_methods=[self.convolution.apply, activation], **kwargs)
class ConvolutionalActivation(Sequence, Initializable): """A convolution followed by an activation function. Parameters ---------- activation : :class:`.BoundApplication` The application method to apply after convolution (i.e. the nonlinear activation function) See Also -------- :class:`Convolutional` for the other parameters. """ @lazy(allocation=['filter_size', 'num_filters', 'num_channels']) def __init__(self, activation, filter_size, num_filters, num_channels, batch_size=None, image_size=None, step=(1, 1), border_mode='valid', **kwargs): self.convolution = Convolutional() self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.batch_size = batch_size self.image_size = image_size self.step = step self.border_mode = border_mode super(ConvolutionalActivation, self).__init__( application_methods=[self.convolution.apply, activation], **kwargs) def _push_allocation_config(self): for attr in ['filter_size', 'num_filters', 'step', 'border_mode', 'batch_size', 'num_channels', 'image_size']: setattr(self.convolution, attr, getattr(self, attr)) def get_dim(self, name): # TODO The name of the activation output doesn't need to be `output` return self.convolution.get_dim(name)
class NoisyConvolutional(Initializable, Feedforward, Random): """Convolutional transformation sent through a learned noisy channel. Parameters (same as Convolutional) """ @lazy(allocation=[ 'filter_size', 'num_filters', 'num_channels', 'noise_batch_size']) def __init__(self, filter_size, num_filters, num_channels, noise_batch_size, image_size=(None, None), step=(1, 1), border_mode='valid', tied_biases=True, prior_mean=0, prior_noise_level=0, **kwargs): self.convolution = Convolutional() self.mask = Convolutional(name='mask') children = [self.convolution, self.mask] kwargs.setdefault('children', []).extend(children) super(NoisyConvolutional, self).__init__(**kwargs) self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.noise_batch_size = noise_batch_size self.image_size = image_size self.step = step self.border_mode = border_mode self.tied_biases = tied_biases self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level def _push_allocation_config(self): self.convolution.filter_size = self.filter_size self.convolution.num_filters = self.num_filters self.convolution.num_channels = self.num_channels # self.convolution.batch_size = self.batch_size self.convolution.image_size = self.image_size self.convolution.step = self.step self.convolution.border_mode = self.border_mode self.convolution.tied_biases = self.tied_biases self.mask.filter_size = (1, 1) self.mask.num_filters = self.num_filters self.mask.num_channels = self.num_filters # self.mask.batch_size = self.batch_size self.mask.image_size = self.convolution.get_dim('output')[1:] # self.mask.step = self.step # self.mask.border_mode = self.border_mode self.mask.tied_biases = self.tied_biases def _allocate(self): out_shape = self.convolution.get_dim('output') N = shared_floatx_zeros((self.noise_batch_size,) + out_shape, name='N') add_role(N, NOISE) self.parameters.append(N) @application(inputs=['input_'], outputs=['output']) def apply(self, input_, application_call): """Apply the linear transformation followed by masking with noise. Parameters ---------- input_ : :class:`~tensor.TensorVariable` The input on which to apply the transformations Returns ------- output : :class:`~tensor.TensorVariable` The transformed input """ from theano.printing import Print pre_noise = self.convolution.apply(input_) # noise_level = self.mask.apply(input_) noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(pre_noise), -16, 16)) noise_level = copy_and_tag_noise( noise_level, self, LOG_SIGMA, 'log_sigma') # Allow incomplete batches by just taking the noise that is needed noise = self.parameters[0][:noise_level.shape[0], :, :, :] # noise = self.theano_rng.normal(noise_level.shape) kl = ( self.prior_noise_level - noise_level + 0.5 * ( tensor.exp(2 * noise_level) + (pre_noise - self.prior_mean) ** 2 ) / tensor.exp(2 * self.prior_noise_level) - 0.5 ) application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits') return pre_noise + tensor.exp(noise_level) * noise def get_dim(self, name): if name == 'input_': return self.convolution.get_dim(name) if name == 'output': return self.convolution.get_dim(name) if name == 'nits': return self.convolution.get_dim('output') return super(NoisyConvolutional, self).get_dim(name) @property def num_output_channels(self): return self.num_filters
class SpatialNoise(NoiseLayer, Initializable, Random): """A learned noise layer. """ @lazy(allocation=['input_dim']) def __init__(self, input_dim, noise_batch_size=None, noise_rate=None, tied_noise=False, tied_sigma=False, prior_mean=0, prior_noise_level=0, **kwargs): self.mask = Convolutional(name='mask') self.flatten = GlobalAverageFlattener() if tied_sigma else None children = list(p for p in [self.mask, self.flatten] if p is not None) kwargs.setdefault('children', []).extend(children) super(SpatialNoise, self).__init__(**kwargs) self.input_dim = input_dim self.tied_noise = tied_noise self.tied_sigma = tied_sigma self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate if noise_rate is not None else 1.0 self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level self._training_mode = [] def _push_allocation_config(self): self.mask.filter_size = (1, 1) self.mask.num_filters = self.num_channels self.mask.num_channels = self.num_channels self.mask.image_size = self.image_size def _allocate(self): if self.noise_batch_size is not None: if self.tied_noise: N = shared_floatx_zeros( (self.noise_batch_size, self.input_dim[0]), name='N') else: N = shared_floatx_zeros( (self.noise_batch_size,) + self.input_dim, name='N') add_role(N, NOISE) self.parameters.append(N) @application(inputs=['input_'], outputs=['output']) def apply(self, input_, application_call): """Apply the linear transformation followed by masking with noise. Parameters ---------- input_ : :class:`~tensor.TensorVariable` The input on which to apply the transformations Returns ------- output : :class:`~tensor.TensorVariable` The transformed input """ # When not in training mode, turn off noise if not self._training_mode: return input_ if self.tied_sigma: average = tensor.shape_padright(self.flatten.apply(input_), 2) noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(average), -16, 16)) noise_level = tensor.patternbroadcast(noise_level, (False, False, True, True)) noise_level = copy_and_tag_noise( noise_level, self, LOG_SIGMA, 'log_sigma') else: average = input_ noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(input_), -16, 16)) noise_level = copy_and_tag_noise( noise_level, self, LOG_SIGMA, 'log_sigma') # Allow incomplete batches by just taking the noise that is needed if self.tied_noise: if self.noise_batch_size is not None: noise = self.parameters[0][:input_.shape[0], :] else: noise = self.theano_rng.normal(input_.shape[0:2]) noise = tensor.shape_padright(2) else: if self.noise_batch_size is not None: noise = self.parameters[0][:input_.shape[0], :, :, :] else: noise = self.theano_rng.normal(input_.shape) kl = ( self.prior_noise_level - noise_level + 0.5 * ( tensor.exp(2 * noise_level) + (average - self.prior_mean) ** 2 ) / tensor.exp(2 * self.prior_noise_level) - 0.5 ) application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits') return input_ + self.noise_rate * tensor.exp(noise_level) * noise # Needed for the Feedforward interface. @property def output_dim(self): return self.input_dim # The following properties allow for BatchNormalization bricks # to be used directly inside of a ConvolutionalSequence. @property def image_size(self): return self.input_dim[-2:] @image_size.setter def image_size(self, value): if not isinstance(self.input_dim, collections.Sequence): self.input_dim = (None,) + tuple(value) else: self.input_dim = (self.input_dim[0],) + tuple(value) @property def num_channels(self): return self.input_dim[0] @num_channels.setter def num_channels(self, value): if not isinstance(self.input_dim, collections.Sequence): self.input_dim = (value,) + (None, None) else: self.input_dim = (value,) + self.input_dim[-2:] def get_dim(self, name): if name in ('input', 'output'): return self.input_dim else: raise KeyError @property def num_output_channels(self): return self.num_channels
from theano import tensor x = tensor.matrix('features') from blocks.bricks import Linear, Rectifier, Softmax from blocks.bricks.conv import Convolutional, ConvolutionalActivation input_to_hidden = Convolutional((5,5), 32, 1,border_mode='same') h = Rectifier().apply(input_to_hidden.apply(x)) hidden_to_output = Linear(name='hidden_to_output', input_dim=100, output_dim=10) y_hat = Softmax().apply(hidden_to_output.apply(h)) y = tensor.lmatrix('targets') from blocks.bricks.cost import CategoricalCrossEntropy cost = CategoricalCrossEntropy().apply(y.flatten(), y_hat) from blocks.bricks import WEIGHT from blocks.graph import ComputationGraph from blocks.filter import VariableFilter cg = ComputationGraph(cost) W1, W2 = VariableFilter(roles=[WEIGHT])(cg.variables) cost = cost + 0.005 * (W1 ** 2).sum() + 0.005 * (W2 ** 2).sum() cost.name = 'cost_with_regularization' from blocks.bricks import MLP mlp = MLP(activations=[Rectifier(), Softmax()], dims=[784, 100, 10]).apply(x) from blocks.initialization import IsotropicGaussian, Constant input_to_hidden.weights_init = hidden_to_output.weights_init = IsotropicGaussian(0.01) input_to_hidden.biases_init = hidden_to_output.biases_init = Constant(0) input_to_hidden.initialize()
from cuboid.bricks import BatchNormalizationConv, BatchNormalization, Dropout from cuboid.extensions import LogToFile from blocks.filter import VariableFilter, get_brick from blocks.model import Model from blocks.graph import ComputationGraph X = T.matrix("features") o = X.reshape((X.shape[0], 1, 28, 28)) l = Convolutional(filter_size=(5, 5), num_filters=32, num_channels=1, image_size=(28,28), weights_init=IsotropicGaussian(std=0.01), biases_init=IsotropicGaussian(std=0.01, mean=1.0), use_bias=True, border_mode="valid", step=(1,1)) l.initialize() o = l.apply(o) l = BatchNormalizationConv(input_shape=l.get_dim("output"), B_init=IsotropicGaussian(std=0.01), Y_init=IsotropicGaussian(std=0.01)) l.initialize() o = l.apply(o) o = Rectifier().apply(o)
dropout_locs = [] # CONVOLUTION LAYERS conv_in = inputt[:, None, :, None] conv_in_channels = 1 conv_in_mask = input_mask cb = [] for i, p in enumerate(convs): # Convolution bricks conv = Convolutional( filter_size=(p["filter_size"], 1), # step=(p['stride'],1), num_filters=p["nfilter"], num_channels=conv_in_channels, batch_size=batch_size, border_mode="valid", tied_biases=True, name="conv%d" % i, ) cb.append(conv) maxpool = MaxPooling(pooling_size=(p["pool_stride"], 1), name="mp%d" % i) conv_out = conv.apply(conv_in)[:, :, :: p["stride"], :] conv_out = maxpool.apply(conv_out) if p["normalize"]: conv_out_mean = conv_out.mean(axis=2).mean(axis=0) conv_out_var = ((conv_out - conv_out_mean[None, :, None, :]) ** 2).mean(axis=2).mean(axis=0).sqrt() conv_out = (conv_out - conv_out_mean[None, :, None, :]) / conv_out_var[None, :, None, :] if p["activation"] is not None: conv_out = p["activation"].apply(conv_out)
def __init__(self, *args, **kwargs): self.mask_type = kwargs.pop('mask_type', None) Convolutional.__init__(self, *args, **kwargs)
class ResidualConvolutional(Initializable): @lazy(allocation=['filter_size', 'num_filters', 'num_channels']) def __init__(self, filter_size, num_filters, num_channels, batch_size=None, mid_noise=False, out_noise=False, tied_noise=False, tied_sigma=False, noise_rate=None, noise_batch_size=None, prior_noise_level=None, image_size=(None, None), step=(1, 1), **kwargs): self.filter_size = filter_size self.num_filters = num_filters self.batch_size = batch_size self.num_channels = num_channels self.image_size = image_size self.mid_noise = mid_noise self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate self.step = step self.border_mode = 'half' self.tied_biases = True depth = 2 self.b0 = SpatialBatchNormalization(name='b0') self.r0 = Rectifier(name='r0') self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if mid_noise else None) self.c0 = Convolutional(name='c0') self.b1 = SpatialBatchNormalization(name='b1') self.r1 = Rectifier(name='r1') self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if out_noise else None) self.c1 = Convolutional(name='c1') kwargs.setdefault('children', []).extend([c for c in [ self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1, self.n1] if c is not None]) super(ResidualConvolutional, self).__init__(**kwargs) def get_dim(self, name): if name == 'input_': return ((self.num_channels,) + self.image_size) if name == 'output': return self.c1.get_dim(name) return super(ResidualConvolutionalUnit, self).get_dim(name) @property def num_output_channels(self): return self.num_filters def _push_allocation_config(self): self.b0.input_dim = self.get_dim('input_') self.b0.push_allocation_config() if self.r0: self.r0.push_allocation_config() if self.n0: self.n0.noise_batch_size = self.noise_batch_size self.n0.num_channels = self.num_channels self.n0.image_size = self.image_size self.c0.filter_size = self.filter_size self.c0.batch_size = self.batch_size self.c0.num_channels = self.num_channels self.c0.num_filters = self.num_filters self.c0.border_mode = self.border_mode self.c0.image_size = self.image_size self.c0.step = self.step self.c0.use_bias = False self.c0.push_allocation_config() c0_shape = self.c0.get_dim('output') self.b1.input_dim = c0_shape self.b1.push_allocation_config() self.r1.push_allocation_config() if self.n1: self.n1.noise_batch_size = self.noise_batch_size self.n1.num_channels = self.num_filters self.n1.image_size = c0_shape[1:] self.c1.filter_size = self.filter_size self.c1.batch_size = self.batch_size self.c1.num_channels = self.num_filters self.c1.num_filters = self.num_filters self.c1.border_mode = self.border_mode self.c1.image_size = c0_shape[1:] self.c1.step = (1, 1) self.c1.use_bias = False self.c1.push_allocation_config() @application(inputs=['input_'], outputs=['output']) def apply(self, input_): shortcut = input_ # Batchnorm, then Relu, then Convolution first_conv = self.b0.apply(input_) first_conv = self.r0.apply(first_conv) if self.n0: first_conv = self.n0.apply(first_conv) first_conv = self.c0.apply(first_conv) # Batchnorm, then Relu, then Convolution (second time) second_conv = self.b1.apply(first_conv) second_conv = self.r1.apply(second_conv) if self.n1: second_conv = self.n1.apply(second_conv) residual = second_conv # Apply stride and zero-padding to match shortcut to output if self.step and self.step != (1, 1): shortcut = shortcut[:,:,::self.step[0],::self.step[1]] if self.num_filters > self.num_channels: padshape = (residual.shape[0], self.num_filters - self.num_channels, residual.shape[2], residual.shape[3]) shortcut = tensor.concatenate( [shortcut, tensor.zeros(padshape, dtype=residual.dtype)], axis=1) elif self.num_filters < self.num_channels: shortcut = shortcut[:,:self.num_channels,:,:] response = shortcut + residual return response