class ConvolutionalActivation(Initializable): """A convolution followed by an activation function. Parameters ---------- activation : :class:`.BoundApplication` The application method to apply after convolution (i.e. the nonlinear activation function) See Also -------- :class:`Convolutional` : For the documentation of other parameters. """ @lazy(allocation=['filter_size', 'num_filters', 'num_channels']) def __init__(self, activation, filter_size, num_filters, num_channels, batch_size=None, image_size=None, step=(1, 1), border_mode='valid', tied_biases=False, **kwargs): self.convolution = Convolutional(name='conv'+ kwargs['name']) self.bn = BatchNorm(name='bn'+ kwargs['name']) self.activation = activation self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.batch_size = batch_size self.image_size = image_size self.step = step self.border_mode = border_mode self.tied_biases = tied_biases super(ConvolutionalActivation, self).__init__(**kwargs) self.children = [self.convolution, self.bn, self.activation] def _push_allocation_config(self): for attr in ['filter_size', 'num_filters', 'step', 'border_mode', 'batch_size', 'num_channels', 'image_size', 'tied_biases']: setattr(self.convolution, attr, getattr(self, attr)) setattr(self.bn, 'input_dim', self.num_filters) def get_dim(self, name): # TODO The name of the activation output doesn't need to be `output` return self.convolution.get_dim(name) def apply(self, input_): out = self.convolution.apply(input_) out = self.bn.apply(out) out = self.activation.apply(out) return out def inference(self, input_): out = self.convolution.apply(input_) out = self.bn.inference(out) out = self.activation.apply(out) return out
def test_convolutional(): x = tensor.tensor4("x") num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional( filter_size, num_filters, num_channels, image_size=(17, 13), weights_init=Constant(1.0), biases_init=Constant(5.0), ) conv.initialize() y = conv.apply(x) func = function([x], y) x_val = numpy.ones((batch_size, num_channels, 17, 13), dtype=theano.config.floatX) assert_allclose( func(x_val), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 15, 11)) + 5 ) conv.image_size = (17, 13) conv.batch_size = 2 # This should have effect on get_dim assert conv.get_dim("output") == (num_filters, 15, 11)
def test_tied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.), tied_biases=True) conv.initialize() y = conv.apply(x) func = function([x], y) # Tied biases allows to pass images of different sizes x_val_1 = numpy.ones((batch_size, num_channels, 10, 12), dtype=theano.config.floatX) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) assert_allclose(func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 8, 10)) + 2) assert_allclose(func(x_val_2), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 21, 17)) + 2)
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] print filter_width_list num_filters = int(config[pref + '_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional(image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1) totfilters += conv.num_filters initialize2(conv, num_feature_map) conv.name = pref + 'conv_' + str(fw) convout = conv.apply(layer0_input) pool_layer = MaxPooling(pooling_size=(num_feature_map, 1)) pool_layer.name = pref + 'pool_' + str(fw) act = Rectifier() act.name = pref + 'act_' + str(fw) outpool = act.apply(pool_layer.apply(convout)).flatten(2) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def test_no_input_size(): # suppose x is outputted by some RNN x = tensor.tensor4('x') filter_size = (1, 3) num_filters = 2 num_channels = 5 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=True, weights_init=Constant(1.), biases_init=Constant(1.)) c.initialize() out = c.apply(x) assert c.get_dim('output') == (2, None, None) assert out.ndim == 4 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=False, weights_init=Constant(1.), biases_init=Constant(1.)) assert_raises_regexp(ValueError, 'Cannot infer bias size \S+', c.initialize)
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] print filter_width_list num_filters = int(config[pref+'_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional( image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1 ) totfilters += conv.num_filters initialize2(conv, num_feature_map) conv.name = pref + 'conv_' + str(fw) convout = conv.apply(layer0_input) pool_layer = MaxPooling( pooling_size=(num_feature_map,1) ) pool_layer.name = pref + 'pool_' + str(fw) act = Rectifier() act.name = pref + 'act_' + str(fw) outpool = act.apply(pool_layer.apply(convout)).flatten(2) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def test_untied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.), image_size=(28, 30), tied_biases=False) conv.initialize() y = conv.apply(x) func = function([x], y) # Untied biases provide a bias for every individual output assert_allclose(conv.b.eval().shape, (3, 26, 28)) # Untied biases require images of a specific size x_val_1 = numpy.ones((batch_size, num_channels, 28, 30), dtype=theano.config.floatX) assert_allclose(func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 26, 28)) + 2) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) def wrongsize(): func(x_val_2) assert_raises_regexp(AssertionError, 'AbstractConv shape mismatch', wrongsize)
def test_tied_biases(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) # Tied biases are the default conv = Convolutional(filter_size, num_filters, num_channels, weights_init=Constant(1.), biases_init=Constant(2.)) conv.initialize() y = conv.apply(x) func = function([x], y) # Tied biases only provide one bias for each filter assert_allclose(conv.b.eval().shape, (3, )) # Tied biases allows to pass images of different sizes x_val_1 = numpy.ones((batch_size, num_channels, 10, 12), dtype=theano.config.floatX) x_val_2 = numpy.ones((batch_size, num_channels, 23, 19), dtype=theano.config.floatX) assert_allclose( func(x_val_1), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 8, 10)) + 2) assert_allclose( func(x_val_2), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 21, 17)) + 2)
def test_convolutional(): x = tensor.tensor4('x') num_channels = 4 num_filters = 3 batch_size = 5 filter_size = (3, 3) conv = Convolutional(filter_size, num_filters, num_channels, image_size=(17, 13), weights_init=Constant(1.), biases_init=Constant(5.)) conv.initialize() y = conv.apply(x) func = function([x], y) x_val = numpy.ones((batch_size, num_channels, 17, 13), dtype=theano.config.floatX) assert_allclose( func(x_val), numpy.prod(filter_size) * num_channels * numpy.ones( (batch_size, num_filters, 15, 11)) + 5) conv.image_size = (17, 13) conv.batch_size = 2 # This should have effect on get_dim assert conv.get_dim('output') == (num_filters, 15, 11)
def create_KmaxPooling_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' k = int(config[pref + '_kpool']) filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] print filter_width_list num_filters = int(config[pref+'_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 print input_len, embedding_size, num_filters for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional( image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1 ) totfilters += conv.num_filters * k # initialize2(conv, num_feature_map) initialize([conv]) conv.name = pref + 'conv_' + str(fw) layer0_input = debug_print(layer0_input, 'inp', False) convout = conv.apply(layer0_input) convout = debug_print(convout, 'convout', False) kpoolout = KmaxPooling(convout, k).apply().flatten(2) kpoolout = debug_print(kpoolout, 'poolout', False) if i == 0: outpools = kpoolout else: outpools = T.concatenate([outpools, kpoolout], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with the same filtersize ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] print filter_width_list num_filters = int(config[pref + '_num_filters']) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional(filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(input_len, embedding_size), name="conv" + str(fw)) pooling = MaxPooling((num_feature_map, 1), name="pool" + str(fw)) initialize([conv]) totfilters += num_filters outpool = Flattener(name="flat" + str(fw)).apply( Rectifier(name=pref + 'act_' + str(fw)).apply( pooling.apply(conv.apply(layer0_input)))) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def test_no_input_size(): # suppose x is outputted by some RNN x = tensor.tensor4('x') filter_size = (1, 3) num_filters = 2 num_channels = 5 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=True, weights_init=Constant(1.), biases_init=Constant(1.)) c.initialize() out = c.apply(x) assert c.get_dim('output') == (2, None, None) assert out.ndim == 4 c = Convolutional(filter_size, num_filters, num_channels, tied_biases=False, weights_init=Constant(1.), biases_init=Constant(1.)) assert_raises_regexp(ValueError, 'Cannot infer bias size \S+', c.initialize)
def build_model(Zs): # first block z1 = conv_block(Zs['Z1'], 8, (3, 3), (3, 256, 256), '1') z2 = conv_block(Zs['Z2'], 8, (3, 3), (3, 128, 128), '2') z3 = conv_block(Zs['Z3'], 8, (3, 3), (3, 64, 64), '3') z4 = conv_block(Zs['Z4'], 8, (3, 3), (3, 32, 32), '4') z5 = conv_block(Zs['Z5'], 8, (3, 3), (3, 16, 16), '5') # merge 4 & 5 m45 = join(z5, z4, 8, 8, (32, 32), '45') # merge 3 & 45 c45 = conv_block(m45, 16, (3, 3), (16, 32, 32), '45') m345 = join(c45, z3, 16, 8, (64, 64)) # merge 2 & 345 c345 = conv_block(m345, 24, (3, 3), (24, 64, 64), '345') m2345 = join(c345, z2, 24, 8, (128, 128), '345') # merge 1 & 2345 c2345 = conv_block(m2345, 32, (3, 3), (32, 128, 128), '2345') m12345 = join(c2345, z1, 32, 8, (256, 256), '2345') last_conv_block = conv_block(m12345, 40, (3, 3), (40, 256, 256), '12345') conv_out = Convolutional((1, 1), 3, 40, image_size=(256, 256), biases_init=Uniform(width=.1), weights_init=Uniform(width=.3), use_bias=True, name='conv_out') conv_out.initialize() conv_out_output = conv_out.apply(last_conv_block) mean = tensor.addbroadcast( theano.shared(MEAN_VALUES).astype('float32'), 0, 2, 3) return 128. * (1 + conv_out_output) - mean
def create_yy_cnn(numConvLayer, conv_input, embedding_size, input_len, config, pref): ''' CNN with several layers of convolution, each with specific filter size. Maxpooling at the end. ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] base_num_filters = int(config[pref + '_num_filters']) assert len(filter_width_list) == numConvLayer convs = [] fmlist = [] last_fm = input_len for i in range(numConvLayer): fw = filter_width_list[i] num_feature_map = last_fm - fw + 1 #39 conv = Convolutional(image_size=(last_fm, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), base_num_filters * fw), num_channels=1) fmlist.append(num_feature_map) last_fm = num_feature_map embedding_size = conv.num_filters convs.append(conv) initialize(convs) for i, conv in enumerate(convs): conv.name = pref + '_conv' + str(i) conv_input = conv.apply(conv_input) conv_input = conv_input.flatten().reshape( (conv_input.shape[0], 1, fmlist[i], conv.num_filters)) lastconv = conv lastconv_out = conv_input pool_layer = MaxPooling(pooling_size=(last_fm, 1)) pool_layer.name = pref + '_pool_' + str(fw) act = Rectifier() act.name = 'act_' + str(fw) outpool = act.apply(pool_layer.apply(lastconv_out).flatten(2)) return outpool, lastconv.num_filters
def create_yy_cnn(numConvLayer, conv_input, embedding_size, input_len, config, pref): ''' CNN with several layers of convolution, each with specific filter size. Maxpooling at the end. ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] base_num_filters = int(config[pref + '_num_filters']) assert len(filter_width_list) == numConvLayer convs = []; fmlist = [] last_fm = input_len for i in range(numConvLayer): fw = filter_width_list[i] num_feature_map = last_fm - fw + 1 #39 conv = Convolutional( image_size=(last_fm, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), base_num_filters * fw), num_channels=1 ) fmlist.append(num_feature_map) last_fm = num_feature_map embedding_size = conv.num_filters convs.append(conv) initialize(convs) for i, conv in enumerate(convs): conv.name = pref+'_conv' + str(i) conv_input = conv.apply(conv_input) conv_input = conv_input.flatten().reshape((conv_input.shape[0], 1, fmlist[i], conv.num_filters)) lastconv = conv lastconv_out = conv_input pool_layer = MaxPooling( pooling_size=(last_fm,1) ) pool_layer.name = pref+'_pool_' + str(fw) act = Rectifier(); act.name = 'act_' + str(fw) outpool = act.apply(pool_layer.apply(lastconv_out).flatten(2)) return outpool, lastconv.num_filters
def create_KmaxPooling_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' k = int(config[pref + '_kpool']) filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] print filter_width_list num_filters = int(config[pref + '_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 print input_len, embedding_size, num_filters for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional(image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1) totfilters += conv.num_filters * k # initialize2(conv, num_feature_map) initialize([conv]) conv.name = pref + 'conv_' + str(fw) layer0_input = debug_print(layer0_input, 'inp', False) convout = conv.apply(layer0_input) convout = debug_print(convout, 'convout', False) kpoolout = KmaxPooling(convout, k).apply().flatten(2) kpoolout = debug_print(kpoolout, 'poolout', False) if i == 0: outpools = kpoolout else: outpools = T.concatenate([outpools, kpoolout], axis=1) name_rep_len = totfilters return outpools, name_rep_len
class ResidualConvolutional(Initializable): @lazy(allocation=['filter_size', 'num_filters', 'num_channels']) def __init__(self, filter_size, num_filters, num_channels, batch_size=None, mid_noise=False, out_noise=False, tied_noise=False, tied_sigma=False, noise_rate=None, noise_batch_size=None, prior_noise_level=None, image_size=(None, None), step=(1, 1), **kwargs): self.filter_size = filter_size self.num_filters = num_filters self.batch_size = batch_size self.num_channels = num_channels self.image_size = image_size self.mid_noise = mid_noise self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate self.step = step self.border_mode = 'half' self.tied_biases = True depth = 2 self.b0 = SpatialBatchNormalization(name='b0') self.r0 = Rectifier(name='r0') self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if mid_noise else None) self.c0 = Convolutional(name='c0') self.b1 = SpatialBatchNormalization(name='b1') self.r1 = Rectifier(name='r1') self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if out_noise else None) self.c1 = Convolutional(name='c1') kwargs.setdefault('children', []).extend([ c for c in [ self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1, self.n1 ] if c is not None ]) super(ResidualConvolutional, self).__init__(**kwargs) def get_dim(self, name): if name == 'input_': return ((self.num_channels, ) + self.image_size) if name == 'output': return self.c1.get_dim(name) return super(ResidualConvolutionalUnit, self).get_dim(name) @property def num_output_channels(self): return self.num_filters def _push_allocation_config(self): self.b0.input_dim = self.get_dim('input_') self.b0.push_allocation_config() if self.r0: self.r0.push_allocation_config() if self.n0: self.n0.noise_batch_size = self.noise_batch_size self.n0.num_channels = self.num_channels self.n0.image_size = self.image_size self.c0.filter_size = self.filter_size self.c0.batch_size = self.batch_size self.c0.num_channels = self.num_channels self.c0.num_filters = self.num_filters self.c0.border_mode = self.border_mode self.c0.image_size = self.image_size self.c0.step = self.step self.c0.use_bias = False self.c0.push_allocation_config() c0_shape = self.c0.get_dim('output') self.b1.input_dim = c0_shape self.b1.push_allocation_config() self.r1.push_allocation_config() if self.n1: self.n1.noise_batch_size = self.noise_batch_size self.n1.num_channels = self.num_filters self.n1.image_size = c0_shape[1:] self.c1.filter_size = self.filter_size self.c1.batch_size = self.batch_size self.c1.num_channels = self.num_filters self.c1.num_filters = self.num_filters self.c1.border_mode = self.border_mode self.c1.image_size = c0_shape[1:] self.c1.step = (1, 1) self.c1.use_bias = False self.c1.push_allocation_config() @application(inputs=['input_'], outputs=['output']) def apply(self, input_): shortcut = input_ # Batchnorm, then Relu, then Convolution first_conv = self.b0.apply(input_) first_conv = self.r0.apply(first_conv) if self.n0: first_conv = self.n0.apply(first_conv) first_conv = self.c0.apply(first_conv) # Batchnorm, then Relu, then Convolution (second time) second_conv = self.b1.apply(first_conv) second_conv = self.r1.apply(second_conv) if self.n1: second_conv = self.n1.apply(second_conv) residual = second_conv # Apply stride and zero-padding to match shortcut to output if self.step and self.step != (1, 1): shortcut = shortcut[:, :, ::self.step[0], ::self.step[1]] if self.num_filters > self.num_channels: padshape = (residual.shape[0], self.num_filters - self.num_channels, residual.shape[2], residual.shape[3]) shortcut = tensor.concatenate( [shortcut, tensor.zeros(padshape, dtype=residual.dtype)], axis=1) elif self.num_filters < self.num_channels: shortcut = shortcut[:, :self.num_channels, :, :] response = shortcut + residual return response
X = T.matrix("features") o = X.reshape((X.shape[0], 1, 28, 28)) l = Convolutional(filter_size=(5, 5), num_filters=32, num_channels=1, image_size=(28, 28), weights_init=IsotropicGaussian(std=0.01), biases_init=IsotropicGaussian(std=0.01, mean=1.0), use_bias=True, border_mode="valid", step=(1, 1)) l.initialize() o = l.apply(o) l = BatchNormalizationConv(input_shape=l.get_dim("output"), B_init=IsotropicGaussian(std=0.01), Y_init=IsotropicGaussian(std=0.01)) l.initialize() o = l.apply(o) o = Rectifier().apply(o) l = MaxPooling(pooling_size=(2, 2), step=(2, 2), input_dim=l.get_dim("output")) l.initialize() o = l.apply(o) #ll = Dropout(p_drop=0.5) #ll.initialize()
acc = normalize(acc, axis=1) # set dims for convolution eeg = eeg.dimshuffle(0, 2, 1, 'x') acc = acc.dimshuffle(0, 2, 1, 'x') # first convolution only on eeg conv_eeg = Convolutional(filter_size=(300, 1), num_filters=20, num_channels=1, border_mode='full', tied_biases=True, name="conv_eeg") maxpool_eeg = MaxPooling(pooling_size=(5, 1), name='maxpool_eeg') # convolve eeg1 = conv_eeg.apply(eeg) # cut borders d1 = (eeg1.shape[2] - eeg.shape[2])/2 eeg1 = eeg1[:, :, d1:d1+eeg.shape[2], :] # subsample eeg1 = maxpool_eeg.apply(eeg1) # activation eeg1 = Tanh(name='act_eeg').apply(eeg1) # second convolution only on eeg conv_eeg2 = Convolutional(filter_size=(100, 1), num_filters=40, num_channels=20, border_mode='full', tied_biases=True, name="conv_eeg2")
class ResidualConvolutional(Initializable): @lazy(allocation=['filter_size', 'num_filters', 'num_channels']) def __init__(self, filter_size, num_filters, num_channels, batch_size=None, mid_noise=False, out_noise=False, tied_noise=False, tied_sigma=False, noise_rate=None, noise_batch_size=None, prior_noise_level=None, image_size=(None, None), step=(1, 1), **kwargs): self.filter_size = filter_size self.num_filters = num_filters self.batch_size = batch_size self.num_channels = num_channels self.image_size = image_size self.mid_noise = mid_noise self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate self.step = step self.border_mode = 'half' self.tied_biases = True depth = 2 self.b0 = SpatialBatchNormalization(name='b0') self.r0 = Rectifier(name='r0') self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if mid_noise else None) self.c0 = Convolutional(name='c0') self.b1 = SpatialBatchNormalization(name='b1') self.r1 = Rectifier(name='r1') self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate, tied_noise=tied_noise, tied_sigma=tied_sigma, prior_noise_level=prior_noise_level) if out_noise else None) self.c1 = Convolutional(name='c1') kwargs.setdefault('children', []).extend([c for c in [ self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1, self.n1] if c is not None]) super(ResidualConvolutional, self).__init__(**kwargs) def get_dim(self, name): if name == 'input_': return ((self.num_channels,) + self.image_size) if name == 'output': return self.c1.get_dim(name) return super(ResidualConvolutionalUnit, self).get_dim(name) @property def num_output_channels(self): return self.num_filters def _push_allocation_config(self): self.b0.input_dim = self.get_dim('input_') self.b0.push_allocation_config() if self.r0: self.r0.push_allocation_config() if self.n0: self.n0.noise_batch_size = self.noise_batch_size self.n0.num_channels = self.num_channels self.n0.image_size = self.image_size self.c0.filter_size = self.filter_size self.c0.batch_size = self.batch_size self.c0.num_channels = self.num_channels self.c0.num_filters = self.num_filters self.c0.border_mode = self.border_mode self.c0.image_size = self.image_size self.c0.step = self.step self.c0.use_bias = False self.c0.push_allocation_config() c0_shape = self.c0.get_dim('output') self.b1.input_dim = c0_shape self.b1.push_allocation_config() self.r1.push_allocation_config() if self.n1: self.n1.noise_batch_size = self.noise_batch_size self.n1.num_channels = self.num_filters self.n1.image_size = c0_shape[1:] self.c1.filter_size = self.filter_size self.c1.batch_size = self.batch_size self.c1.num_channels = self.num_filters self.c1.num_filters = self.num_filters self.c1.border_mode = self.border_mode self.c1.image_size = c0_shape[1:] self.c1.step = (1, 1) self.c1.use_bias = False self.c1.push_allocation_config() @application(inputs=['input_'], outputs=['output']) def apply(self, input_): shortcut = input_ # Batchnorm, then Relu, then Convolution first_conv = self.b0.apply(input_) first_conv = self.r0.apply(first_conv) if self.n0: first_conv = self.n0.apply(first_conv) first_conv = self.c0.apply(first_conv) # Batchnorm, then Relu, then Convolution (second time) second_conv = self.b1.apply(first_conv) second_conv = self.r1.apply(second_conv) if self.n1: second_conv = self.n1.apply(second_conv) residual = second_conv # Apply stride and zero-padding to match shortcut to output if self.step and self.step != (1, 1): shortcut = shortcut[:,:,::self.step[0],::self.step[1]] if self.num_filters > self.num_channels: padshape = (residual.shape[0], self.num_filters - self.num_channels, residual.shape[2], residual.shape[3]) shortcut = tensor.concatenate( [shortcut, tensor.zeros(padshape, dtype=residual.dtype)], axis=1) elif self.num_filters < self.num_channels: shortcut = shortcut[:,:self.num_channels,:,:] response = shortcut + residual return response
for i, p in enumerate(convs): # Convolution bricks conv = Convolutional( filter_size=(p["filter_size"], 1), # step=(p['stride'],1), num_filters=p["nfilter"], num_channels=conv_in_channels, batch_size=batch_size, border_mode="valid", tied_biases=True, name="conv%d" % i, ) cb.append(conv) maxpool = MaxPooling(pooling_size=(p["pool_stride"], 1), name="mp%d" % i) conv_out = conv.apply(conv_in)[:, :, :: p["stride"], :] conv_out = maxpool.apply(conv_out) if p["normalize"]: conv_out_mean = conv_out.mean(axis=2).mean(axis=0) conv_out_var = ((conv_out - conv_out_mean[None, :, None, :]) ** 2).mean(axis=2).mean(axis=0).sqrt() conv_out = (conv_out - conv_out_mean[None, :, None, :]) / conv_out_var[None, :, None, :] if p["activation"] is not None: conv_out = p["activation"].apply(conv_out) if p["dropout"] > 0: b = [p["activation"] if p["activation"] is not None else conv] dropout_locs.append((VariableFilter(bricks=b, name="output"), p["dropout"])) if p["skip"] is not None and len(p["skip"]) > 0: maxpooladd = MaxPooling(pooling_size=(p["stride"] * p["pool_stride"], 1), name="Mp%d" % i) skip = [] if "max" in p["skip"]: skip.append(maxpooladd.apply(conv_in)[:, :, : conv_out.shape[2], :])
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with the same filtersize ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] print filter_width_list num_filters = int(config[pref+'_num_filters']) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional( filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(input_len, embedding_size), name="conv" + str(fw)) pooling = MaxPooling((num_feature_map,1), name="pool"+str(fw)) initialize([conv]) totfilters += num_filters outpool = Flattener(name="flat"+str(fw)).apply(Rectifier(name=pref+'act_'+str(fw)).apply(pooling.apply(conv.apply(layer0_input)))) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
from theano import tensor x = tensor.matrix('features') from blocks.bricks import Linear, Rectifier, Softmax from blocks.bricks.conv import Convolutional, ConvolutionalActivation input_to_hidden = Convolutional((5,5), 32, 1,border_mode='same') h = Rectifier().apply(input_to_hidden.apply(x)) hidden_to_output = Linear(name='hidden_to_output', input_dim=100, output_dim=10) y_hat = Softmax().apply(hidden_to_output.apply(h)) y = tensor.lmatrix('targets') from blocks.bricks.cost import CategoricalCrossEntropy cost = CategoricalCrossEntropy().apply(y.flatten(), y_hat) from blocks.bricks import WEIGHT from blocks.graph import ComputationGraph from blocks.filter import VariableFilter cg = ComputationGraph(cost) W1, W2 = VariableFilter(roles=[WEIGHT])(cg.variables) cost = cost + 0.005 * (W1 ** 2).sum() + 0.005 * (W2 ** 2).sum() cost.name = 'cost_with_regularization' from blocks.bricks import MLP mlp = MLP(activations=[Rectifier(), Softmax()], dims=[784, 100, 10]).apply(x) from blocks.initialization import IsotropicGaussian, Constant input_to_hidden.weights_init = hidden_to_output.weights_init = IsotropicGaussian(0.01) input_to_hidden.biases_init = hidden_to_output.biases_init = Constant(0) input_to_hidden.initialize()
class SpatialNoise(NoiseLayer, Initializable, Random): """A learned noise layer. """ @lazy(allocation=['input_dim']) def __init__(self, input_dim, noise_batch_size=None, noise_rate=None, tied_noise=False, tied_sigma=False, prior_mean=0, prior_noise_level=0, **kwargs): self.mask = Convolutional(name='mask') self.flatten = GlobalAverageFlattener() if tied_sigma else None children = list(p for p in [self.mask, self.flatten] if p is not None) kwargs.setdefault('children', []).extend(children) super(SpatialNoise, self).__init__(**kwargs) self.input_dim = input_dim self.tied_noise = tied_noise self.tied_sigma = tied_sigma self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate if noise_rate is not None else 1.0 self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level self._training_mode = [] def _push_allocation_config(self): self.mask.filter_size = (1, 1) self.mask.num_filters = self.num_channels self.mask.num_channels = self.num_channels self.mask.image_size = self.image_size def _allocate(self): if self.noise_batch_size is not None: if self.tied_noise: N = shared_floatx_zeros( (self.noise_batch_size, self.input_dim[0]), name='N') else: N = shared_floatx_zeros( (self.noise_batch_size,) + self.input_dim, name='N') add_role(N, NOISE) self.parameters.append(N) @application(inputs=['input_'], outputs=['output']) def apply(self, input_, application_call): """Apply the linear transformation followed by masking with noise. Parameters ---------- input_ : :class:`~tensor.TensorVariable` The input on which to apply the transformations Returns ------- output : :class:`~tensor.TensorVariable` The transformed input """ # When not in training mode, turn off noise if not self._training_mode: return input_ if self.tied_sigma: average = tensor.shape_padright(self.flatten.apply(input_), 2) noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(average), -16, 16)) noise_level = tensor.patternbroadcast(noise_level, (False, False, True, True)) noise_level = copy_and_tag_noise( noise_level, self, LOG_SIGMA, 'log_sigma') else: average = input_ noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(input_), -16, 16)) noise_level = copy_and_tag_noise( noise_level, self, LOG_SIGMA, 'log_sigma') # Allow incomplete batches by just taking the noise that is needed if self.tied_noise: if self.noise_batch_size is not None: noise = self.parameters[0][:input_.shape[0], :] else: noise = self.theano_rng.normal(input_.shape[0:2]) noise = tensor.shape_padright(2) else: if self.noise_batch_size is not None: noise = self.parameters[0][:input_.shape[0], :, :, :] else: noise = self.theano_rng.normal(input_.shape) kl = ( self.prior_noise_level - noise_level + 0.5 * ( tensor.exp(2 * noise_level) + (average - self.prior_mean) ** 2 ) / tensor.exp(2 * self.prior_noise_level) - 0.5 ) application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits') return input_ + self.noise_rate * tensor.exp(noise_level) * noise # Needed for the Feedforward interface. @property def output_dim(self): return self.input_dim # The following properties allow for BatchNormalization bricks # to be used directly inside of a ConvolutionalSequence. @property def image_size(self): return self.input_dim[-2:] @image_size.setter def image_size(self, value): if not isinstance(self.input_dim, collections.Sequence): self.input_dim = (None,) + tuple(value) else: self.input_dim = (self.input_dim[0],) + tuple(value) @property def num_channels(self): return self.input_dim[0] @num_channels.setter def num_channels(self, value): if not isinstance(self.input_dim, collections.Sequence): self.input_dim = (value,) + (None, None) else: self.input_dim = (value,) + self.input_dim[-2:] def get_dim(self, name): if name in ('input', 'output'): return self.input_dim else: raise KeyError @property def num_output_channels(self): return self.num_channels
d1 = (eeg1.shape[2] - eeg.shape[2]) / 2 eeg = eeg1[:, :, d1:d1 + eeg.shape[2]:eeg_gaussian_filter_step, :] # first convolutions only on eeg eeg_channels = 1 for i, cp in enumerate(conv_eeg): bconv = Convolutional(filter_size=(cp['filter_size'], 1), num_filters=cp['num_filters'], num_channels=eeg_channels, border_mode='full', tied_biases=True, name="conv_eeg_%d" % i) bmaxpool = MaxPooling(pooling_size=(cp['pool_size'], 1), name='maxpool_eeg_%d' % i) # convolve eeg1 = bconv.apply(eeg) # cut borders d1 = (eeg1.shape[2] - eeg.shape[2]) / 2 eeg = eeg1[:, :, d1:d1 + eeg.shape[2], :] # subsample eeg = bmaxpool.apply(eeg) # normalize if cp['normalize']: eeg = normalize(eeg, axis=(0, 2)) # activation act = cp['activation'](name='act_eeg%d' % i) eeg = act.apply(eeg) # stuff bricks += [bconv, bmaxpool] eeg_channels = cp['num_filters'] if cp['dropout'] > 0:
class NoisyConvolutional(Initializable, Feedforward, Random): """Convolutional transformation sent through a learned noisy channel. Parameters (same as Convolutional) """ @lazy(allocation=[ 'filter_size', 'num_filters', 'num_channels', 'noise_batch_size' ]) def __init__(self, filter_size, num_filters, num_channels, noise_batch_size, image_size=(None, None), step=(1, 1), border_mode='valid', tied_biases=True, prior_mean=0, prior_noise_level=0, **kwargs): self.convolution = Convolutional() self.mask = Convolutional(name='mask') children = [self.convolution, self.mask] kwargs.setdefault('children', []).extend(children) super(NoisyConvolutional, self).__init__(**kwargs) self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.noise_batch_size = noise_batch_size self.image_size = image_size self.step = step self.border_mode = border_mode self.tied_biases = tied_biases self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level def _push_allocation_config(self): self.convolution.filter_size = self.filter_size self.convolution.num_filters = self.num_filters self.convolution.num_channels = self.num_channels # self.convolution.batch_size = self.batch_size self.convolution.image_size = self.image_size self.convolution.step = self.step self.convolution.border_mode = self.border_mode self.convolution.tied_biases = self.tied_biases self.mask.filter_size = (1, 1) self.mask.num_filters = self.num_filters self.mask.num_channels = self.num_filters # self.mask.batch_size = self.batch_size self.mask.image_size = self.convolution.get_dim('output')[1:] # self.mask.step = self.step # self.mask.border_mode = self.border_mode self.mask.tied_biases = self.tied_biases def _allocate(self): out_shape = self.convolution.get_dim('output') N = shared_floatx_zeros((self.noise_batch_size, ) + out_shape, name='N') add_role(N, NOISE) self.parameters.append(N) @application(inputs=['input_'], outputs=['output']) def apply(self, input_, application_call): """Apply the linear transformation followed by masking with noise. Parameters ---------- input_ : :class:`~tensor.TensorVariable` The input on which to apply the transformations Returns ------- output : :class:`~tensor.TensorVariable` The transformed input """ from theano.printing import Print pre_noise = self.convolution.apply(input_) # noise_level = self.mask.apply(input_) noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(pre_noise), -16, 16)) noise_level = copy_and_tag_noise(noise_level, self, LOG_SIGMA, 'log_sigma') # Allow incomplete batches by just taking the noise that is needed noise = self.parameters[0][:noise_level.shape[0], :, :, :] # noise = self.theano_rng.normal(noise_level.shape) kl = (self.prior_noise_level - noise_level + 0.5 * (tensor.exp(2 * noise_level) + (pre_noise - self.prior_mean)**2) / tensor.exp(2 * self.prior_noise_level) - 0.5) application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits') return pre_noise + tensor.exp(noise_level) * noise def get_dim(self, name): if name == 'input_': return self.convolution.get_dim(name) if name == 'output': return self.convolution.get_dim(name) if name == 'nits': return self.convolution.get_dim('output') return super(NoisyConvolutional, self).get_dim(name) @property def num_output_channels(self): return self.num_filters
X = T.matrix("features") o = X.reshape((X.shape[0], 1, 28, 28)) l = Convolutional(filter_size=(5, 5), num_filters=32, num_channels=1, image_size=(28,28), weights_init=IsotropicGaussian(std=0.01), biases_init=IsotropicGaussian(std=0.01, mean=1.0), use_bias=True, border_mode="valid", step=(1,1)) l.initialize() o = l.apply(o) l = BatchNormalizationConv(input_shape=l.get_dim("output"), B_init=IsotropicGaussian(std=0.01), Y_init=IsotropicGaussian(std=0.01)) l.initialize() o = l.apply(o) o = Rectifier().apply(o) l = MaxPooling(pooling_size=(2, 2), step=(2, 2), input_dim=l.get_dim("output")) l.initialize() o = l.apply(o)
class SpatialNoise(NoiseLayer, Initializable, Random): """A learned noise layer. """ @lazy(allocation=['input_dim']) def __init__(self, input_dim, noise_batch_size=None, noise_rate=None, tied_noise=False, tied_sigma=False, prior_mean=0, prior_noise_level=0, **kwargs): self.mask = Convolutional(name='mask') self.flatten = GlobalAverageFlattener() if tied_sigma else None children = list(p for p in [self.mask, self.flatten] if p is not None) kwargs.setdefault('children', []).extend(children) super(SpatialNoise, self).__init__(**kwargs) self.input_dim = input_dim self.tied_noise = tied_noise self.tied_sigma = tied_sigma self.noise_batch_size = noise_batch_size self.noise_rate = noise_rate if noise_rate is not None else 1.0 self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level self._training_mode = [] def _push_allocation_config(self): self.mask.filter_size = (1, 1) self.mask.num_filters = self.num_channels self.mask.num_channels = self.num_channels self.mask.image_size = self.image_size def _allocate(self): if self.noise_batch_size is not None: if self.tied_noise: N = shared_floatx_zeros( (self.noise_batch_size, self.input_dim[0]), name='N') else: N = shared_floatx_zeros( (self.noise_batch_size, ) + self.input_dim, name='N') add_role(N, NOISE) self.parameters.append(N) @application(inputs=['input_'], outputs=['output']) def apply(self, input_, application_call): """Apply the linear transformation followed by masking with noise. Parameters ---------- input_ : :class:`~tensor.TensorVariable` The input on which to apply the transformations Returns ------- output : :class:`~tensor.TensorVariable` The transformed input """ # When not in training mode, turn off noise if not self._training_mode: return input_ if self.tied_sigma: average = tensor.shape_padright(self.flatten.apply(input_), 2) noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(average), -16, 16)) noise_level = tensor.patternbroadcast(noise_level, (False, False, True, True)) noise_level = copy_and_tag_noise(noise_level, self, LOG_SIGMA, 'log_sigma') else: average = input_ noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(input_), -16, 16)) noise_level = copy_and_tag_noise(noise_level, self, LOG_SIGMA, 'log_sigma') # Allow incomplete batches by just taking the noise that is needed if self.tied_noise: if self.noise_batch_size is not None: noise = self.parameters[0][:input_.shape[0], :] else: noise = self.theano_rng.normal(input_.shape[0:2]) noise = tensor.shape_padright(2) else: if self.noise_batch_size is not None: noise = self.parameters[0][:input_.shape[0], :, :, :] else: noise = self.theano_rng.normal(input_.shape) kl = (self.prior_noise_level - noise_level + 0.5 * (tensor.exp(2 * noise_level) + (average - self.prior_mean)**2) / tensor.exp(2 * self.prior_noise_level) - 0.5) application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits') return input_ + self.noise_rate * tensor.exp(noise_level) * noise # Needed for the Feedforward interface. @property def output_dim(self): return self.input_dim # The following properties allow for BatchNormalization bricks # to be used directly inside of a ConvolutionalSequence. @property def image_size(self): return self.input_dim[-2:] @image_size.setter def image_size(self, value): if not isinstance(self.input_dim, collections.Sequence): self.input_dim = (None, ) + tuple(value) else: self.input_dim = (self.input_dim[0], ) + tuple(value) @property def num_channels(self): return self.input_dim[0] @num_channels.setter def num_channels(self, value): if not isinstance(self.input_dim, collections.Sequence): self.input_dim = (value, ) + (None, None) else: self.input_dim = (value, ) + self.input_dim[-2:] def get_dim(self, name): if name in ('input', 'output'): return self.input_dim else: raise KeyError @property def num_output_channels(self): return self.num_channels
class NoisyConvolutional(Initializable, Feedforward, Random): """Convolutional transformation sent through a learned noisy channel. Parameters (same as Convolutional) """ @lazy(allocation=[ 'filter_size', 'num_filters', 'num_channels', 'noise_batch_size']) def __init__(self, filter_size, num_filters, num_channels, noise_batch_size, image_size=(None, None), step=(1, 1), border_mode='valid', tied_biases=True, prior_mean=0, prior_noise_level=0, **kwargs): self.convolution = Convolutional() self.mask = Convolutional(name='mask') children = [self.convolution, self.mask] kwargs.setdefault('children', []).extend(children) super(NoisyConvolutional, self).__init__(**kwargs) self.filter_size = filter_size self.num_filters = num_filters self.num_channels = num_channels self.noise_batch_size = noise_batch_size self.image_size = image_size self.step = step self.border_mode = border_mode self.tied_biases = tied_biases self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level def _push_allocation_config(self): self.convolution.filter_size = self.filter_size self.convolution.num_filters = self.num_filters self.convolution.num_channels = self.num_channels # self.convolution.batch_size = self.batch_size self.convolution.image_size = self.image_size self.convolution.step = self.step self.convolution.border_mode = self.border_mode self.convolution.tied_biases = self.tied_biases self.mask.filter_size = (1, 1) self.mask.num_filters = self.num_filters self.mask.num_channels = self.num_filters # self.mask.batch_size = self.batch_size self.mask.image_size = self.convolution.get_dim('output')[1:] # self.mask.step = self.step # self.mask.border_mode = self.border_mode self.mask.tied_biases = self.tied_biases def _allocate(self): out_shape = self.convolution.get_dim('output') N = shared_floatx_zeros((self.noise_batch_size,) + out_shape, name='N') add_role(N, NOISE) self.parameters.append(N) @application(inputs=['input_'], outputs=['output']) def apply(self, input_, application_call): """Apply the linear transformation followed by masking with noise. Parameters ---------- input_ : :class:`~tensor.TensorVariable` The input on which to apply the transformations Returns ------- output : :class:`~tensor.TensorVariable` The transformed input """ from theano.printing import Print pre_noise = self.convolution.apply(input_) # noise_level = self.mask.apply(input_) noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(pre_noise), -16, 16)) noise_level = copy_and_tag_noise( noise_level, self, LOG_SIGMA, 'log_sigma') # Allow incomplete batches by just taking the noise that is needed noise = self.parameters[0][:noise_level.shape[0], :, :, :] # noise = self.theano_rng.normal(noise_level.shape) kl = ( self.prior_noise_level - noise_level + 0.5 * ( tensor.exp(2 * noise_level) + (pre_noise - self.prior_mean) ** 2 ) / tensor.exp(2 * self.prior_noise_level) - 0.5 ) application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits') return pre_noise + tensor.exp(noise_level) * noise def get_dim(self, name): if name == 'input_': return self.convolution.get_dim(name) if name == 'output': return self.convolution.get_dim(name) if name == 'nits': return self.convolution.get_dim('output') return super(NoisyConvolutional, self).get_dim(name) @property def num_output_channels(self): return self.num_filters