def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): """ Allocate a LeNetConvPoolLayer with shared variable internal parameters. :type rng: numpy.random.RandomState :param rng: a random number generator used to initialize weights :type input: theano.tensor.dtensor4 :param input: symbolic image tensor, of shape image_shape :type filter_shape: tuple or list of length 4 :param filter_shape: (number of filters, num input feature maps, filter height,filter width) :type image_shape: tuple or list of length 4 :param image_shape: (batch size, num input feature maps, image height, image width) :type poolsize: tuple or list of length 2 :param poolsize: the downsampling (pooling) factor (#rows,#cols) """ assert image_shape[1] == filter_shape[1] self.input = input # there are "num input feature maps * filter height * filter width" # inputs to each hidden unit fan_in = numpy.prod(filter_shape[1:]) # each unit in the lower layer receives a gradient from: # "num output feature maps * filter height * filter width" / # pooling size fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)) # initialize weights with random weights W_bound = numpy.sqrt(6. / (fan_in + fan_out)) self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX), borrow=True) # the bias is a 1D tensor -- one bias per output feature map b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX) self.b = theano.shared(value=b_values, borrow=True) # convolve input feature maps with filters # conv_out = conv.conv2d(input=input, filters=self.W, # filter_shape=filter_shape, image_shape=image_shape) input_shuffled = input.dimshuffle(1, 2, 3, 0) # bc01 to c01b filters_shuffled = self.W.dimshuffle(1, 2, 3, 0) # bc01 to c01b conv_op = FilterActs(stride=1, partial_sum=1) contiguous_input = gpu_contiguous(input_shuffled) contiguous_filters = gpu_contiguous(filters_shuffled) conv_out_shuffled = conv_op(contiguous_input, contiguous_filters) # downsample each feature map individually, using maxpooling # pooled_out = downsample.max_pool_2d(input=conv_out, # ds=poolsize, ignore_border=True) pool_op = MaxPool(ds=poolsize[0], stride=poolsize[0]) pooled_out_shuffled = pool_op(conv_out_shuffled) pooled_out = pooled_out_shuffled.dimshuffle(3, 0, 1, 2) # c01b to bc01 # add the bias term. Since the bias is a vector (1D array), we first # reshape it to a tensor of shape (1,n_filters,1,1). Each bias will # thus be broadcasted across mini-batches and feature map # width & height self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) # store parameters of this layer self.params = [self.W, self.b]
def compilePredictActivation(self, net, layerNum): variable = net.x if layerNum == 0 else net.varArrayAc[layerNum - 1] #Calc shapes for reshape function on-the-fly. Assume we have square images as input. sX = T.cast(T.sqrt(T.shape(variable)[0] / self.kernel_shape[1]), 'int32') #Converts input from 2 to 4 dimensions Xr = T.reshape(variable.T, (T.shape(variable)[1], self.kernel_shape[1], sX, sX)) if self.optimized: out_size = T.cast( T.ceil((T.shape(Xr)[-1] - T.shape(net.varWeights[layerNum]['w'])[-1] + 1) / np.float32(self.stride)), 'int32') conv_op = FilterActs(stride=self.stride) input_shuffled = Xr.dimshuffle(1, 2, 3, 0) # bc01 to c01b filters_shuffled = net.varWeights[layerNum]['w'].dimshuffle( 1, 2, 3, 0) # bc01 to c01b filters_flipped = filters_shuffled[:, ::-1, :: -1, :] # flip rows and columns contiguous_input = gpu_contiguous(input_shuffled) contiguous_filters = gpu_contiguous( filters_flipped * (self.dropout if self.dropout else 1.0)) a = conv_op(contiguous_input, contiguous_filters) a = a[:, :out_size, :out_size, :] #Add bias a = a + net.varWeights[layerNum]['b'].dimshuffle(0, 'x', 'x', 'x') else: a = T.nnet.conv2d(Xr, net.varWeights[layerNum]['w'] * (net.dropOutVectors[layerNum].dimshuffle( 'x', 'x', 0, 1) if self.dropout else 1.0), border_mode='valid', subsample=(self.stride, self.stride)) #Add bias a = a + net.varWeights[layerNum]['b'].dimshuffle('x', 0, 'x', 'x') if self.pooling: if self.optimized: #Pooling # ds - side of square pool window # stride - Defines the stride size between successive pooling squares. # Setting this parameter smaller than sizeX produces overlapping pools. # Setting it equal to sizeX gives the usual, non-overlapping pools. Values greater than sizeX are not allowed. pool_op = MaxPool(ds=self.pooling_shape, stride=self.pooling_shape) contiguous_input = gpu_contiguous( a.astype(theano.config.floatX)) a = pool_op(contiguous_input) a = a.dimshuffle(3, 0, 1, 2) # c01b to bc01 else: a = downsample.max_pool_2d( a, (self.pooling_shape, self.pooling_shape), ignore_border=False) else: if self.optimized: a = a.dimshuffle(3, 0, 1, 2) # c01b to bc01 a = T.flatten(a, outdim=2).T #Sigmoid a = self.activation(a, self.pool_size) net.varArrayAc.append(a)
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2), cuda_convnet=0, W=None, b=None, activation=T.tanh, border_mode='valid', partial_sum=1, pad=0): """ Allocate a LeNetConvPoolLayer with shared variable internal parameters. :type rng: numpy.random.RandomState :param rng: a random number generator used to initialize weights :type input: theano.tensor.dtensor4 :param input: symbolic image tensor, of shape image_shape :type filter_shape: tuple or list of length 4 :param filter_shape: (number of filters, num input feature maps, filter height, filter width) :type image_shape: tuple or list of length 4 :param image_shape: (batch size, num input feature maps, image height, image width) :type poolsize: tuple or list of length 2 :param poolsize: the downsampling (pooling) factor (#rows, #cols) """ channels_idx = 0 if cuda_convnet else 1 assert image_shape[channels_idx] == filter_shape[channels_idx] self.input = input if W is None: if cuda_convnet: fan_in = numpy.prod(filter_shape[0:3]) fan_out = (filter_shape[3] * numpy.prod(filter_shape[1:3]) / numpy.prod(poolsize)) # TODO: correct numpy.prod(poolsize). Theano's max_pool_2d uses a tuple # to signify (x,y) poolsize and doesn't overlap these. cuda-convnet uses # the tuple to signify a square pool of size x^2, with a stride y. So only # if x == y in this case does numpy.prod(poolsize) work. else: # there are "num input feature maps * filter height * filter width" # inputs to __each__ hidden unit fan_in = numpy.prod(filter_shape[1:]) # __each__ unit in the lower layer receives a gradient from: # "num output feature maps * filter height * filter width" / # pooling size fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)) # initialize weights with random weights W_bound = numpy.sqrt(6. / (fan_in + fan_out)) W_values = numpy.asarray(rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX) else: W_values = numpy.asarray(W, dtype=theano.config.floatX) # the bias is a 1D tensor -- one bias per output feature map if b is None: numchan = filter_shape[3] if cuda_convnet else filter_shape[0] b_values = numpy.zeros((numchan, ), dtype=theano.config.floatX) else: b_values = numpy.asarray(b, dtype=theano.config.floatX) self.W = theano.shared(value=W_values, borrow=True) self.b = theano.shared(value=b_values, borrow=True) # convolve input feature maps with filters if cuda_convnet: conv_op = FilterActs(partial_sum=partial_sum, pad=pad) contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W) conv_out = conv_op(contiguous_input, contiguous_filters) else: conv_out = conv.conv2d(input=input, filters=self.W, image_shape=image_shape, filter_shape=filter_shape, border_mode=border_mode) # downsample each feature map individually, using maxpooling if (poolsize[0] == 1 and cuda_convnet) or (poolsize[0] == 1 and poolsize[1] == 1 and not cuda_convnet): pooled_out = conv_out elif cuda_convnet: pool_op = MaxPool(ds=poolsize[0], stride=poolsize[1]) contiguous_input = gpu_contiguous(conv_out) pooled_out = pool_op(contiguous_input) else: pooled_out = downsample.max_pool_2d(input=conv_out, ds=poolsize, ignore_border=True) # add the bias term. Since the bias is a vector (1D array), we first # reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will # thus be broadcasted across mini-batches and feature map # width & height if cuda_convnet: self.output = activation(pooled_out + self.b.dimshuffle(0, 'x', 'x', 'x')) else: self.output = activation(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) # store parameters of this layer self.params = [self.W, self.b]
def __init__(self, numpy_rng=None, input=None, is_input_layer=False, input_shape=(1, 28, 28), filter_shape=(2, 1, 5, 5), poolsize=(1, 1), activation=T.tanh, flatten=False, border_mode='valid', non_maximum_erasing=False, W=None, b=None): assert input_shape[1] == filter_shape[1] if is_input_layer: self.input = input.reshape(input_shape).dimshuffle(1, 2, 3, 0) else: self.input = input # Now reconstruct the input_shape and filter_shape input_shape = (input_shape[1], input_shape[2], input_shape[3], input_shape[0]) filter_shape = (filter_shape[1], filter_shape[2], filter_shape[3], filter_shape[0]) self.input_shape = input_shape self.filter_shape = filter_shape self.poolsize = poolsize self.activation = activation self.flatten = flatten fan_in = numpy.prod(filter_shape[:3]) fan_out = (filter_shape[3] * numpy.prod(filter_shape[1:3]) / numpy.prod(poolsize)) # initialize weights with random weights if W is None: W_bound = numpy.sqrt(6. / (fan_in + fan_out)) initial_W = numpy.asarray(numpy_rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX) if activation == T.nnet.sigmoid: initial_W *= 4 W = theano.shared(value=initial_W, name='W') self.W = W # the bias is a 1D tensor -- one bias per output feature map if b is None: b_values = numpy.zeros((filter_shape[3], ), dtype=theano.config.floatX) b = theano.shared(value=b_values, name='b') self.b = b # for momentum self.delta_W = theano.shared(value=numpy.zeros( filter_shape, dtype=theano.config.floatX), name='delta_W') self.delta_b = theano.shared(value=numpy.zeros_like( self.b.get_value(borrow=True), dtype=theano.config.floatX), name='delta_b') # convolve input feature maps with filters conv_op = FilterActs() contiguous_input = gpu_contiguous(self.input) contiguous_filters = gpu_contiguous(self.W) conv_out = conv_op(contiguous_input, contiguous_filters) y_out = activation(conv_out + self.b.dimshuffle(0, 'x', 'x', 'x')) pool_op = MaxPool(ds=poolsize[0], stride=poolsize[0]) pooled_out = pool_op(y_out) if non_maximum_erasing: ds = tuple(poolsize) po = pooled_out.repeat(ds[0], axis=2).repeat(ds[1], axis=3) self.output = T.eq(y_out, po) * y_out else: self.output = pooled_out if flatten: self.output = self.output.dimshuffle(3, 0, 1, 2) # c01b to bc01 self.output = self.output.flatten(2) self.params = [self.W, self.b] self.delta_params = [self.delta_W, self.delta_b]
def __init__( self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', ): ''' lib_conv can be cudnn (recommended)or cudaconvnet ''' self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudaconvnet': self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous(input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op(contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous(input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op(contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu self.output = T.maximum(conv_out, 0) conv_out = gpu_contiguous(conv_out) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) elif lib_conv == 'cudnn': input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 # print image_shape_shuffled # print filter_shape_shuffled if group == 1: W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv( img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out0 = \ dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2, :, :], kerns=W0_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out1 = \ dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:, :, :], kerns=W1_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b else: NotImplementedError("lib_conv can only be cudaconvnet or cudnn") # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] print "conv ({}) layer with shape_in: {}".format( lib_conv, str(image_shape))
def __init__(self, rng, input=None, filt_def=None, pool_def=(2, 2), \ activation=None, drop_rate=0., input_noise=0., bias_noise=0., \ W=None, b=None, name="", W_scale=1.0): # Setup a shared random generator for this layer #self.rng = theano.tensor.shared_randomstreams.RandomStreams( \ # rng.randint(100000)) self.rng = CURAND_RandomStreams(rng.randint(1000000)) self.clean_input = input # Add gaussian noise to the input (if desired) if (input_noise > 1e-4): self.fuzzy_input = input + self.rng.normal(size=input.shape, \ avg=0.0, std=input_noise, dtype=theano.config.floatX) else: self.fuzzy_input = input # Apply masking noise to the input (if desired) if (drop_rate > 1e-4): self.noisy_input = self._drop_from_input(self.fuzzy_input, drop_rate) else: self.noisy_input = self.fuzzy_input # Set the activation function for the conv filters if activation: self.activation = activation else: self.activation = lambda x: relu_actfun(x) # initialize weights with random weights W_init = 0.01 * np.asarray(rng.normal( \ size=filt_def), dtype=theano.config.floatX) self.W = theano.shared(value=(W_scale*W_init), \ name="{0:s}_W".format(name)) # the bias is a 1D tensor -- one bias per output feature map b_init = np.zeros((filt_def[0], ), dtype=theano.config.floatX) + 0.1 self.b = theano.shared(value=b_init, name="{0:s}_b".format(name)) # convolve input feature maps with filters input_c01b = self.noisy_input.dimshuffle(1, 2, 3, 0) # bc01 to c01b filters_c01b = self.W.dimshuffle(1, 2, 3, 0) # bc01 to c01b conv_op = FilterActs(stride=1, partial_sum=1) contig_input = gpu_contiguous(input_c01b) contig_filters = gpu_contiguous(filters_c01b) conv_out_c01b = conv_op(contig_input, contig_filters) if (bias_noise > 1e-4): noisy_conv_out_c01b = conv_out_c01b + self.rng.normal( \ size=conv_out_c01b.shape, avg=0.0, std=bias_noise, \ dtype=theano.config.floatX) else: noisy_conv_out_c01b = conv_out_c01b # downsample each feature map individually, using maxpooling pool_op = MaxPool(ds=pool_def[0], stride=pool_def[1]) mp_out_c01b = pool_op(noisy_conv_out_c01b) mp_out_bc01 = mp_out_c01b.dimshuffle(3, 0, 1, 2) # c01b to bc01 # add the bias term. Since the bias is a vector (1D array), we first # reshape it to a tensor of shape (1,n_filters,1,1). Each bias will # thus be broadcasted across mini-batches and feature map # width & height self.noisy_linear_output = mp_out_bc01 + self.b.dimshuffle( 'x', 0, 'x', 'x') self.linear_output = self.noisy_linear_output self.output = self.activation(self.noisy_linear_output) # store parameters of this layer self.params = [self.W, self.b] return
def test_pool(): try: if hasattr(mode_with_gpu, 'check_isfinite'): mode_with_gpu_check_is_finite_prev = mode_with_gpu.check_isfinite if hasattr(mode_without_gpu, 'check_isfinite'): mode_without_gpu_check_is_finite_prev = mode_without_gpu.check_isfinite mode_with_gpu.check_isfinite = False mode_without_gpu.check_isfinite = False #(batch, channel, x, y) shps = [ (1, 1, 2, 2), (1, 1, 1, 1), (1, 1, 4, 4), (1, 2, 2, 2), (1, 1, 4, 4), (3, 1, 4, 4), (1, 5, 4, 4), (3, 5, 4, 4), (25, 1, 7, 7), (1, 1, 12, 12), (1, 1, 14, 14), (1, 1, 16, 16), (1, 1, 18, 18), (1, 1, 24, 24), (1, 6, 24, 24), (10, 1, 24, 24), (10, 6, 24, 24), (30, 6, 12, 12), (30, 2, 24, 24), (30, 6, 24, 24), (65536, 1, 10, 10), #(1, 65536, 10, 10),#crash as too much channel (30, 3, 40, 40), ] shps = [(channel, x, y, batch) for (batch, channel, x, y) in shps] #numpy.random.RandomState(unittest_tools.fetch_seed()).shuffle(shps) for shp in shps: for ds in range(1, min(4, shp[2] + 1)): # for start in range(shp[2] + 1): for start in [0]: for stride in range(1, min(shp[2], ds, 4) + 1): print('test_pool shape=%s, ds=%d, stride=%d start=%d' % (str(shp), ds, stride, start)) a = tcn.shared_constructor(my_rand(*shp), 'a') op = MaxPool(ds=ds, stride=stride) f = theano.function([], op(a), mode=mode_with_gpu) assert any([ isinstance(node.op, MaxPool) for node in f.maker.fgraph.toposort() ]) out = numpy.asarray(f()) #Compute the gold version with a Theano graph. gold_out = gold_max_pool_c01b(a, (ds, ds), (stride, stride), shp[1:3]) f2 = theano.function([], gold_out, mode=mode_without_gpu) assert not any([ isinstance(node.op, MaxPool) for node in f2.maker.fgraph.toposort() ]) out2 = f2() numpy.testing.assert_allclose(out, out2, err_msg=str(out - out2)) # grad testing # The code support grad only in this case. if shp[0] % 16 != 0: shp2 = list(shp) shp2[0] *= 16 # This make it crash due to not enough memory. # On a GPU with 1279M of ram. if numpy.prod(shp2) >= (16 * 10 * 10 * 65536): continue a.set_value(my_rand(*shp2)) g = theano.function([], grad(op(a).sum(), a), mode=mode_with_gpu) g2 = theano.function([], grad(gold_out.sum(), a), mode=mode_without_gpu) assert any([ isinstance(node.op, MaxPoolGrad) for node in g.maker.fgraph.toposort() ]) assert not any([ isinstance(node.op, MaxPoolGrad) for node in g2.maker.fgraph.toposort() ]) numpy.testing.assert_allclose(g(), g2(), err_msg=str(shp)) # Don't call verify_grad. There was problem with # the test and we already assert that 2 version # are equals. Also, it will be slower to verify # like that then the comparison. continue theano.tests.unittest_tools.verify_grad( op, [a.get_value()]) finally: if 'mode_with_gpu_check_is_finite_prev' in locals(): mode_with_gpu.check_isfinite = mode_with_gpu_check_is_finite_prev if 'mode_without_gpu_check_is_finite_prev' in locals(): mode_without_gpu.check_isfinite = mode_without_gpu_check_is_finite_prev
def __init__(self, numpy_rng=None, input=None, input_shape=(256, 1, 28, 28), filter_shape=(2, 1, 5, 5), poolsize=(1, 1), activation=T.tanh, flatten=False, border_mode='valid', non_maximum_erasing=False, W=None, b=None, use_fast=False, testing=False): self.type = 'conv' assert input_shape[1] == filter_shape[1] if testing: self.input = input.reshape((input.shape[0], input_shape[1], input_shape[2], input_shape[3])) input_shape = None else: self.input = input.reshape(input_shape) self.filter_shape = filter_shape self.poolsize = poolsize self.activation = activation self.flatten = flatten fan_in = numpy.prod(filter_shape[1:]) fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)) # initialize weights with random weights if W is None: W_bound = numpy.sqrt(6. / (fan_in + fan_out)) initial_W = numpy.asarray(numpy_rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX) if activation == T.nnet.sigmoid: initial_W *= 4 W = theano.shared(value=initial_W, name='W') self.W = W # the bias is a 1D tensor -- one bias per output feature map if b is None: b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX) b = theano.shared(value=b_values, name='b') self.b = b # for momentum self.delta_W = theano.shared(value=numpy.zeros( filter_shape, dtype=theano.config.floatX), name='delta_W') self.delta_b = theano.shared(value=numpy.zeros_like( self.b.get_value(borrow=True), dtype=theano.config.floatX), name='delta_b') # convolve input feature maps with filters if use_fast: from theano.sandbox.cuda.basic_ops import gpu_contiguous from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs from pylearn2.sandbox.cuda_convnet.pool import MaxPool input_shuffled = self.input.dimshuffle(1, 2, 3, 0) # bc01 to c01b filters_shuffled = self.W.dimshuffle(1, 2, 3, 0) # bc01 to c01b conv_op = FilterActs() contiguous_input = gpu_contiguous(input_shuffled) contiguous_filters = gpu_contiguous(filters_shuffled) conv_out_shuffled = conv_op(contiguous_input, contiguous_filters) y_out_shuffled = activation(conv_out_shuffled + self.b.dimshuffle(0, 'x', 'x', 'x')) pool_op = MaxPool(ds=poolsize[0], stride=poolsize[0]) pooled_out = pool_op(y_out_shuffled).dimshuffle(3, 0, 1, 2) else: conv_out = conv.conv2d(input=self.input, filters=self.W, filter_shape=filter_shape, image_shape=input_shape, border_mode=border_mode) y_out = activation(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) # downsample each feature map individually, using maxpooling pooled_out = downsample.max_pool_2d(input=y_out, ds=poolsize, ignore_border=True) if non_maximum_erasing: ds = tuple(poolsize) po = pooled_out.repeat(ds[0], axis=2).repeat(ds[1], axis=3) self.output = T.eq(y_out, po) * y_out else: self.output = pooled_out if flatten: self.output = self.output.flatten(2) self.params = [self.W, self.b] self.delta_params = [self.delta_W, self.delta_b]
def __init__(self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', ): """ lib_conv can be cudnn or cudaconvet """ assert group in [1, 2] assert lib_conv in ['cudnn', 'cudaconvnet'] self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride if lib_conv == 'cudnn': self.channel = image_shape[1] else: self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) if lib_conv == 'cudnn': self.b = Weight(self.filter_shape[0], bias_init, std=0) else: self.b = Weight(self.filter_shape[3], bias_init, std=0) else: if lib_conv == 'cudnn': self.filter_shape[1] /= 2 self.filter_shape[0] /= 2 self.image_shape[1] /= 2 self.image_shape[0] /= 2 else: self.filter_shape[0] /= 2 self.filter_shape[3] /= 2 self.image_shape[0] /= 2 self.image_shape[3] /= 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) if lib_conv=='cudnn': self.b0 = Weight(self.filter_shape[0], bias_init, std=0) self.b1 = Weight(self.filter_shape[0], bias_init, std=0) else: self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudaconvnet': self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous( input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op( contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous( input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op( contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu conv_out = T.maximum(conv_out, 0) self.output = gpu_contiguous(conv_out) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) else: if group == 1: conv_out = dnn.dnn_conv(img=input, kerns=self.W.val, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: input1, input2 = \ theano.tensor.split(input, [self.channel/2, self.channel/2], 2, axis=1) conv_out0 = \ dnn.dnn_conv(img=input1, kerns=self.W0.val, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + self.b0.val.dimshuffle('x', 0, 'x', 'x') conv_out1 = \ dnn.dnn_conv(img=input2, kerns=self.W1.val, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + self.b1.val.dimshuffle('x', 0, 'x', 'x') # self.conv_out = conv_out1 conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) # LRN if self.lrn: self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] print "conv ({}) layer with shape_in: {}".format(lib_conv, str(image_shape))
def __init__(self, numpy_rng=None, input=None, filter_shape=(2, 1, 5, 5), poolsize=(1, 1), activation=T.nnet.sigmoid, flatten=False, use_fast=False): self.type = 'conv' self.input = input self.filter_shape = filter_shape self.poolsize = poolsize self.activation = activation self.flatten = flatten fan_in = numpy.prod(filter_shape[1:]) fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)) # initialize weights with random weights W_bound = numpy.sqrt(6. / (fan_in + fan_out)) initial_W = numpy.asarray(numpy_rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX) if activation == T.nnet.sigmoid: initial_W *= 4 W = theano.shared(value=initial_W, name='W') self.W = W # the bias is a 1D tensor -- one bias per output feature map b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX) self.b = theano.shared(value=b_values, name='b') # convolve input feature maps with filters if use_fast: from theano.sandbox.cuda.basic_ops import gpu_contiguous from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs from pylearn2.sandbox.cuda_convnet.pool import MaxPool input_shuffled = self.input.dimshuffle(1, 2, 3, 0) # bc01 to c01b filters_shuffled = self.W.dimshuffle(1, 2, 3, 0) # bc01 to c01b conv_op = FilterActs() contiguous_input = gpu_contiguous(input_shuffled) contiguous_filters = gpu_contiguous(filters_shuffled) conv_out_shuffled = conv_op(contiguous_input, contiguous_filters) y_out_shuffled = activation(conv_out_shuffled + self.b.dimshuffle(0, 'x', 'x', 'x')) pool_op = MaxPool(ds=poolsize[0], stride=poolsize[0]) self.output = pool_op(y_out_shuffled).dimshuffle(3, 0, 1, 2) else: conv_out = conv.conv2d(input=self.input, filters=self.W, filter_shape=filter_shape) y_out = activation(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) # downsample each feature map individually, using maxpooling self.output = downsample.max_pool_2d(input=y_out, ds=poolsize, ignore_border=True) if self.flatten: self.output = self.output.flatten(2) self.params = [self.W, self.b]
def __init__(self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False): self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous(input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op(contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous(input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op(contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu self.output = T.maximum(conv_out, 0) conv_out = gpu_contiguous(conv_out) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] print "conv layer with shape_in: " + str(image_shape)
def __init__(self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', verbose=False ): ''' lib_conv can be cudnn (recommended)or cudaconvnet ''' self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv self.verbose = verbose assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudaconvnet': self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) from theano.sandbox.cuda.basic_ops import gpu_contiguous # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous( input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op( contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous( input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op( contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu conv_out = gpu_contiguous(conv_out) self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) elif lib_conv == 'corrmm': from theano.sandbox.cuda.basic_ops import gpu_contiguous from theano.sandbox.cuda.blas import GpuCorrMM border_mode = 'half' if padsize == (filter_shape[1]-1)/2 else (padsize, padsize) self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride), border_mode=border_mode) flip_filters=True input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 if group==1: filters = self.W.val.dimshuffle(3, 0, 1, 2) if flip_filters: filters = filters[:, :, ::-1, ::-1] # flip top-down, left-right contiguous_filters = gpu_contiguous(filters) contiguous_input = gpu_contiguous(input_shuffled) conv_out = self.corr_mm_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 if flip_filters: W0_shuffled = W0_shuffled[:, :, ::-1, ::-1] contiguous_filters0 = gpu_contiguous(W0_shuffled) contiguous_input0 = gpu_contiguous(input_shuffled[:, :self.channel / 2,:, :]) conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 if flip_filters: W1_shuffled = W1_shuffled[:, :, ::-1, ::-1] contiguous_filters1 = gpu_contiguous(W1_shuffled) contiguous_input1 = gpu_contiguous(input_shuffled[:, self.channel / 2:,:, :]) conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: from theano.tensor.signal import downsample self.output = downsample.max_pool_2d(self.output, ds=(poolsize,poolsize), st=(poolstride,poolstride), ignore_border=False, padding=(0,0), mode='max', ) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b elif lib_conv == 'cudnn': input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 # print image_shape_shuffled # print filter_shape_shuffled if group == 1: W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv(img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out0 = \ dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2, :, :], kerns=W0_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out1 = \ dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:, :, :], kerns=W1_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b else: NotImplementedError("lib_conv can only be cudaconvnet or cudnn") # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] if self.verbose: print "conv ({}) layer with shape_in: {}".format(lib_conv, str(image_shape))
def conv(data, filter_gen, feature_batch_size, num_feature_batches, data_batch_size, cuda_convnet=True, symmetric_relu=True, start_feature_batch=0, pool_type='avg', pool_size=14, pad=0, bias=1.0, ps=6): cuda_convnet = True outX = int(math.ceil((data.shape[2] - ps + 1) / float(pool_size))) outY = int(math.ceil((data.shape[3] - ps + 1) / float(pool_size))) outFilters = feature_batch_size * num_feature_batches if (symmetric_relu): outFilters = 2 * outFilters print "Out Shape ", outX, "x", outY, "x", outFilters XFinal = np.zeros((data.shape[0], outFilters, outX, outY), 'float32') filters = [] numImages = data.shape[0] # Convert to cuda-convnet order if (cuda_convnet): data = data.transpose(1, 2, 3, 0) # POOL OP CREATION if (cuda_convnet): if (pool_type == 'avg'): pool_op = AvgPool(ds=pool_size, stride=pool_size) elif (pool_type == 'max'): pool_op = MaxPool(ds=pool_size, stride=pool_size) else: raise Exception('Unsupported pool type') else: pool_op = lambda X: T.signal.pool.pool_2d( X, (pool_size, pool_size), ignore_border=False, mode='max') if (cuda_convnet): conv_op = FilterActs(pad=pad) else: conv_op = lambda X, F: T.nnet.conv2d(X, F) CHANNEL_AXIS = 1 for j in range(num_feature_batches): F = filter_gen(feature_batch_size) if (cuda_convnet): F = F.transpose(1, 2, 3, 0) CHANNEL_AXIS = 0 filters.append(F) FTheano = shared(F.astype('float32')) start_filters = j * feature_batch_size end_filters = (j + 1) * feature_batch_size if symmetric_relu: start_filters *= 2 end_filters *= 2 for i in range(int(np.ceil(numImages / float(data_batch_size)))): start = i * data_batch_size end = min((i + 1) * data_batch_size, numImages) print "FEATURE BATCH #", ( j + start_feature_batch ), "DATA BATCH #", i, " SIZE IS ", end - start if (cuda_convnet): XBlock = shared(data[:, :, :, start:end]) else: XBlock = shared(data[start:end, :, :, :]) if (cuda_convnet): XBlock_gpu = gpu_contiguous(XBlock) FTheano_gpu = gpu_contiguous(FTheano) # CONV XBlock_conv_out = conv_op(XBlock_gpu, FTheano_gpu) # RELU XBlock0 = T.nnet.relu(XBlock_conv_out - bias, 0) if (symmetric_relu): XBlock1 = T.nnet.relu(-1.0 * XBlock_conv_out - bias, 0) XBlock0 = pool_op(XBlock0) if (symmetric_relu): XBlock1 = pool_op(XBlock1) XBlockOut = np.concatenate((XBlock0.eval(), XBlock1.eval()), axis=CHANNEL_AXIS) else: XBlockOut = np.array(XBlock0.eval()) if (cuda_convnet): XBlockOut = XBlockOut.transpose(3, 0, 1, 2) F = F.transpose(3, 0, 1, 2) XBlock.set_value([[[[]]]]) XFinal[start:end, start_filters:end_filters, :, :] = XBlockOut FTheano.set_value([[[[]]]]) filters = np.concatenate(filters, axis=0) return (XFinal, filters)
#Calc avg activation for convolution results sprs = 0 if sparsity: sparse = T.mean(res, axis=(1, 2, 3)) epsilon = 1e-20 sparse = T.clip(sparse, epsilon, 1 - epsilon) KL = T.sum(sparsityParam * T.log(sparsityParam / sparse) + (1 - sparsityParam) * T.log((1 - sparsityParam) / (1 - sparse))) sprs = KL * beta #Pooling #res = downsample.max_pool_2d(res, pool_shape, ignore_border=True) #res_p = downsample.max_pool_2d(res_p, pool_shape, ignore_border=True) pool_op = MaxPool(ds=2, stride=2) contiguous_input = gpu_contiguous(res) res = pool_op(contiguous_input) contiguous_input_p = gpu_contiguous(res_p) res_p = pool_op(contiguous_input_p) res = res.dimshuffle(3, 0, 1, 2) # c01b to bc01 res_p = res_p.dimshuffle(3, 0, 1, 2) # c01b to bc01 #Separate function if U want to estimate output just after pooling #cnn = theano.function(inputs=[X], outputs=res, allow_input_downcast=True) #------# CV_size = 6000