def f_conv(self, x, spec, in_dim, weight_name): layer_type, dims = spec num_filters = dims[0] filter_size = (dims[1], dims[1]) stride = (dims[2], dims[2]) bm = 'full' if 'convf' in layer_type else 'valid' num_channels = in_dim[0] W = self.weight(self.rand_init_conv( (num_filters, num_channels) + filter_size), weight_name) if stride != (1, 1): f = GpuCorrMM(subsample=stride, border_mode=bm, pad=(0, 0)) y = f(gpu_contiguous(x), gpu_contiguous(W)) else: assert self.p.batch_size == self.p.valid_batch_size y = conv2d(x, W, image_shape=(2*self.p.batch_size, ) + in_dim, filter_shape=((num_filters, num_channels) + filter_size), border_mode=bm) output_size = ((num_filters,) + ConvOp.getOutputShape(in_dim[1:], filter_size, stride, bm)) return y, output_size
def local_conv2d_gradinputs_cpu(node): kern, topgrad, shape = node.inputs if ((not isinstance(kern.type, TensorType) or not isinstance(topgrad.type, TensorType))): return None if node.op.border_mode not in ['full', 'valid']: return None if not node.op.filter_flip: # Not tested yet return None # Conv 3d implementation, needed when subsample > 2 if node.op.border_mode == 'valid' and node.op.subsample != (1, 1): kern = kern[:, :, ::-1, ::-1] shuffled_kern = kern.dimshuffle(0, 2, 3, 'x', 1) shuffled_topgrad = topgrad.dimshuffle(0, 2, 3, 'x', 1) b = theano.tensor.zeros_like(shuffled_kern[0, 0, 0, 0, :]) rval = convTransp3D(W=shuffled_kern, b=b, d=(node.op.subsample[0], node.op.subsample[1], 1), H=shuffled_topgrad, RShape=(shape[0], shape[1], 1)) rval = theano.tensor.addbroadcast(rval, 3) rval = rval.dimshuffle(0, 4, 1, 2) rval = patternbroadcast(rval, node.outputs[0].broadcastable) return [rval] # Conv2d Implementation dx, dy = node.op.subsample if dx not in (1, 2) or dy not in (1, 2): # Not implemented in the gradient of ConvOp return None if node.op.imshp is None: op_imshp = (None, None, None, None) else: op_imshp = node.op.imshp if node.op.kshp is None: op_kshp = (None, None, None, None) else: op_kshp = node.op.kshp if None in op_imshp or None in op_kshp: if (dx, dy) != (1, 1): return None mode = 'valid' if not node.op.border_mode == 'full': mode = 'full' filters = kern.dimshuffle((1, 0, 2, 3)) filters = filters[:, :, ::-1, ::-1] outshp = ConvOp.getOutputShape(op_imshp[2:], op_kshp[2:], node.op.subsample, node.op.border_mode) fulloutshp = ConvOp.getOutputShape(op_imshp[2:], op_kshp[2:], (1, 1), node.op.border_mode) nkern = op_imshp[1] imshp = (op_kshp[0], outshp[0], outshp[1]) imshp_logical = (op_kshp[0], fulloutshp[0], fulloutshp[1]) din = ConvOp(imshp, op_kshp[2:], nkern, op_imshp[0], 1, 1, output_mode=mode, unroll_batch=None, unroll_kern=None, unroll_patch=None, imshp_logical=imshp_logical, kshp_logical=None, version=-1, direction_hint='bprop inputs') din = din(topgrad, filters) din = patternbroadcast(din, node.outputs[0].broadcastable) return [din]
def local_conv2d_gradweight_cpu(node): img, topgrad, shape = node.inputs if ((not isinstance(img.type, TensorType) or not isinstance(topgrad.type, TensorType))): return None if node.op.border_mode not in ['full', 'valid']: return None if not node.op.filter_flip: # Not tested yet return if node.op.border_mode == 'valid' and \ (node.op.subsample != (1, 1)): # Use the gradient as defined in conv3D, because the implementation # by Conv is slow (about 3x slower than conv3D, and probably 10x # slower than it could be), nad incorrect when subsample > 2. # build a "node", that should be equivalent to the one given by # self.make_node, but using convGrad3D instead. shuffled_img = img.dimshuffle(0, 2, 3, 'x', 1) shuffled_topgrad = topgrad.dimshuffle(0, 2, 3, 'x', 1) rval = convGrad3D(V=shuffled_img, d=(node.op.subsample[0], node.op.subsample[1], 1), WShape=(shuffled_topgrad.shape[4], shape[0], shape[1], 1, shuffled_img.shape[4]), dCdH=shuffled_topgrad) rval = theano.tensor.addbroadcast(rval, 3) rval = rval.dimshuffle(0, 4, 1, 2) rval = rval[:, :, ::-1, ::-1] rval = patternbroadcast(rval, node.outputs[0].broadcastable) return [rval] dx, dy = node.op.subsample if dx not in (1, 2) or dy not in (1, 2): # Not implemented in the gradient of ConvOp return None if node.op.imshp is None: op_imshp = (None, None, None, None) else: op_imshp = node.op.imshp if node.op.kshp is None: op_kshp = (None, None, None, None) else: op_kshp = node.op.kshp if None in op_imshp or None in op_kshp: if (dx, dy) != (1, 1): # We cannot infer the shapes return None # Determine gradient on kernels assert len(op_imshp) == 4 and len(op_kshp) == 4 outshp = ConvOp.getOutputShape(op_imshp[2:], op_kshp[2:], node.op.subsample, node.op.border_mode) fulloutshp = ConvOp.getOutputShape(op_imshp[2:], op_kshp[2:], (1, 1), node.op.border_mode) newimg = img.dimshuffle((1, 0, 2, 3)) newtopgrad = topgrad.dimshuffle((1, 0, 2, 3)) if node.op.border_mode == 'valid': (img, filters) = (newimg, newtopgrad) kshp_logical = fulloutshp kshp_logical_top_aligned = False imshp_logical = None (bsize, nkern) = (op_imshp[1], op_kshp[0]) imshp = (op_imshp[0], op_imshp[2], op_imshp[3]) kshp = outshp elif node.op.border_mode == 'full': (img, filters) = (newtopgrad, newimg) kshp_logical = None kshp_logical_top_aligned = True imshp_logical = (op_imshp[0], fulloutshp[0], fulloutshp[1]) (bsize, nkern) = (op_kshp[0], op_imshp[1]) imshp = (op_imshp[0], outshp[0], outshp[1]) kshp = op_imshp[2:] else: raise NotImplementedError( 'Only [full,valid] modes are currently supported.') # Flip the kernels filters = filters[:, :, ::-1, ::-1] dw = ConvOp(imshp, kshp, nkern, bsize, 1, 1, output_mode='valid', unroll_batch=None, unroll_kern=None, unroll_patch=None, imshp_logical=imshp_logical, kshp_logical=kshp_logical, kshp_logical_top_aligned=kshp_logical_top_aligned, direction_hint='bprop weights') res = dw(img, filters) if node.op.border_mode == 'valid': res = res.dimshuffle((1, 0, 2, 3)) res = res[:, :, ::-1, ::-1] res = patternbroadcast(res, node.outputs[0].broadcastable) return [res]
def __init__(self, inputs=None, params=None, outdir='outputs/conv1d', n_filters=None, filter_size=None, stride=None, border_mode='valid', weights_init='uniform', weights_interval='montreal', weights_mean=0, weights_std=5e-3, bias_init=0, activation='rectifier', convolution='mc0', mrg=RNG_MRG.MRG_RandomStreams(1), **kwargs): """ Initialize a 1-D convolutional layer. Parameters ---------- inputs : tuple(shape, `Theano.TensorType`) The dimensionality of the inputs for this model, and the routing information for the model to accept inputs from elsewhere. `shape` will be a monad tuple representing known sizes for each dimension in the `Theano.TensorType`. Shape of the incoming data: (batch_size, num_channels, data_dimensionality). Most likely, your channels will be 1. For example, batches of text will be of the form (N, 1, D) where N=examples in minibatch and D=dimensionality (chars, words, etc.) params : Dict(string_name: theano SharedVariable), optional A dictionary of model parameters (shared theano variables) that you should use when constructing this model (instead of initializing your own shared variables). This parameter is useful when you want to have two versions of the model that use the same parameters - such as siamese networks or pretraining some weights. outdir : str The directory you want outputs (parameters, images, etc.) to save to. If None, nothing will be saved. n_filters : int The number of filters to use (convolution kernels). filter_size : int The size of the convolution filter. stride : int The distance between the receptive field centers of neighboring units. This is the 'stride' of the convolution operation. border_mode : str, one of 'valid', 'full', 'same' A string indicating the convolution border mode. If 'valid', the convolution is only computed where the input and the filter fully overlap. If 'full', the convolution is computed wherever the input and the filter overlap by at least one position. weights_init : str Determines the method for initializing model weights. See opendeep.utils.nnet for options. weights_interval : str or float If Uniform `weights_init`, the +- interval to use. See opendeep.utils.nnet for options. weights_mean : float If Gaussian `weights_init`, the mean value to use. weights_std : float If Gaussian `weights_init`, the standard deviation to use. bias_init : float The initial value to use for the bias parameter. Most often, the default of 0.0 is preferred. activation : str or Callable The activation function to apply to the layer. See opendeep.utils.activation for options. convolution : str or Callable The 1-dimensional convolution implementation to use. The default of 'mc0' is normally fine. See opendeep.utils.conv1d_implementations for alternatives. (This is necessary because Theano only supports 2D convolutions at the moment). mrg : random A random number generator that is used when adding noise. I recommend using Theano's sandbox.rng_mrg.MRG_RandomStreams. Notes ----- Theano's default convolution function (`theano.tensor.nnet.conv.conv2d`) does not support the 'same' border mode by default. This layer emulates it by performing a 'full' convolution and then cropping the result, which may negatively affect performance. """ initial_parameters = locals().copy() initial_parameters.pop('self') super(Conv1D, self).__init__(**initial_parameters) if self.inputs is None: return ################## # specifications # ################## # grab info from the inputs_hook, or from parameters # expect input to be in the form (B, C, I) (batch, channel, input data) # inputs_hook is a tuple of (Shape, Input) # self.inputs is a list of all the input expressions (we enforce only 1, so self.inputs[0] is the input) input_shape, self.input = self.inputs[0] assert self.input.ndim == 3, "Expected 3D input variable with form (batch, channel, input_data)" assert len(input_shape) == 3, "Expected 3D input shape with form (batch, channel, input_data)" n_channels = input_shape[1] filter_shape = (n_filters, n_channels, filter_size) # activation function! activation_func = get_activation_function(activation) # convolution function! convolution_func = get_conv1d_function(convolution) outshape = ConvOp.getOutputShape( inshp=(input_shape[-1],), kshp=(filter_size,), stride=(stride,), mode=border_mode ) self.output_size = (input_shape[0], n_filters) + outshape ########## # Params # ########## W = self.params.get( "W", get_weights(weights_init=weights_init, shape=filter_shape, name="W", rng=mrg, # if gaussian mean=weights_mean, std=weights_std, # if uniform interval=weights_interval) ) b = self.params.get( "b", get_bias(shape=(n_filters,), name="b", init_values=bias_init) ) # Finally have the two parameters! self.params = OrderedDict([("W", W), ("b", b)]) ######################## # Computational Graph! # ######################## if border_mode in ['valid', 'full']: conved = convolution_func(self.input, W, subsample=(stride,), image_shape=input_shape, filter_shape=filter_shape, border_mode=border_mode) else: log.error("Invalid border mode: '%s'" % border_mode) raise RuntimeError("Invalid border mode: '%s'" % border_mode) self.output = activation_func(conved + b.dimshuffle('x', 0, 'x'))
def __init__(self, inputs=None, params=None, outdir='outputs/conv2d', n_filters=None, filter_size=None, stride=(1, 1), border_mode='valid', weights_init='uniform', weights_interval='montreal', weights_mean=0, weights_std=5e-3, bias_init=0, activation='rectifier', convolution='conv2d', mrg=RNG_MRG.MRG_RandomStreams(1), **kwargs): """ Initialize a 2-dimensional convolutional layer. Parameters ---------- inputs : tuple(shape, `Theano.TensorType`) The dimensionality of the inputs for this model, and the routing information for the model to accept inputs from elsewhere. `shape` will be a monad tuple representing known sizes for each dimension in the `Theano.TensorType`. Shape of the incoming data: (batch_size, num_channels, input_height, input_width). If input_size is None, it can be inferred. However, border_mode can't be 'same'. params : Dict(string_name: theano SharedVariable), optional A dictionary of model parameters (shared theano variables) that you should use when constructing this model (instead of initializing your own shared variables). This parameter is useful when you want to have two versions of the model that use the same parameters - such as siamese networks or pretraining some weights. outdir : str The directory you want outputs (parameters, images, etc.) to save to. If None, nothing will be saved. n_filters : int The number of filters to use (convolution kernels). filter_size : tuple(int) or int (filter_height, filter_width). If it is an int, size will be duplicated across height and width. stride : tuple(int) The distance between the receptive field centers of neighboring units. This is the 'stride' of the convolution operation. border_mode : str, one of 'valid', 'full' A string indicating the convolution border mode. If 'valid', the convolution is only computed where the input and the filter fully overlap. If 'full', the convolution is computed wherever the input and the filter overlap by at least one position. weights_init : str Determines the method for initializing model weights. See opendeep.utils.nnet for options. weights_interval : str or float If Uniform `weights_init`, the +- interval to use. See opendeep.utils.nnet for options. weights_mean : float If Gaussian `weights_init`, the mean value to use. weights_std : float If Gaussian `weights_init`, the standard deviation to use. bias_init : float The initial value to use for the bias parameter. Most often, the default of 0.0 is preferred. activation : str or Callable The activation function to apply to the layer. See opendeep.utils.activation for options. convolution : str or Callable The 2-dimensional convolution implementation to use. The default of 'conv2d' is normally fine because it uses theano's tensor.nnet.conv.conv2d, which cherry-picks the best implementation with a meta-optimizer if you set the theano configuration flag 'optimizer_including=conv_meta'. Otherwise, you could pass a callable function, such as cudnn or cuda-convnet if you don't want to use the meta-optimizer. mrg : random A random number generator that is used when adding noise. I recommend using Theano's sandbox.rng_mrg.MRG_RandomStreams. Notes ----- Theano's default convolution function (`theano.tensor.nnet.conv.conv2d`) does not support the 'same' border mode by default. This layer emulates it by performing a 'full' convolution and then cropping the result, which may negatively affect performance. """ super(Conv2D, self).__init__(**{arg: val for (arg, val) in locals().items() if arg is not 'self'}) ################## # specifications # ################## # expect input to be in the form (B, C, 0, 1) (batch, channel, rows, cols) # self.inputs is a list of all the input expressions (we enforce only 1, so self.inputs[0] is the input) input_shape, self.input = self.inputs[0] assert self.input.ndim == 4, "Expected 4D input variable with form (batch, channel, rows, cols)" assert len(input_shape) == 4, "Expected 4D input shape with form (batch, channel, rows, cols)" n_channels = input_shape[1] if isinstance(filter_size, int): filter_size = (filter_size, )*2 # activation function! activation_func = get_activation_function(activation) # convolution function! if convolution == 'conv2d': # using the theano flag optimizer_including=conv_meta will let this conv function optimize itself. convolution_func = conv2d else: assert callable(convolution), "Input convolution was not 'conv2d' and was not Callable." convolution_func = convolution # filter shape should be in the form (num_filters, num_channels, filter_size[0], filter_size[1]) outshape = ConvOp.getOutputShape( inshp=input_shape[-2:], kshp=filter_size, stride=stride, mode=border_mode ) self.output_size = (input_shape[0], n_filters) + outshape filter_shape = (n_filters, n_channels) + filter_size ########## # Params # ########## W = self.params.get( "W", get_weights(weights_init=weights_init, shape=filter_shape, name="W", rng=mrg, # if gaussian mean=weights_mean, std=weights_std, # if uniform interval=weights_interval) ) b = self.params.get( "b", get_bias(shape=(n_filters, ), name="b", init_values=bias_init) ) # Finally have the two parameters! self.params = OrderedDict([("W", W), ("b", b)]) ######################## # Computational Graph! # ######################## if border_mode in ['valid', 'full']: conved = convolution_func(self.input, W, subsample=stride, image_shape=input_shape, filter_shape=filter_shape, border_mode=border_mode) else: raise RuntimeError("Invalid border mode: '%s'" % border_mode) self.output = activation_func(conved + b.dimshuffle('x', 0, 'x', 'x'))