def predict_test(_x, _params, _batchnorm, n_layers=3): w = _params[0] h0 = lrelu(dnn_conv(_x, w, subsample=(2, 2), border_mode=(2, 2))) hs = [h0] for n in range(n_layers): hin = hs[-1] w, g, b = _params[1 + 3 * n:1 + 3 * (n + 1)] u = _batchnorm[n] s = _batchnorm[n + n_layers] hout = lrelu(batchnorm(dnn_conv(hin, w, subsample=(2, 2), border_mode=(2, 2)), u=u, s=s, g=g, b=b)) hs.append(hout) h = T.flatten(hs[-1], 2) y = tanh(T.dot(h, _params[-1])) return y
def discrim(X, Y, w, w2, w3, wy): yb = Y.dimshuffle(0, 1, 'x', 'x') X = conv_cond_concat(X, yb) h = T.nnet.relu(dnn_conv(X, w, subsample=(2, 2), border_mode=(2, 2)), alpha=0.2) h = conv_cond_concat(h, yb) h2 = T.nnet.relu(batchnorm( dnn_conv(h, w2, subsample=(2, 2), border_mode=(2, 2))), alpha=0.2) h2 = T.flatten(h2, 2) h2 = T.concatenate([h2, Y], axis=1) h3 = T.nnet.relu(batchnorm(T.dot(h2, w3))) h3 = T.concatenate([h3, Y], axis=1) y = T.nnet.sigmoid(T.dot(h3, wy)) return y
def conv(self, X, subsample=(2, 2), border_mode=(2, 2), atype='sigmoid', testF=False): #ConH0 = dnn_conv(X , self.W.dimshuffle(1,0,2,3), subsample=subsample, border_mode=border_mode) ConH0 = dnn_conv(X, self.W, subsample=subsample, border_mode=border_mode) if testF: ConH1 = (ConH0 - self.stat_mean.dimshuffle('x', 0, 'x', 'x')) \ / (self.stat_std.dimshuffle('x', 0, 'x', 'x') + TINY) else: mean = ConH0.mean(axis=[0, 2, 3]).dimshuffle('x', 0, 'x', 'x') std = T.mean(T.sqr(ConH0 - mean), axis=[0, 2, 3]).dimshuffle('x', 0, 'x', 'x') ConH1 = (ConH0 - mean) / T.sqrt(std + TINY) ConH2 = ConH1 * self.eta.dimshuffle('x', 0, 'x', 'x')\ + self.beta.dimshuffle('x', 0, 'x', 'x') return activation_fn_th(ConH2, atype=atype)
def run_fwd_runtime_algorithm(algo): inputs = theano.tensor.TensorType(dtype, _broadcastable)() filters = theano.tensor.TensorType(dtype, _broadcastable)() # Scale down the input values to prevent very large absolute errors # due to float rounding lower_inputs = inputs / 10 lower_filters = filters / 10 conv = dnn_conv(img=lower_inputs, kerns=lower_filters, algo=algo, precision=dtype, subsample=unit_shape, dilation=unit_shape) f = theano.function([inputs, filters], conv, mode=mode_with_gpu) if self.ndim == 3: flipped_filters = lower_filters[:, :, ::-1, ::-1, ::-1] else: flipped_filters = lower_filters[:, :, ::-1, ::-1] conv_ref = self.cpu_conv_class(subsample=unit_shape)(ref_cast(lower_inputs), flipped_filters) f_ref = theano.function([inputs, filters], conv_ref, mode='FAST_RUN') runtime_shapes = self.runtime_shapes if algo in ('time_once', 'guess_once'): runtime_shapes = [list(runtime_shapes[0])] runtime_shapes[0][0] = 5 for ntimes, (inputs_shape, filters_shape) in runtime_shapes: print('Shapes:', inputs_shape, filters_shape) for i in range(ntimes): inputs_val = np.random.random(inputs_shape).astype(dtype) filters_val = np.random.random(filters_shape).astype(dtype) gpu_res = np.asarray(f(inputs_val, filters_val)) cpu_res = f_ref(inputs_val, filters_val) self.scale_numpy_arrays_inplace(cpu_res, gpu_res, 1) utt.assert_allclose(cpu_res, gpu_res)
def run_gradweight_runtime_algorithm(algo): theano.config.dnn.conv.algo_bwd_filter = algo inputs = theano.tensor.TensorType(dtype, _broadcastable)() filters = theano.tensor.TensorType(dtype, _broadcastable)() conv = dnn_conv(img=inputs, kerns=filters, algo=algo, precision=dtype, subsample=unit_shape, dilation=unit_shape) grad_w = theano.tensor.grad(conv.sum(), [filters]) f = theano.function([inputs, filters], grad_w, mode=mode_with_gpu) assert 1 == len([node for node in f.maker.fgraph.apply_nodes if isinstance(node.op, GpuDnnConvGradW)]) assert not any(isinstance(node.op, GpuDnnConv) for node in f.maker.fgraph.apply_nodes) assert not any(isinstance(node.op, GpuDnnConvGradI) for node in f.maker.fgraph.apply_nodes) if self.ndim == 3: flipped_filters = filters[:, :, ::-1, ::-1, ::-1] else: flipped_filters = filters[:, :, ::-1, ::-1] conv_ref = self.cpu_conv_class(subsample=unit_shape)(ref_cast(inputs), flipped_filters) grad_w_ref = theano.tensor.grad(conv_ref.sum(), [filters]) f_ref = theano.function([inputs, filters], grad_w_ref, mode='FAST_RUN') runtime_shapes = self.runtime_shapes if algo in ('time_once', 'guess_once'): runtime_shapes = [list(runtime_shapes[0])] runtime_shapes[0][0] = 5 for ntimes, (inputs_shape, filters_shape) in runtime_shapes: print('Shapes:', inputs_shape, filters_shape) for i in range(ntimes): inputs_val = np.random.random(inputs_shape).astype(dtype) filters_val = np.random.random(filters_shape).astype(dtype) gpu_res = f(inputs_val, filters_val) cpu_res = f_ref(inputs_val, filters_val) utt.assert_allclose(cpu_res, np.asarray(gpu_res))
def forward(self, x, train=True): conv_out = dnn_conv( img=x, kerns=self.W, border_mode=self.border_mode, subsample=self.subsample ) return conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
def __call__(self, seq): seq = expand_dims(seq, -1).dimshuffle((0,2,1,3)) #result = T.nnet.conv2d(seq, self.W, border_mode='full', subsample=(self.stride, 1)) # T.nnet.conv2d crashes when using non-unit stride result = dnn_conv(seq, self.W, border_mode='full', subsample=(self.stride, 1)) result = squeeze(result, 3).dimshuffle((0,2,1)) return self.activation(result)
def forward(self, input): self.output = dnn_conv(input, self.weight, subsample=self.step, border_mode=self.border_mode) if self.use_bias: self.output += self.bias.dimshuffle('x', 0, 'x', 'x') return self.output
def conv(self, X, subsample=(2, 2), border_mode=(2, 2), atype='sigmoid'): ConH0 = dnn_conv(X, self.W.dimshuffle(1, 0, 2, 3), subsample=subsample, border_mode=border_mode) #ConH0 = dnn_conv(X , self.W, subsample=subsample, border_mode=border_mode) #return activation_fn_th(ConH0, atype=atype) return activation_fn_th(ConH0 + self.c.dimshuffle('x', 0, 'x', 'x'), atype=atype)
def __call__(self, seq): seq = expand_dims(seq, -1).dimshuffle((0, 2, 1, 3)) #result = T.nnet.conv2d(seq, self.W, border_mode='full', subsample=(self.stride, 1)) # T.nnet.conv2d crashes when using non-unit stride result = dnn_conv(seq, self.W, border_mode='full', subsample=(self.stride, 1)) result = squeeze(result, 3).dimshuffle((0, 2, 1)) return self.activation(result)
def apply(self, input_): if self.use_bias: W, b = self.parameters else: W, = self.parameters output = dnn_conv(input_, W, subsample=self.stride, border_mode=self.pad) if self.use_bias: output += b.dimshuffle('x', 0, 'x', 'x') return output
def run_conv_fwd(self, algo, dtype, precision, parameters): inputs_shape, filters_shape, subsample, dilation, border_mode, conv_mode, alpha, beta = parameters inputs_val = np.random.random(inputs_shape).astype(dtype) filters_val = np.random.random(filters_shape).astype(dtype) # Scale down the input values to prevent very large absolute errors # due to float rounding inputs_val /= 10 filters_val /= 10 inputs = theano.shared(inputs_val) filters = theano.shared(filters_val) if beta == 0: out = None else: out = self.array_like_conv_output(inputs_shape, filters_shape, border_mode, subsample, dilation, dtype) out /= 10 # Compile a theano function for the cuDNN implementation conv = dnn_conv(img=inputs, kerns=filters, alpha=alpha, beta=beta, out=out, border_mode=border_mode, subsample=subsample, dilation=dilation, conv_mode=conv_mode, algo=algo, precision=precision) f = theano.function([], conv, mode=mode_with_gpu) # If conv_mode is 'conv' the reference implementation should use # filters flipped according to the width, height and time axis if conv_mode == 'conv': if inputs.ndim == 5: flipped_filters = filters[:, :, ::-1, ::-1, ::-1] else: flipped_filters = filters[:, :, ::-1, ::-1] else: flipped_filters = filters # Compile a theano function for the reference implementation conv_ref = self.cpu_conv_class(border_mode=border_mode, subsample=subsample, filter_dilation=dilation)(ref_cast(inputs), flipped_filters) f_ref = theano.function([], conv_ref, mode="FAST_RUN") # Compare the results of the two implementations res_ref = f_ref() res = np.asarray(f()) if algo in cudnn.deterministic_fwd_algorithms: utt.assert_allclose(res, np.asarray(f())) atol, rtol = self.get_atol_rtol(algo, dtype, precision) if beta == 0: cpu_res = alpha * res_ref else: cpu_res = alpha * res_ref + beta * out self.scale_numpy_arrays_inplace(cpu_res, res, alpha) utt.assert_allclose(cpu_res, res, rtol=rtol, atol=atol)
def convolve(self, input, **kwargs): # by default we assume 'cross', consistent with corrmm. conv_mode = 'conv' if self.flip_filters else 'cross' border_mode = self.pad if border_mode == 'same': border_mode = tuple(s // 2 for s in self.filter_size) conved = dnn.dnn_conv(img=input, kerns=self.W, subsample=self.stride, border_mode=border_mode, conv_mode=conv_mode) return conved
def convolve(self, input, deterministic=False, train_clip=False, thresh=3, **kwargs): log_alpha = self.clip(self.log_sigma2 - T.log(self.W**2 + 1e-8)) conv_mode = 'conv' if self.flip_filters else 'cross' border_mode = self.pad clip_mask = T.ge(log_alpha, thresh) if border_mode == 'same': border_mode = tuple(s // 2 for s in self.filter_size) if deterministic: conved = dnn.dnn_conv(img=input, kerns=T.switch(T.ge(log_alpha, thresh), 0, self.W), subsample=self.stride, border_mode=border_mode, conv_mode=conv_mode) else: W = self.W if train_clip: W = T.switch(clip_mask, 0, W) conved_mu = dnn.dnn_conv(img=input, kerns=W, subsample=self.stride, border_mode=border_mode, conv_mode=conv_mode) conved_si = T.sqrt(1e-8 + dnn.dnn_conv(img=input * input, kerns=T.exp(log_alpha) * W * W, subsample=self.stride, border_mode=border_mode, conv_mode=conv_mode)) conved = conved_mu + conved_si * self._srng.normal( conved_mu.shape, avg=0, std=1) return conved
def check_dtype_config_support(dtype, precision): # We use FWD 2D to check it. # Based on documentation, algo small (CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM) # should support all configurations, for both v5.1, v6 and v7. inputs = theano.shared(np.zeros((1, 1, 2, 2), dtype=dtype)) filters = theano.shared(np.zeros((1, 1, 2, 2), dtype=dtype)) conv = dnn_conv(inputs, filters, precision=precision, algo='small') f = theano.function([], conv, mode=mode_with_gpu) try: f() except RuntimeError as e: assert 'CUDNN_STATUS_ARCH_MISMATCH' in e.message return False return True
def check_dtype_config_support(dtype, precision): # We use FWD 2D to check it. # Based on documentation, algo small (CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM) # should support all configurations, for both v5.1, v6 and v7. inputs = theano.shared(np.zeros((1, 1, 2, 2), dtype=dtype)) filters = theano.shared(np.zeros((1, 1, 2, 2), dtype=dtype)) conv = dnn_conv(inputs, filters, precision=precision, algo="small") f = theano.function([], conv, mode=mode_with_gpu) try: f() except RuntimeError as e: assert "CUDNN_STATUS_ARCH_MISMATCH" in str(e) return False return True
def __init__(self, input, convstride, padsize, b, W = None, filter_shape = None, lib_conv='cudnn', printinfo=True, input_shape=None, output_shape=None): if W == None and filter_shape == None: raise AttributeError('need to specify at least one of W and filtershape') self.get_input_shape(input,input_shape) self.filter_shape = filter_shape self.convstride = convstride self.padsize = padsize self.lib_conv = lib_conv if W: self.W = W #Weight(self.filter_shape,) else: self.W = Normal(filter_shape, mean = 0.0, std=0.1) self.b = b #Weight(self.filter_shape[3]) if filter_shape: assert W.shape == filter_shape conv_out = dnn.dnn_conv(img=self.input, # bc01 kerns=self.W.val, #bc01 subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') # broadcasting b # ReLu self.output = T.maximum(conv_out, 0) self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] if output_shape: self.output_shape = output_shape else: self.output_shape = self.get_output_shape(self.input_shape) self.name = 'Conv ({})'.format(lib_conv) if printinfo: self.print_shape()
def __init__(self, input, convstride, padsize, poolsize, poolstride, group, b, W = None, filter_shape = None, poolpad=0, mode = 'max', lrn=False, lib_conv='cudnn', printinfo=True, input_shape=None, output_shape=None, ): ''' ConvPoolLRN layer To be used in AlexNet lib_conv can be cudnn (recommended)or cudaconvnet ''' self.get_input_shape(input,input_shape) self.convstride = convstride self.padsize = padsize self.lib_conv = lib_conv self.poolsize = poolsize self.poolstride = poolstride self.poolpad = poolpad self.lrn = lrn if self.lrn: self.lrn_func = CrossChannelNormalization() if W == None and filter_shape!=None: assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) if group == 1: self.W = Normal(self.filter_shape, mean=0, std=0.01) self.b = Constant(self.filter_shape[3], val=b) else: self.filter_shape[0] = self.filter_shape[0] // 2 self.filter_shape[3] = self.filter_shape[3] // 2 # self.input_shape[0] = self.input_shape[0] / 2 # self.input_shape[3] = self.input_shape[3] / 2 channel = self.input_shape[0] self.W0 = Normal(self.filter_shape, mean=0, std=0.01) self.W1 = Normal(self.filter_shape, mean=0, std=0.01) self.b0 = Constant(self.filter_shape[3], val=b) self.b1 = Constant(self.filter_shape[3], val=b) elif W!=None and filter_shape==None: assert group ==1 self.filter_shape = W.val.shape.eval() self.W=W self.b = Constant(self.filter_shape[3], val=b) else: raise AttributeError('need to specify exactly one of W and filtershape') if lib_conv == 'cudnn': input_shuffled = self.input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 # print image_shape_shuffled # print filter_shape_shuffled if group == 1: W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv(img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 # print W0_shuffled.shape.eval()# c01b to bc01 # 96, 5, 5, 256 -> 128, 48, 5, 5 # # x_in = np.zeros((96, 27, 27, 128), dtype=np.float32) # c01b to bc01 # 96, 27, 27, 128 -> 128, 48, 27, 27 # test = input_shuffled[:, :self.channel / 2,:, :] # # print test.shape conv_out0 = \ dnn.dnn_conv(img=input_shuffled[:, :channel//2, :, :], kerns=W0_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out1 = \ dnn.dnn_conv(img=input_shuffled[:, channel//2:, :, :], kerns=W1_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b # elif lib_conv == 'cudaconvnet': # # from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs # # self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, # partial_sum=1) # # from theano.gpuarray.basic_ops import gpu_contiguous # # # Conv # if group == 1: # contiguous_input = gpu_contiguous(self.input) # contiguous_filters = gpu_contiguous(self.W.val) # conv_out = self.conv_op(contiguous_input, contiguous_filters) # conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') # else: # contiguous_input0 = gpu_contiguous( # self.input[:channel//2, :, :, :]) # contiguous_filters0 = gpu_contiguous(self.W0.val) # conv_out0 = self.conv_op( # contiguous_input0, contiguous_filters0) # conv_out0 = conv_out0 + \ # self.b0.val.dimshuffle(0, 'x', 'x', 'x') # # contiguous_input1 = gpu_contiguous( # self.input[channel//2:, :, :, :]) # contiguous_filters1 = gpu_contiguous(self.W1.val) # conv_out1 = self.conv_op( # contiguous_input1, contiguous_filters1) # conv_out1 = conv_out1 + \ # self.b1.val.dimshuffle(0, 'x', 'x', 'x') # conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # # # ReLu # conv_out = gpu_contiguous(conv_out) # self.output = T.maximum(conv_out, 0) # # # Pooling # if poolsize != 1: # from pylearn2.sandbox.cuda_convnet.pool import MaxPool # self.pool_op = MaxPool(ds=poolsize, stride=poolstride) # self.output = self.pool_op(self.output) elif lib_conv == 'corrmm': from theano.gpuarray.basic_ops import gpu_contiguous from theano.gpuarray.blas import GpuCorrMM border_mode = 'half' if padsize == (filter_shape[1]-1)//2 else (padsize, padsize) self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride), border_mode=border_mode) input_shuffled = self.input.dimshuffle(3, 0, 1, 2) # c01b to bc01 if group==1: filters = self.W.val.dimshuffle(3, 0, 1, 2) # flip top-down, left-right to compute convolution instead of correlation contiguous_filters = gpu_contiguous(filters[:, :, ::-1, ::-1]) contiguous_input = gpu_contiguous(input_shuffled) conv_out = self.corr_mm_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 contiguous_filters0 = gpu_contiguous(W0_shuffled[:, :, ::-1, ::-1]) contiguous_input0 = gpu_contiguous(input_shuffled[:, :channel // 2,:, :]) conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 contiguous_filters1 = gpu_contiguous(W1_shuffled[:, :, ::-1, ::-1]) contiguous_input1 = gpu_contiguous(input_shuffled[:, channel // 2:,:, :]) conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if poolsize != 1: from theano.gpuarray.pool import GpuPool ds_op = GpuPool(ignore_border=False, mode='max', ndim=2) self.output = ds_op(inp=self.output, ws=(poolsize,poolsize), stride=(poolstride,poolstride), pad=(0,0)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b else: NotImplementedError("lib_conv can only be cudnn or cudaconvnet for now") # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] if output_shape: self.output_shape = output_shape else: self.output_shape = self.get_output_shape(self.input_shape) self.name = 'ConvPoolLRN(%s)' % lib_conv if printinfo: self.print_shape()
def __init__(self, input, convstride, padsize, poolsize, poolstride, b, W = None, filter_shape = None, poolpad=0, mode = 'max', lrn=False, lib_conv='cudnn', printinfo=True, input_shape=None, output_shape=None ): if W == None and filter_shape == None: raise AttributeError('need to specify at least one of W and filtershape') self.get_input_shape(input,input_shape) self.filter_shape = filter_shape self.convstride = convstride self.padsize = padsize self.lib_conv = lib_conv if W: self.W = W #Weight(self.filter_shape,) else: self.W = Normal(filter_shape, mean = 0.0, std=0.1) self.b = b #Weight(self.filter_shape[3]) self.channel = self.input_shape[1] self.lrn = lrn conv_out = dnn.dnn_conv(img=self.input, kerns=self.W.val, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') # ReLu self.output = T.maximum(conv_out, 0) # Pool self.poolsize = poolsize self.poolstride = poolstride self.poolpad = poolpad if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride), mode='max', pad=(poolpad, poolpad)) # LRN if self.lrn: self.lrn_func = CrossChannelNormalization() # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] if output_shape: self.output_shape = output_shape else: self.output_shape = self.get_output_shape(self.input_shape) self.name = 'ConvPoolLRN(%s)' % lib_conv if printinfo: self.print_shape()
def op(self, state): X = self.l_in.op(state=state) return dnn_conv(X, self.w, subsample=self.stride, border_mode=self.pad)
def conv(self, X, subsample=(2, 2), border_mode=(2, 2), conv_mode='conv', atype='sigmoid'): ConH0 = dnn_conv(X , self.W, subsample=subsample, border_mode=border_mode) return activation_fn_th(ConH0, atype=atype)
def local_abstractconv_cudnn_alt(node): if not isinstance(node.op, (AbstractConv2d, AbstractConv2d_gradWeights, AbstractConv2d_gradInputs)): return if version(raises=False) < 6000 and node.op.filter_dilation != (1, 1): return None if node.op.unshared: return None if isinstance(node.op.border_mode, tuple) and any( isinstance(p, tuple) for p in node.op.border_mode): # Asymmetric padding not yet supported return None inp1 = node.inputs[0] inp2 = node.inputs[1] if not dnn_available(inp1.type.context_name): return op = node.op border_mode = node.op.border_mode subsample = node.op.subsample filter_dilation = node.op.filter_dilation num_groups = node.op.num_groups precision, _ = get_precision(None, [inp1, inp2]) if node.op.filter_flip: conv_mode = "conv" else: conv_mode = "cross" if isinstance(op, AbstractConv2d): if border_mode == "half" or subsample != (1, 1) or num_groups != 1: return None if border_mode == "full": direction_hint = "bprop inputs" elif border_mode == "valid" and filter_dilation == (1, 1): direction_hint = "bprop weights" else: return None rval = dnn_conv( inp1, inp2, border_mode=border_mode, subsample=subsample, dilation=filter_dilation, direction_hint=direction_hint, conv_mode=conv_mode, num_groups=num_groups, ) elif isinstance(op, AbstractConv2d_gradWeights): if (border_mode == "valid" and subsample == (1, 1) and filter_dilation == (1, 1) and num_groups == 1): img = gpu_contiguous(inp1) topgrad = gpu_contiguous(inp2) ctx_name = infer_context_name(img, topgrad) img = gpu_contiguous(img.dimshuffle(1, 0, 2, 3)) topgrad = gpu_contiguous(topgrad.dimshuffle(1, 0, 2, 3)) ishape = [shape_i_op(i)(img) for i in range(img.ndim)] tshape = [shape_i_op(i)(topgrad) for i in range(topgrad.ndim)] out_shp = get_conv_output_shape( ishape, tshape, border_mode=border_mode, subsample=subsample, filter_dilation=filter_dilation, ) out_shp = assert_conv_shape(out_shp) out = GpuAllocEmpty(dtype=img.dtype, context_name=ctx_name)(*out_shp) desc = GpuDnnConvDesc( border_mode=border_mode, subsample=subsample, dilation=filter_dilation, conv_mode="cross", precision=precision, )(out.shape) conv = GpuDnnConv(algo=None, num_groups=num_groups)(img, topgrad, out, desc) if conv_mode == "conv": conv = conv[:, :, ::-1, ::-1] rval = as_gpuarray_variable(conv.dimshuffle(1, 0, 2, 3), ctx_name) else: return None elif isinstance(op, AbstractConv2d_gradInputs): if border_mode == "valid" and subsample == (1, 1) and num_groups == 1: kerns = gpu_contiguous(inp1.dimshuffle(1, 0, 2, 3)) topgrad = gpu_contiguous(inp2) ctx_name = infer_context_name(kerns, topgrad) conv_mode = "cross" if conv_mode == "conv" else "conv" desc = GpuDnnConvDesc( border_mode="full", subsample=subsample, dilation=filter_dilation, conv_mode=conv_mode, precision=precision, )(kerns.shape) tshape = [shape_i_op(i)(topgrad) for i in range(topgrad.ndim)] kshape = [shape_i_op(i)(kerns) for i in range(kerns.ndim)] shape = get_conv_output_shape( tshape, kshape, border_mode="full", subsample=subsample, filter_dilation=filter_dilation, ) shape = assert_conv_shape(shape) out = GpuAllocEmpty(dtype=topgrad.dtype, context_name=ctx_name)(*shape) rval = GpuDnnConv(algo=None, num_groups=num_groups)(topgrad, kerns, out, desc) else: return None return [rval]
def op(self, state): X = self.l_in.op(state=state) return dnn_conv(X, self.w, subsample=self.stride, border_mode=self.pad)
def __init__(self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', verbose=False): ''' lib_conv can be cudnn (recommended)or cudaconvnet ''' self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv self.verbose = verbose assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudnn': input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 # print image_shape_shuffled # print filter_shape_shuffled if group == 1: W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv( img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out0 = \ dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2, :, :], kerns=W0_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out1 = \ dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:, :, :], kerns=W1_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b # elif lib_conv == 'cudaconvnet': # from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs # # self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, # partial_sum=1) # # from theano.sandbox.cuda.basic_ops import gpu_contiguous # # # Conv # if group == 1: # contiguous_input = gpu_contiguous(input) # contiguous_filters = gpu_contiguous(self.W.val) # conv_out = self.conv_op(contiguous_input, contiguous_filters) # conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') # else: # contiguous_input0 = gpu_contiguous( # input[:self.channel / 2, :, :, :]) # contiguous_filters0 = gpu_contiguous(self.W0.val) # conv_out0 = self.conv_op( # contiguous_input0, contiguous_filters0) # conv_out0 = conv_out0 + \ # self.b0.val.dimshuffle(0, 'x', 'x', 'x') # # contiguous_input1 = gpu_contiguous( # input[self.channel / 2:, :, :, :]) # contiguous_filters1 = gpu_contiguous(self.W1.val) # conv_out1 = self.conv_op( # contiguous_input1, contiguous_filters1) # conv_out1 = conv_out1 + \ # self.b1.val.dimshuffle(0, 'x', 'x', 'x') # conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # # # ReLu # conv_out = gpu_contiguous(conv_out) # self.output = T.maximum(conv_out, 0) # # # Pooling # if self.poolsize != 1: # from pylearn2.sandbox.cuda_convnet.pool import MaxPool # self.pool_op = MaxPool(ds=poolsize, stride=poolstride) # self.output = self.pool_op(self.output) # # elif lib_conv == 'corrmm': # # from theano.sandbox.cuda.basic_ops import gpu_contiguous # from theano.sandbox.cuda.blas import GpuCorrMM # # border_mode = 'half' if padsize == (filter_shape[1]-1)/2 else (padsize, padsize) # self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride), # border_mode=border_mode) # flip_filters=True # input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # # # if group==1: # # filters = self.W.val.dimshuffle(3, 0, 1, 2) # # if flip_filters: # filters = filters[:, :, ::-1, ::-1] # flip top-down, left-right # contiguous_filters = gpu_contiguous(filters) # contiguous_input = gpu_contiguous(input_shuffled) # # conv_out = self.corr_mm_op(contiguous_input, contiguous_filters) # conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') # # else: # # W0_shuffled = \ # self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 # if flip_filters: # W0_shuffled = W0_shuffled[:, :, ::-1, ::-1] # # contiguous_filters0 = gpu_contiguous(W0_shuffled) # contiguous_input0 = gpu_contiguous(input_shuffled[:, :self.channel / 2,:, :]) # # conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0) # conv_out0 = conv_out0 + \ # self.b0.val.dimshuffle('x', 0, 'x', 'x') # # W1_shuffled = \ # self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 # if flip_filters: # W1_shuffled = W1_shuffled[:, :, ::-1, ::-1] # # contiguous_filters1 = gpu_contiguous(W1_shuffled) # contiguous_input1 = gpu_contiguous(input_shuffled[:, self.channel / 2:,:, :]) # # conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1) # conv_out1 = conv_out1 + \ # self.b1.val.dimshuffle('x', 0, 'x', 'x') # conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # # # ReLu # self.output = T.maximum(conv_out, 0) # # # Pooling # if self.poolsize != 1: # from theano.tensor.signal import downsample # self.output = downsample.max_pool_2d(self.output, # ds=(poolsize,poolsize), # st=(poolstride,poolstride), # ignore_border=False, # padding=(0,0), # mode='max', # ) # # self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b else: NotImplementedError("lib_conv can only be cudnn for now") # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] if self.verbose: print "conv ({}) layer with shape_in: {}".format( lib_conv, str(image_shape))
def __init__(self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', verbose=False ): ''' lib_conv can be cudnn (recommended)or cudaconvnet ''' self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv self.verbose = verbose assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudnn': input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 # print image_shape_shuffled # print filter_shape_shuffled if group == 1: W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv(img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out0 = \ dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2, :, :], kerns=W0_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out1 = \ dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:, :, :], kerns=W1_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b # elif lib_conv == 'cudaconvnet': # from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs # # self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, # partial_sum=1) # # from theano.sandbox.cuda.basic_ops import gpu_contiguous # # # Conv # if group == 1: # contiguous_input = gpu_contiguous(input) # contiguous_filters = gpu_contiguous(self.W.val) # conv_out = self.conv_op(contiguous_input, contiguous_filters) # conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') # else: # contiguous_input0 = gpu_contiguous( # input[:self.channel / 2, :, :, :]) # contiguous_filters0 = gpu_contiguous(self.W0.val) # conv_out0 = self.conv_op( # contiguous_input0, contiguous_filters0) # conv_out0 = conv_out0 + \ # self.b0.val.dimshuffle(0, 'x', 'x', 'x') # # contiguous_input1 = gpu_contiguous( # input[self.channel / 2:, :, :, :]) # contiguous_filters1 = gpu_contiguous(self.W1.val) # conv_out1 = self.conv_op( # contiguous_input1, contiguous_filters1) # conv_out1 = conv_out1 + \ # self.b1.val.dimshuffle(0, 'x', 'x', 'x') # conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # # # ReLu # conv_out = gpu_contiguous(conv_out) # self.output = T.maximum(conv_out, 0) # # # Pooling # if self.poolsize != 1: # from pylearn2.sandbox.cuda_convnet.pool import MaxPool # self.pool_op = MaxPool(ds=poolsize, stride=poolstride) # self.output = self.pool_op(self.output) # # elif lib_conv == 'corrmm': # # from theano.sandbox.cuda.basic_ops import gpu_contiguous # from theano.sandbox.cuda.blas import GpuCorrMM # # border_mode = 'half' if padsize == (filter_shape[1]-1)/2 else (padsize, padsize) # self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride), # border_mode=border_mode) # flip_filters=True # input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # # # if group==1: # # filters = self.W.val.dimshuffle(3, 0, 1, 2) # # if flip_filters: # filters = filters[:, :, ::-1, ::-1] # flip top-down, left-right # contiguous_filters = gpu_contiguous(filters) # contiguous_input = gpu_contiguous(input_shuffled) # # conv_out = self.corr_mm_op(contiguous_input, contiguous_filters) # conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') # # else: # # W0_shuffled = \ # self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 # if flip_filters: # W0_shuffled = W0_shuffled[:, :, ::-1, ::-1] # # contiguous_filters0 = gpu_contiguous(W0_shuffled) # contiguous_input0 = gpu_contiguous(input_shuffled[:, :self.channel / 2,:, :]) # # conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0) # conv_out0 = conv_out0 + \ # self.b0.val.dimshuffle('x', 0, 'x', 'x') # # W1_shuffled = \ # self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 # if flip_filters: # W1_shuffled = W1_shuffled[:, :, ::-1, ::-1] # # contiguous_filters1 = gpu_contiguous(W1_shuffled) # contiguous_input1 = gpu_contiguous(input_shuffled[:, self.channel / 2:,:, :]) # # conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1) # conv_out1 = conv_out1 + \ # self.b1.val.dimshuffle('x', 0, 'x', 'x') # conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # # # ReLu # self.output = T.maximum(conv_out, 0) # # # Pooling # if self.poolsize != 1: # from theano.tensor.signal import downsample # self.output = downsample.max_pool_2d(self.output, # ds=(poolsize,poolsize), # st=(poolstride,poolstride), # ignore_border=False, # padding=(0,0), # mode='max', # ) # # self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b else: NotImplementedError("lib_conv can only be cudnn for now") # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] if self.verbose: print "conv ({}) layer with shape_in: {}".format(lib_conv, str(image_shape))