def predict_test(_x, _params, _batchnorm, n_layers=3):
w = _params[0]
h0 = lrelu(dnn_conv(_x, w, subsample=(2, 2), border_mode=(2, 2)))
hs = [h0]
for n in range(n_layers):
hin = hs[-1]
w, g, b = _params[1 + 3 * n:1 + 3 * (n + 1)]
u = _batchnorm[n]
s = _batchnorm[n + n_layers]
hout = lrelu(batchnorm(dnn_conv(hin, w, subsample=(2, 2), border_mode=(2, 2)), u=u, s=s, g=g, b=b))
hs.append(hout)
h = T.flatten(hs[-1], 2)
y = tanh(T.dot(h, _params[-1]))
return y
def discrim(X, Y, w, w2, w3, wy):
yb = Y.dimshuffle(0, 1, 'x', 'x')
X = conv_cond_concat(X, yb)
h = T.nnet.relu(dnn_conv(X, w, subsample=(2, 2), border_mode=(2, 2)),
alpha=0.2)
h = conv_cond_concat(h, yb)
h2 = T.nnet.relu(batchnorm(
dnn_conv(h, w2, subsample=(2, 2), border_mode=(2, 2))),
alpha=0.2)
h2 = T.flatten(h2, 2)
h2 = T.concatenate([h2, Y], axis=1)
h3 = T.nnet.relu(batchnorm(T.dot(h2, w3)))
h3 = T.concatenate([h3, Y], axis=1)
y = T.nnet.sigmoid(T.dot(h3, wy))
return y
def conv(self,
X,
subsample=(2, 2),
border_mode=(2, 2),
atype='sigmoid',
testF=False):
#ConH0 = dnn_conv(X , self.W.dimshuffle(1,0,2,3), subsample=subsample, border_mode=border_mode)
ConH0 = dnn_conv(X,
self.W,
subsample=subsample,
border_mode=border_mode)
if testF:
ConH1 = (ConH0 - self.stat_mean.dimshuffle('x', 0, 'x', 'x')) \
/ (self.stat_std.dimshuffle('x', 0, 'x', 'x') + TINY)
else:
mean = ConH0.mean(axis=[0, 2, 3]).dimshuffle('x', 0, 'x', 'x')
std = T.mean(T.sqr(ConH0 - mean),
axis=[0, 2, 3]).dimshuffle('x', 0, 'x', 'x')
ConH1 = (ConH0 - mean) / T.sqrt(std + TINY)
ConH2 = ConH1 * self.eta.dimshuffle('x', 0, 'x', 'x')\
+ self.beta.dimshuffle('x', 0, 'x', 'x')
return activation_fn_th(ConH2, atype=atype)
def run_fwd_runtime_algorithm(algo):
inputs = theano.tensor.TensorType(dtype, _broadcastable)()
filters = theano.tensor.TensorType(dtype, _broadcastable)()
# Scale down the input values to prevent very large absolute errors
# due to float rounding
lower_inputs = inputs / 10
lower_filters = filters / 10
conv = dnn_conv(img=lower_inputs, kerns=lower_filters, algo=algo, precision=dtype,
subsample=unit_shape, dilation=unit_shape)
f = theano.function([inputs, filters], conv, mode=mode_with_gpu)
if self.ndim == 3:
flipped_filters = lower_filters[:, :, ::-1, ::-1, ::-1]
else:
flipped_filters = lower_filters[:, :, ::-1, ::-1]
conv_ref = self.cpu_conv_class(subsample=unit_shape)(ref_cast(lower_inputs), flipped_filters)
f_ref = theano.function([inputs, filters], conv_ref, mode='FAST_RUN')
runtime_shapes = self.runtime_shapes
if algo in ('time_once', 'guess_once'):
runtime_shapes = [list(runtime_shapes[0])]
runtime_shapes[0][0] = 5
for ntimes, (inputs_shape, filters_shape) in runtime_shapes:
print('Shapes:', inputs_shape, filters_shape)
for i in range(ntimes):
inputs_val = np.random.random(inputs_shape).astype(dtype)
filters_val = np.random.random(filters_shape).astype(dtype)
gpu_res = np.asarray(f(inputs_val, filters_val))
cpu_res = f_ref(inputs_val, filters_val)
self.scale_numpy_arrays_inplace(cpu_res, gpu_res, 1)
utt.assert_allclose(cpu_res, gpu_res)
def run_gradweight_runtime_algorithm(algo):
theano.config.dnn.conv.algo_bwd_filter = algo
inputs = theano.tensor.TensorType(dtype, _broadcastable)()
filters = theano.tensor.TensorType(dtype, _broadcastable)()
conv = dnn_conv(img=inputs, kerns=filters, algo=algo, precision=dtype,
subsample=unit_shape, dilation=unit_shape)
grad_w = theano.tensor.grad(conv.sum(), [filters])
f = theano.function([inputs, filters], grad_w, mode=mode_with_gpu)
assert 1 == len([node for node in f.maker.fgraph.apply_nodes if isinstance(node.op, GpuDnnConvGradW)])
assert not any(isinstance(node.op, GpuDnnConv) for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, GpuDnnConvGradI) for node in f.maker.fgraph.apply_nodes)
if self.ndim == 3:
flipped_filters = filters[:, :, ::-1, ::-1, ::-1]
else:
flipped_filters = filters[:, :, ::-1, ::-1]
conv_ref = self.cpu_conv_class(subsample=unit_shape)(ref_cast(inputs), flipped_filters)
grad_w_ref = theano.tensor.grad(conv_ref.sum(), [filters])
f_ref = theano.function([inputs, filters], grad_w_ref, mode='FAST_RUN')
runtime_shapes = self.runtime_shapes
if algo in ('time_once', 'guess_once'):
runtime_shapes = [list(runtime_shapes[0])]
runtime_shapes[0][0] = 5
for ntimes, (inputs_shape, filters_shape) in runtime_shapes:
print('Shapes:', inputs_shape, filters_shape)
for i in range(ntimes):
inputs_val = np.random.random(inputs_shape).astype(dtype)
filters_val = np.random.random(filters_shape).astype(dtype)
gpu_res = f(inputs_val, filters_val)
cpu_res = f_ref(inputs_val, filters_val)
utt.assert_allclose(cpu_res, np.asarray(gpu_res))
def forward(self, x, train=True):
conv_out = dnn_conv(
img=x,
kerns=self.W,
border_mode=self.border_mode,
subsample=self.subsample
)
return conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
def __call__(self, seq):
seq = expand_dims(seq, -1).dimshuffle((0,2,1,3))
#result = T.nnet.conv2d(seq, self.W, border_mode='full', subsample=(self.stride, 1))
# T.nnet.conv2d crashes when using non-unit stride
result = dnn_conv(seq, self.W, border_mode='full', subsample=(self.stride, 1))
result = squeeze(result, 3).dimshuffle((0,2,1))
return self.activation(result)
def forward(self, input):
self.output = dnn_conv(input,
self.weight,
subsample=self.step,
border_mode=self.border_mode)
if self.use_bias:
self.output += self.bias.dimshuffle('x', 0, 'x', 'x')
return self.output
def conv(self, X, subsample=(2, 2), border_mode=(2, 2), atype='sigmoid'):
ConH0 = dnn_conv(X,
self.W.dimshuffle(1, 0, 2, 3),
subsample=subsample,
border_mode=border_mode)
#ConH0 = dnn_conv(X , self.W, subsample=subsample, border_mode=border_mode)
#return activation_fn_th(ConH0, atype=atype)
return activation_fn_th(ConH0 + self.c.dimshuffle('x', 0, 'x', 'x'),
atype=atype)
def __call__(self, seq):
seq = expand_dims(seq, -1).dimshuffle((0, 2, 1, 3))
#result = T.nnet.conv2d(seq, self.W, border_mode='full', subsample=(self.stride, 1))
# T.nnet.conv2d crashes when using non-unit stride
result = dnn_conv(seq,
self.W,
border_mode='full',
subsample=(self.stride, 1))
result = squeeze(result, 3).dimshuffle((0, 2, 1))
return self.activation(result)
def apply(self, input_):
if self.use_bias:
W, b = self.parameters
else:
W, = self.parameters
output = dnn_conv(input_,
W,
subsample=self.stride,
border_mode=self.pad)
if self.use_bias:
output += b.dimshuffle('x', 0, 'x', 'x')
return output
def run_conv_fwd(self, algo, dtype, precision, parameters):
inputs_shape, filters_shape, subsample, dilation, border_mode, conv_mode, alpha, beta = parameters
inputs_val = np.random.random(inputs_shape).astype(dtype)
filters_val = np.random.random(filters_shape).astype(dtype)
# Scale down the input values to prevent very large absolute errors
# due to float rounding
inputs_val /= 10
filters_val /= 10
inputs = theano.shared(inputs_val)
filters = theano.shared(filters_val)
if beta == 0:
out = None
else:
out = self.array_like_conv_output(inputs_shape, filters_shape, border_mode, subsample, dilation, dtype)
out /= 10
# Compile a theano function for the cuDNN implementation
conv = dnn_conv(img=inputs, kerns=filters, alpha=alpha, beta=beta, out=out, border_mode=border_mode,
subsample=subsample, dilation=dilation, conv_mode=conv_mode, algo=algo, precision=precision)
f = theano.function([], conv, mode=mode_with_gpu)
# If conv_mode is 'conv' the reference implementation should use
# filters flipped according to the width, height and time axis
if conv_mode == 'conv':
if inputs.ndim == 5:
flipped_filters = filters[:, :, ::-1, ::-1, ::-1]
else:
flipped_filters = filters[:, :, ::-1, ::-1]
else:
flipped_filters = filters
# Compile a theano function for the reference implementation
conv_ref = self.cpu_conv_class(border_mode=border_mode,
subsample=subsample,
filter_dilation=dilation)(ref_cast(inputs), flipped_filters)
f_ref = theano.function([], conv_ref, mode="FAST_RUN")
# Compare the results of the two implementations
res_ref = f_ref()
res = np.asarray(f())
if algo in cudnn.deterministic_fwd_algorithms:
utt.assert_allclose(res, np.asarray(f()))
atol, rtol = self.get_atol_rtol(algo, dtype, precision)
if beta == 0:
cpu_res = alpha * res_ref
else:
cpu_res = alpha * res_ref + beta * out
self.scale_numpy_arrays_inplace(cpu_res, res, alpha)
utt.assert_allclose(cpu_res, res, rtol=rtol, atol=atol)
def convolve(self, input, **kwargs):
# by default we assume 'cross', consistent with corrmm.
conv_mode = 'conv' if self.flip_filters else 'cross'
border_mode = self.pad
if border_mode == 'same':
border_mode = tuple(s // 2 for s in self.filter_size)
conved = dnn.dnn_conv(img=input,
kerns=self.W,
subsample=self.stride,
border_mode=border_mode,
conv_mode=conv_mode)
return conved
def convolve(self,
input,
deterministic=False,
train_clip=False,
thresh=3,
**kwargs):
log_alpha = self.clip(self.log_sigma2 - T.log(self.W**2 + 1e-8))
conv_mode = 'conv' if self.flip_filters else 'cross'
border_mode = self.pad
clip_mask = T.ge(log_alpha, thresh)
if border_mode == 'same':
border_mode = tuple(s // 2 for s in self.filter_size)
if deterministic:
conved = dnn.dnn_conv(img=input,
kerns=T.switch(T.ge(log_alpha, thresh), 0,
self.W),
subsample=self.stride,
border_mode=border_mode,
conv_mode=conv_mode)
else:
W = self.W
if train_clip:
W = T.switch(clip_mask, 0, W)
conved_mu = dnn.dnn_conv(img=input,
kerns=W,
subsample=self.stride,
border_mode=border_mode,
conv_mode=conv_mode)
conved_si = T.sqrt(1e-8 +
dnn.dnn_conv(img=input * input,
kerns=T.exp(log_alpha) * W * W,
subsample=self.stride,
border_mode=border_mode,
conv_mode=conv_mode))
conved = conved_mu + conved_si * self._srng.normal(
conved_mu.shape, avg=0, std=1)
return conved
def check_dtype_config_support(dtype, precision):
# We use FWD 2D to check it.
# Based on documentation, algo small (CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM)
# should support all configurations, for both v5.1, v6 and v7.
inputs = theano.shared(np.zeros((1, 1, 2, 2), dtype=dtype))
filters = theano.shared(np.zeros((1, 1, 2, 2), dtype=dtype))
conv = dnn_conv(inputs, filters, precision=precision, algo='small')
f = theano.function([], conv, mode=mode_with_gpu)
try:
f()
except RuntimeError as e:
assert 'CUDNN_STATUS_ARCH_MISMATCH' in e.message
return False
return True
def check_dtype_config_support(dtype, precision):
# We use FWD 2D to check it.
# Based on documentation, algo small (CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM)
# should support all configurations, for both v5.1, v6 and v7.
inputs = theano.shared(np.zeros((1, 1, 2, 2), dtype=dtype))
filters = theano.shared(np.zeros((1, 1, 2, 2), dtype=dtype))
conv = dnn_conv(inputs, filters, precision=precision, algo="small")
f = theano.function([], conv, mode=mode_with_gpu)
try:
f()
except RuntimeError as e:
assert "CUDNN_STATUS_ARCH_MISMATCH" in str(e)
return False
return True
def __init__(self, input, convstride, padsize,
b, W = None, filter_shape = None,
lib_conv='cudnn', printinfo=True,
input_shape=None, output_shape=None):
if W == None and filter_shape == None:
raise AttributeError('need to specify at least one of W and filtershape')
self.get_input_shape(input,input_shape)
self.filter_shape = filter_shape
self.convstride = convstride
self.padsize = padsize
self.lib_conv = lib_conv
if W:
self.W = W #Weight(self.filter_shape,)
else:
self.W = Normal(filter_shape, mean = 0.0, std=0.1)
self.b = b #Weight(self.filter_shape[3])
if filter_shape:
assert W.shape == filter_shape
conv_out = dnn.dnn_conv(img=self.input, # bc01
kerns=self.W.val, #bc01
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') # broadcasting b
# ReLu
self.output = T.maximum(conv_out, 0)
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
if output_shape:
self.output_shape = output_shape
else:
self.output_shape = self.get_output_shape(self.input_shape)
self.name = 'Conv ({})'.format(lib_conv)
if printinfo: self.print_shape()
def __init__(self, input, convstride, padsize, poolsize, poolstride, group,
b, W = None, filter_shape = None,
poolpad=0, mode = 'max',
lrn=False, lib_conv='cudnn', printinfo=True,
input_shape=None, output_shape=None,
):
'''
ConvPoolLRN layer
To be used in AlexNet
lib_conv can be cudnn (recommended)or cudaconvnet
'''
self.get_input_shape(input,input_shape)
self.convstride = convstride
self.padsize = padsize
self.lib_conv = lib_conv
self.poolsize = poolsize
self.poolstride = poolstride
self.poolpad = poolpad
self.lrn = lrn
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if W == None and filter_shape!=None:
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
if group == 1:
self.W = Normal(self.filter_shape, mean=0, std=0.01)
self.b = Constant(self.filter_shape[3], val=b)
else:
self.filter_shape[0] = self.filter_shape[0] // 2
self.filter_shape[3] = self.filter_shape[3] // 2
# self.input_shape[0] = self.input_shape[0] / 2
# self.input_shape[3] = self.input_shape[3] / 2
channel = self.input_shape[0]
self.W0 = Normal(self.filter_shape, mean=0, std=0.01)
self.W1 = Normal(self.filter_shape, mean=0, std=0.01)
self.b0 = Constant(self.filter_shape[3], val=b)
self.b1 = Constant(self.filter_shape[3], val=b)
elif W!=None and filter_shape==None:
assert group ==1
self.filter_shape = W.val.shape.eval()
self.W=W
self.b = Constant(self.filter_shape[3], val=b)
else:
raise AttributeError('need to specify exactly one of W and filtershape')
if lib_conv == 'cudnn':
input_shuffled = self.input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
# print image_shape_shuffled
# print filter_shape_shuffled
if group == 1:
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
# print W0_shuffled.shape.eval()# c01b to bc01 # 96, 5, 5, 256 -> 128, 48, 5, 5
#
# x_in = np.zeros((96, 27, 27, 128), dtype=np.float32) # c01b to bc01 # 96, 27, 27, 128 -> 128, 48, 27, 27
# test = input_shuffled[:, :self.channel / 2,:, :]
#
# print test.shape
conv_out0 = \
dnn.dnn_conv(img=input_shuffled[:, :channel//2,
:, :],
kerns=W0_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out1 = \
dnn.dnn_conv(img=input_shuffled[:, channel//2:,
:, :],
kerns=W1_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
# elif lib_conv == 'cudaconvnet':
#
# from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
#
# self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride,
# partial_sum=1)
#
# from theano.gpuarray.basic_ops import gpu_contiguous
#
# # Conv
# if group == 1:
# contiguous_input = gpu_contiguous(self.input)
# contiguous_filters = gpu_contiguous(self.W.val)
# conv_out = self.conv_op(contiguous_input, contiguous_filters)
# conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
# else:
# contiguous_input0 = gpu_contiguous(
# self.input[:channel//2, :, :, :])
# contiguous_filters0 = gpu_contiguous(self.W0.val)
# conv_out0 = self.conv_op(
# contiguous_input0, contiguous_filters0)
# conv_out0 = conv_out0 + \
# self.b0.val.dimshuffle(0, 'x', 'x', 'x')
#
# contiguous_input1 = gpu_contiguous(
# self.input[channel//2:, :, :, :])
# contiguous_filters1 = gpu_contiguous(self.W1.val)
# conv_out1 = self.conv_op(
# contiguous_input1, contiguous_filters1)
# conv_out1 = conv_out1 + \
# self.b1.val.dimshuffle(0, 'x', 'x', 'x')
# conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
#
# # ReLu
# conv_out = gpu_contiguous(conv_out)
# self.output = T.maximum(conv_out, 0)
#
# # Pooling
# if poolsize != 1:
# from pylearn2.sandbox.cuda_convnet.pool import MaxPool
# self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
# self.output = self.pool_op(self.output)
elif lib_conv == 'corrmm':
from theano.gpuarray.basic_ops import gpu_contiguous
from theano.gpuarray.blas import GpuCorrMM
border_mode = 'half' if padsize == (filter_shape[1]-1)//2 else (padsize, padsize)
self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride),
border_mode=border_mode)
input_shuffled = self.input.dimshuffle(3, 0, 1, 2) # c01b to bc01
if group==1:
filters = self.W.val.dimshuffle(3, 0, 1, 2)
# flip top-down, left-right to compute convolution instead of correlation
contiguous_filters = gpu_contiguous(filters[:, :, ::-1, ::-1])
contiguous_input = gpu_contiguous(input_shuffled)
conv_out = self.corr_mm_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
contiguous_filters0 = gpu_contiguous(W0_shuffled[:, :, ::-1, ::-1])
contiguous_input0 = gpu_contiguous(input_shuffled[:, :channel // 2,:, :])
conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
contiguous_filters1 = gpu_contiguous(W1_shuffled[:, :, ::-1, ::-1])
contiguous_input1 = gpu_contiguous(input_shuffled[:, channel // 2:,:, :])
conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if poolsize != 1:
from theano.gpuarray.pool import GpuPool
ds_op = GpuPool(ignore_border=False, mode='max', ndim=2)
self.output = ds_op(inp=self.output, ws=(poolsize,poolsize),
stride=(poolstride,poolstride), pad=(0,0))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
else:
NotImplementedError("lib_conv can only be cudnn or cudaconvnet for now")
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
if output_shape:
self.output_shape = output_shape
else:
self.output_shape = self.get_output_shape(self.input_shape)
self.name = 'ConvPoolLRN(%s)' % lib_conv
if printinfo: self.print_shape()
def __init__(self, input, convstride, padsize, poolsize, poolstride,
b, W = None, filter_shape = None,
poolpad=0, mode = 'max',
lrn=False, lib_conv='cudnn', printinfo=True,
input_shape=None, output_shape=None
):
if W == None and filter_shape == None:
raise AttributeError('need to specify at least one of W and filtershape')
self.get_input_shape(input,input_shape)
self.filter_shape = filter_shape
self.convstride = convstride
self.padsize = padsize
self.lib_conv = lib_conv
if W:
self.W = W #Weight(self.filter_shape,)
else:
self.W = Normal(filter_shape, mean = 0.0, std=0.1)
self.b = b #Weight(self.filter_shape[3])
self.channel = self.input_shape[1]
self.lrn = lrn
conv_out = dnn.dnn_conv(img=self.input,
kerns=self.W.val,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
# ReLu
self.output = T.maximum(conv_out, 0)
# Pool
self.poolsize = poolsize
self.poolstride = poolstride
self.poolpad = poolpad
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride),
mode='max', pad=(poolpad, poolpad))
# LRN
if self.lrn:
self.lrn_func = CrossChannelNormalization()
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
if output_shape:
self.output_shape = output_shape
else:
self.output_shape = self.get_output_shape(self.input_shape)
self.name = 'ConvPoolLRN(%s)' % lib_conv
if printinfo: self.print_shape()
def op(self, state):
X = self.l_in.op(state=state)
return dnn_conv(X, self.w, subsample=self.stride, border_mode=self.pad)
def conv(self, X, subsample=(2, 2), border_mode=(2, 2), conv_mode='conv', atype='sigmoid'):
ConH0 = dnn_conv(X , self.W, subsample=subsample, border_mode=border_mode)
return activation_fn_th(ConH0, atype=atype)
def local_abstractconv_cudnn_alt(node):
if not isinstance(node.op, (AbstractConv2d, AbstractConv2d_gradWeights,
AbstractConv2d_gradInputs)):
return
if version(raises=False) < 6000 and node.op.filter_dilation != (1, 1):
return None
if node.op.unshared:
return None
if isinstance(node.op.border_mode, tuple) and any(
isinstance(p, tuple) for p in node.op.border_mode):
# Asymmetric padding not yet supported
return None
inp1 = node.inputs[0]
inp2 = node.inputs[1]
if not dnn_available(inp1.type.context_name):
return
op = node.op
border_mode = node.op.border_mode
subsample = node.op.subsample
filter_dilation = node.op.filter_dilation
num_groups = node.op.num_groups
precision, _ = get_precision(None, [inp1, inp2])
if node.op.filter_flip:
conv_mode = "conv"
else:
conv_mode = "cross"
if isinstance(op, AbstractConv2d):
if border_mode == "half" or subsample != (1, 1) or num_groups != 1:
return None
if border_mode == "full":
direction_hint = "bprop inputs"
elif border_mode == "valid" and filter_dilation == (1, 1):
direction_hint = "bprop weights"
else:
return None
rval = dnn_conv(
inp1,
inp2,
border_mode=border_mode,
subsample=subsample,
dilation=filter_dilation,
direction_hint=direction_hint,
conv_mode=conv_mode,
num_groups=num_groups,
)
elif isinstance(op, AbstractConv2d_gradWeights):
if (border_mode == "valid" and subsample == (1, 1)
and filter_dilation == (1, 1) and num_groups == 1):
img = gpu_contiguous(inp1)
topgrad = gpu_contiguous(inp2)
ctx_name = infer_context_name(img, topgrad)
img = gpu_contiguous(img.dimshuffle(1, 0, 2, 3))
topgrad = gpu_contiguous(topgrad.dimshuffle(1, 0, 2, 3))
ishape = [shape_i_op(i)(img) for i in range(img.ndim)]
tshape = [shape_i_op(i)(topgrad) for i in range(topgrad.ndim)]
out_shp = get_conv_output_shape(
ishape,
tshape,
border_mode=border_mode,
subsample=subsample,
filter_dilation=filter_dilation,
)
out_shp = assert_conv_shape(out_shp)
out = GpuAllocEmpty(dtype=img.dtype,
context_name=ctx_name)(*out_shp)
desc = GpuDnnConvDesc(
border_mode=border_mode,
subsample=subsample,
dilation=filter_dilation,
conv_mode="cross",
precision=precision,
)(out.shape)
conv = GpuDnnConv(algo=None, num_groups=num_groups)(img, topgrad,
out, desc)
if conv_mode == "conv":
conv = conv[:, :, ::-1, ::-1]
rval = as_gpuarray_variable(conv.dimshuffle(1, 0, 2, 3), ctx_name)
else:
return None
elif isinstance(op, AbstractConv2d_gradInputs):
if border_mode == "valid" and subsample == (1, 1) and num_groups == 1:
kerns = gpu_contiguous(inp1.dimshuffle(1, 0, 2, 3))
topgrad = gpu_contiguous(inp2)
ctx_name = infer_context_name(kerns, topgrad)
conv_mode = "cross" if conv_mode == "conv" else "conv"
desc = GpuDnnConvDesc(
border_mode="full",
subsample=subsample,
dilation=filter_dilation,
conv_mode=conv_mode,
precision=precision,
)(kerns.shape)
tshape = [shape_i_op(i)(topgrad) for i in range(topgrad.ndim)]
kshape = [shape_i_op(i)(kerns) for i in range(kerns.ndim)]
shape = get_conv_output_shape(
tshape,
kshape,
border_mode="full",
subsample=subsample,
filter_dilation=filter_dilation,
)
shape = assert_conv_shape(shape)
out = GpuAllocEmpty(dtype=topgrad.dtype,
context_name=ctx_name)(*shape)
rval = GpuDnnConv(algo=None, num_groups=num_groups)(topgrad, kerns,
out, desc)
else:
return None
return [rval]
def op(self, state):
X = self.l_in.op(state=state)
return dnn_conv(X, self.w, subsample=self.stride, border_mode=self.pad)
def __init__(self,
input,
image_shape,
filter_shape,
convstride,
padsize,
group,
poolsize,
poolstride,
bias_init,
lrn=False,
lib_conv='cudnn',
verbose=False):
'''
lib_conv can be cudnn (recommended)or cudaconvnet
'''
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
self.verbose = verbose
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudnn':
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
# print image_shape_shuffled
# print filter_shape_shuffled
if group == 1:
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(
img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out0 = \
dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2,
:, :],
kerns=W0_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out1 = \
dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:,
:, :],
kerns=W1_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
# elif lib_conv == 'cudaconvnet':
# from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
#
# self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride,
# partial_sum=1)
#
# from theano.sandbox.cuda.basic_ops import gpu_contiguous
#
# # Conv
# if group == 1:
# contiguous_input = gpu_contiguous(input)
# contiguous_filters = gpu_contiguous(self.W.val)
# conv_out = self.conv_op(contiguous_input, contiguous_filters)
# conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
# else:
# contiguous_input0 = gpu_contiguous(
# input[:self.channel / 2, :, :, :])
# contiguous_filters0 = gpu_contiguous(self.W0.val)
# conv_out0 = self.conv_op(
# contiguous_input0, contiguous_filters0)
# conv_out0 = conv_out0 + \
# self.b0.val.dimshuffle(0, 'x', 'x', 'x')
#
# contiguous_input1 = gpu_contiguous(
# input[self.channel / 2:, :, :, :])
# contiguous_filters1 = gpu_contiguous(self.W1.val)
# conv_out1 = self.conv_op(
# contiguous_input1, contiguous_filters1)
# conv_out1 = conv_out1 + \
# self.b1.val.dimshuffle(0, 'x', 'x', 'x')
# conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
#
# # ReLu
# conv_out = gpu_contiguous(conv_out)
# self.output = T.maximum(conv_out, 0)
#
# # Pooling
# if self.poolsize != 1:
# from pylearn2.sandbox.cuda_convnet.pool import MaxPool
# self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
# self.output = self.pool_op(self.output)
#
# elif lib_conv == 'corrmm':
#
# from theano.sandbox.cuda.basic_ops import gpu_contiguous
# from theano.sandbox.cuda.blas import GpuCorrMM
#
# border_mode = 'half' if padsize == (filter_shape[1]-1)/2 else (padsize, padsize)
# self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride),
# border_mode=border_mode)
# flip_filters=True
# input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
#
#
# if group==1:
#
# filters = self.W.val.dimshuffle(3, 0, 1, 2)
#
# if flip_filters:
# filters = filters[:, :, ::-1, ::-1] # flip top-down, left-right
# contiguous_filters = gpu_contiguous(filters)
# contiguous_input = gpu_contiguous(input_shuffled)
#
# conv_out = self.corr_mm_op(contiguous_input, contiguous_filters)
# conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
#
# else:
#
# W0_shuffled = \
# self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
# if flip_filters:
# W0_shuffled = W0_shuffled[:, :, ::-1, ::-1]
#
# contiguous_filters0 = gpu_contiguous(W0_shuffled)
# contiguous_input0 = gpu_contiguous(input_shuffled[:, :self.channel / 2,:, :])
#
# conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0)
# conv_out0 = conv_out0 + \
# self.b0.val.dimshuffle('x', 0, 'x', 'x')
#
# W1_shuffled = \
# self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
# if flip_filters:
# W1_shuffled = W1_shuffled[:, :, ::-1, ::-1]
#
# contiguous_filters1 = gpu_contiguous(W1_shuffled)
# contiguous_input1 = gpu_contiguous(input_shuffled[:, self.channel / 2:,:, :])
#
# conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1)
# conv_out1 = conv_out1 + \
# self.b1.val.dimshuffle('x', 0, 'x', 'x')
# conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
#
# # ReLu
# self.output = T.maximum(conv_out, 0)
#
# # Pooling
# if self.poolsize != 1:
# from theano.tensor.signal import downsample
# self.output = downsample.max_pool_2d(self.output,
# ds=(poolsize,poolsize),
# st=(poolstride,poolstride),
# ignore_border=False,
# padding=(0,0),
# mode='max',
# )
#
# self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
else:
NotImplementedError("lib_conv can only be cudnn for now")
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
if self.verbose:
print "conv ({}) layer with shape_in: {}".format(
lib_conv, str(image_shape))
def __init__(self, input, image_shape, filter_shape, convstride, padsize,
group, poolsize, poolstride, bias_init, lrn=False,
lib_conv='cudnn',
verbose=False
):
'''
lib_conv can be cudnn (recommended)or cudaconvnet
'''
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
self.verbose = verbose
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudnn':
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
# print image_shape_shuffled
# print filter_shape_shuffled
if group == 1:
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out0 = \
dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2,
:, :],
kerns=W0_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out1 = \
dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:,
:, :],
kerns=W1_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
# elif lib_conv == 'cudaconvnet':
# from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
#
# self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride,
# partial_sum=1)
#
# from theano.sandbox.cuda.basic_ops import gpu_contiguous
#
# # Conv
# if group == 1:
# contiguous_input = gpu_contiguous(input)
# contiguous_filters = gpu_contiguous(self.W.val)
# conv_out = self.conv_op(contiguous_input, contiguous_filters)
# conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
# else:
# contiguous_input0 = gpu_contiguous(
# input[:self.channel / 2, :, :, :])
# contiguous_filters0 = gpu_contiguous(self.W0.val)
# conv_out0 = self.conv_op(
# contiguous_input0, contiguous_filters0)
# conv_out0 = conv_out0 + \
# self.b0.val.dimshuffle(0, 'x', 'x', 'x')
#
# contiguous_input1 = gpu_contiguous(
# input[self.channel / 2:, :, :, :])
# contiguous_filters1 = gpu_contiguous(self.W1.val)
# conv_out1 = self.conv_op(
# contiguous_input1, contiguous_filters1)
# conv_out1 = conv_out1 + \
# self.b1.val.dimshuffle(0, 'x', 'x', 'x')
# conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
#
# # ReLu
# conv_out = gpu_contiguous(conv_out)
# self.output = T.maximum(conv_out, 0)
#
# # Pooling
# if self.poolsize != 1:
# from pylearn2.sandbox.cuda_convnet.pool import MaxPool
# self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
# self.output = self.pool_op(self.output)
#
# elif lib_conv == 'corrmm':
#
# from theano.sandbox.cuda.basic_ops import gpu_contiguous
# from theano.sandbox.cuda.blas import GpuCorrMM
#
# border_mode = 'half' if padsize == (filter_shape[1]-1)/2 else (padsize, padsize)
# self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride),
# border_mode=border_mode)
# flip_filters=True
# input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
#
#
# if group==1:
#
# filters = self.W.val.dimshuffle(3, 0, 1, 2)
#
# if flip_filters:
# filters = filters[:, :, ::-1, ::-1] # flip top-down, left-right
# contiguous_filters = gpu_contiguous(filters)
# contiguous_input = gpu_contiguous(input_shuffled)
#
# conv_out = self.corr_mm_op(contiguous_input, contiguous_filters)
# conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
#
# else:
#
# W0_shuffled = \
# self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
# if flip_filters:
# W0_shuffled = W0_shuffled[:, :, ::-1, ::-1]
#
# contiguous_filters0 = gpu_contiguous(W0_shuffled)
# contiguous_input0 = gpu_contiguous(input_shuffled[:, :self.channel / 2,:, :])
#
# conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0)
# conv_out0 = conv_out0 + \
# self.b0.val.dimshuffle('x', 0, 'x', 'x')
#
# W1_shuffled = \
# self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
# if flip_filters:
# W1_shuffled = W1_shuffled[:, :, ::-1, ::-1]
#
# contiguous_filters1 = gpu_contiguous(W1_shuffled)
# contiguous_input1 = gpu_contiguous(input_shuffled[:, self.channel / 2:,:, :])
#
# conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1)
# conv_out1 = conv_out1 + \
# self.b1.val.dimshuffle('x', 0, 'x', 'x')
# conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
#
# # ReLu
# self.output = T.maximum(conv_out, 0)
#
# # Pooling
# if self.poolsize != 1:
# from theano.tensor.signal import downsample
# self.output = downsample.max_pool_2d(self.output,
# ds=(poolsize,poolsize),
# st=(poolstride,poolstride),
# ignore_border=False,
# padding=(0,0),
# mode='max',
# )
#
# self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
else:
NotImplementedError("lib_conv can only be cudnn for now")
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
if self.verbose:
print "conv ({}) layer with shape_in: {}".format(lib_conv,
str(image_shape))
