def relevance_conv_z_plus(out_relevances, inputs, weights, bias=None):
if bias is not None:
log.warning("Bias not respected for conv z_plus")
# hack for negative inputs
# inputs = T.abs_(inputs)
weights_plus = weights * T.gt(weights, 0)
norms_for_relevances = conv2d(inputs, weights_plus)
# prevent division by 0...
# adds 1 to every entry that is 0 -> sets 0s to 1
relevances_are_0 = T.eq(norms_for_relevances, 0)
norms_for_relevances += relevances_are_0 * 1
normed_relevances = out_relevances / norms_for_relevances
# upconv
in_relevances = conv2d(normed_relevances, weights_plus.dimshuffle(1, 0, 2, 3)[:, :, ::-1, ::-1], border_mode="full")
in_relevances_proper = in_relevances * inputs
# Correct for those parts where all inputs of a relevance were
# zero, spread relevance equally them
pool_ones = T.ones(weights_plus.shape, dtype=np.float32)
# mean across channel, 0, 1 dims (hope this is correct?)
pool_fractions = pool_ones / T.prod(weights_plus.shape[1:]).astype(theano.config.floatX)
in_relevances_from_0 = conv2d(
out_relevances * relevances_are_0, pool_fractions.dimshuffle(1, 0, 2, 3), subsample=(1, 1), border_mode="full"
)
in_relevances_proper += in_relevances_from_0
return in_relevances_proper
def lecun_lcn(self, X, kernel_size=7, threshold = 1e-4, use_divisor=False):
"""
Yann LeCun's local contrast normalization
Orginal code in Theano by: Guillaume Desjardins
"""
filter_shape = (1, 1, kernel_size, kernel_size)
filters = gaussian_filter(kernel_size).reshape(filter_shape)
filters = shared(_asarray(filters, dtype=floatX), borrow=True)
convout = conv2d(X, filters=filters, filter_shape=filter_shape,
border_mode='full')
# For each pixel, remove mean of kernel_sizexkernel_size neighborhood
mid = int(floor(kernel_size/2.))
new_X = X - convout[:,:,mid:-mid,mid:-mid]
if use_divisor:
# Scale down norm of kernel_sizexkernel_size patch
sum_sqr_XX = conv2d(T.sqr(T.abs_(X)), filters=filters,
filter_shape=filter_shape, border_mode='full')
denom = T.sqrt(sum_sqr_XX[:,:,mid:-mid,mid:-mid])
per_img_mean = denom.mean(axis=[2,3])
divisor = T.largest(per_img_mean.dimshuffle(0,1,'x','x'), denom)
divisor = T.maximum(divisor, threshold)
new_X /= divisor
return new_X#T.cast(new_X, floatX)
def relevance_conv_a_b_abs(inputs, weights, out_relevances, a, b, bias=None):
assert a is not None
assert b is not None
assert a - b == 1
weights_plus = weights * T.gt(weights, 0)
weights_neg = weights * T.lt(weights, 0)
plus_norm = conv2d(T.abs_(inputs), weights_plus)
# stabilize, prevent division by 0
eps = 1e-4
plus_norm += T.eq(plus_norm, 0) * eps
plus_rel_normed = out_relevances / plus_norm
in_rel_plus = conv2d(plus_rel_normed, weights_plus.dimshuffle(1, 0, 2, 3)[:, :, ::-1, ::-1], border_mode="full")
in_rel_plus *= T.abs_(inputs)
# minuses to get positive outputs, since will be subtracted
# at end of function
neg_norm = -conv2d(T.abs_(inputs), weights_neg)
neg_norm += T.eq(neg_norm, 0) * eps
neg_rel_normed = out_relevances / neg_norm
in_rel_neg = -conv2d(neg_rel_normed, weights_neg.dimshuffle(1, 0, 2, 3)[:, :, ::-1, ::-1], border_mode="full")
in_rel_neg *= T.abs_(inputs)
in_relevance = a * in_rel_plus - b * in_rel_neg
return in_relevance
def convolutional_model(X, w_1, w_2, w_3, w_4, w_5, w_6, p_1, p_2, p_3, p_4, p_5):
l1 = dropout(T.tanh( max_pool_2d(T.maximum(conv2d(X, w_1, border_mode='full'),0.), (2, 2),ignore_border=True) + b_1.dimshuffle('x', 0, 'x', 'x') ), p_1)
l2 = dropout(T.tanh( max_pool_2d(T.maximum(conv2d(l1, w_2), 0.), (2, 2),ignore_border=True) + b_2.dimshuffle('x', 0, 'x', 'x') ), p_2)
l3 = dropout(T.flatten(T.tanh( max_pool_2d(T.maximum(conv2d(l2, w_3), 0.), (2, 2),ignore_border=True) + b_3.dimshuffle('x', 0, 'x', 'x') ), outdim=2), p_3)# flatten to switch back to 1d layers
l4 = dropout(T.maximum(T.dot(l3, w_4), 0.), p_4)
l5 = dropout(T.maximum(T.dot(l4, w_5), 0.), p_5)
return T.dot(l5, w_6)
def _backward_negative_z(inputs, weights, normed_relevances, bias=None):
inputs_plus = inputs * T.gt(inputs, 0)
weights_plus = weights * T.gt(weights, 0)
inputs_minus = inputs * T.lt(inputs, 0)
weights_minus = weights * T.lt(weights, 0)
# Compute weights+ * inputs- and weights- * inputs+
negative_part_a = conv2d(
normed_relevances, weights_plus.dimshuffle(1, 0, 2, 3)[:, :, ::-1, ::-1], border_mode="full"
)
negative_part_a *= inputs_minus
negative_part_b = conv2d(
normed_relevances, weights_minus.dimshuffle(1, 0, 2, 3)[:, :, ::-1, ::-1], border_mode="full"
)
negative_part_b *= inputs_plus
together = negative_part_a + negative_part_b
if bias is not None:
bias_negative = bias * T.lt(bias, 0)
bias_relevance = bias_negative.dimshuffle("x", 0, "x", "x") * normed_relevances
# Divide bias by weight size before convolving back
# mean across channel, 0, 1 dims (hope this is correct?)
fraction_bias = bias_relevance / T.prod(weights.shape[1:]).astype(theano.config.floatX)
bias_rel_in = conv2d(
fraction_bias, T.ones_like(weights).dimshuffle(1, 0, 2, 3)[:, :, ::-1, ::-1], border_mode="full"
)
together += bias_rel_in
return together
def theano_kernel_derivative(imshp,kshp,featshp,stride=1):
features = T.tensor4(dtype=theano.config.floatX)
kernel = T.tensor4(dtype=theano.config.floatX)
image = T.tensor4(dtype=theano.config.floatX)
# Need to transpose first two dimensions of kernel, and reverse index kernel image dims (for correlation)
kernel_rotated = T.transpose(kernel[:,:,::-1,::-1],axes=[1,0,2,3])
featshp_logical = (featshp[0],featshp[1],featshp[2]*stride,featshp[3]*stride)
kshp_rotated = (kshp[1], kshp[0], kshp[2], kshp[3])
image_estimate = conv2d(features,kernel_rotated,border_mode='full',
image_shape=featshp,filter_shape=kshp_rotated,
imshp_logical=featshp_logical[1:],kshp_logical=kshp[2:])
image_error = image - image_estimate
image_error_rot = T.transpose(image_error,[1,0,2,3])[:,:,::-1,::-1]
imshp_rot = (imshp[1],imshp[0],imshp[2],imshp[3])
featshp_rot = (featshp[1],featshp[0],featshp[2],featshp[3])
features_rot = T.transpose(features,[1,0,2,3])
featshp_rot_logical = (featshp_rot[0],featshp_rot[1],featshp_rot[2]*stride,featshp_rot[3]*stride)
kernel_grad_rot = -conv2d(image_error_rot,features_rot,
image_shape=imshp_rot,filter_shape=featshp_rot,
imshp_logical=imshp_rot[1:],kshp_logical=featshp_rot_logical[2:])
kernel_grad = T.transpose(kernel_grad_rot,[1,0,2,3])
return function(inputs=[image,features,kernel],outputs=kernel_grad)
def model(X, params, featMaps, pieces, pDropConv, pDropHidden):
lnum = 0 # conv: (32, 32) pool: (16, 16)
layer = conv2d(X, params[lnum][0], border_mode='half') + \
params[lnum][1].dimshuffle('x', 0, 'x', 'x')
layer = maxout(layer, featMaps[lnum], pieces[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1 # conv: (16, 16) pool: (8, 8)
layer = conv2d(layer, params[lnum][0], border_mode='half') + \
params[lnum][1].dimshuffle('x', 0, 'x', 'x')
layer = maxout(layer, featMaps[lnum], pieces[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1 # conv: (8, 8) pool: (4, 4)
layer = conv2d(layer, params[lnum][0], border_mode='half') + \
params[lnum][1].dimshuffle('x', 0, 'x', 'x')
layer = maxout(layer, featMaps[lnum], pieces[lnum])
layer = pool_2d(layer, (2, 2), st=(2, 2), ignore_border=False, mode='max')
layer = basicUtils.dropout(layer, pDropConv)
lnum += 1
layer = T.flatten(layer, outdim=2)
layer = T.dot(layer, params[lnum][0]) + params[lnum][1].dimshuffle('x', 0)
layer = relu(layer, alpha=0)
layer = basicUtils.dropout(layer, pDropHidden)
lnum += 1
layer = T.dot(layer, params[lnum][0]) + params[lnum][1].dimshuffle('x', 0)
layer = relu(layer, alpha=0)
layer = basicUtils.dropout(layer, pDropHidden)
lnum += 1
return softmax(T.dot(layer, params[lnum][0]) + params[lnum][1].dimshuffle('x', 0)) # 如果使用nnet中的softmax训练产生NAN
def relevance_conv_z(out_relevances, inputs, weights, bias=None):
norms_for_relevances = conv2d(inputs, weights)
if bias is not None:
norms_for_relevances += bias.dimshuffle("x", 0, "x", "x")
# stabilize
# prevent division by 0 and division by small numbers
eps = 1e-4
norms_for_relevances += T.sgn(norms_for_relevances) * eps
norms_for_relevances += T.eq(norms_for_relevances, 0) * eps
normed_relevances = out_relevances / norms_for_relevances
# upconv
in_relevances = conv2d(normed_relevances, weights.dimshuffle(1, 0, 2, 3)[:, :, ::-1, ::-1], border_mode="full")
in_relevances_proper = in_relevances * inputs
if bias is not None:
bias_relevance = bias.dimshuffle("x", 0, "x", "x") * normed_relevances
# Divide bias by weight size before convolving back
# mean across channel, 0, 1 dims (hope this is correct?)
fraction_bias = bias_relevance / T.prod(weights.shape[1:]).astype(theano.config.floatX)
bias_rel_in = conv2d(
fraction_bias, T.ones_like(weights).dimshuffle(1, 0, 2, 3)[:, :, ::-1, ::-1], border_mode="full"
)
in_relevances_proper += bias_rel_in
return in_relevances_proper
def metaOp1(i, j, X, w1, w2, b1, b2):
# (n,1,r,c)**(16,1,3,3)=(n,16,r,c)
hiddens = conv2d(X[:, j, :, :, :], w1[i, j, :, :, :, :], border_mode='half') + b1[i, j, :, :, :, :]
hiddens = relu(hiddens, alpha=0)
# (n,16,r,c)**(1,16,1,1)=(n,1,r,c)
outputs = conv2d(hiddens, w2[i, j, :, :, :, :], border_mode='valid') + b2[i, j, :, :, :, :]
outputs = relu(outputs, alpha=0)
return outputs
def metaOp(i, j, X, w1, w2, b1, b2):
# (n,1,r,c)**(16,1,3,3)=(n,16,r,c)
hiddens = conv2d(X[:, j, :, :, :], w1[i, j, :, :, :, :], border_mode='half') + b1[i, j, :, :, :, :]
hiddens = relu(hiddens, alpha=0)
# 在元操作中就需要包含relu激活
# return conv2d(hiddens, w2[i, j, :, :, :, :], border_mode='valid') + b2[i, j, :, :, :, :]
# (n,16,r,c)**(1,16,1,1)=(n,1,r,c)
outputs = conv2d(hiddens, w2[i, j, :, :, :, :], border_mode='valid') + b2[i, j, :, :, :, :]
return T.nnet.relu(outputs)
def connect_through(self, alm1_in=None, Pl=None, Ll=None):
""" connect_through
@note Note that I made connect_through a separate class method, separate from the automatic initialization,
because you can then make changes to the "layer units" or "nodes" before "connecting the layers"
"""
if alm1_in is not None:
self.alm1 = alm1_in
self.Pl = Pl
alm1 = self.alm1
c = self.c
C_lm1,C_l = self.C_ls
Wl = self.Wl
filter_shape = (C_l,C_lm1)+Wl
assert len(filter_shape) == (2+len(Wl))
# convolve input feature maps with filters
if Ll is not None:
batch_size = alm1.shape[0] # This is m, number of input examples
# image_shape = (batch_size,C_lm1)+Ll
image_shape = (None,C_lm1)+Ll
conv_out = conv2d(
input=alm1,
filters=c,
filter_shape=filter_shape,
input_shape=image_shape)
else:
conv_out = conv2d(
input=alm1,
filters=c,
filter_shape=filter_shape)
# pool each feature map individually, using maxpooling
if Pl is not None:
pooled_out = pool.pool_2d(
input=conv_out,
ws=Pl,
ignore_border=True)
# add bias term
if self.psi is None:
self.al = pooled_out + self.b.dimshuffle('x',0,'x','x')
else:
self.al = self.psi( pooled_out + self.b.dimshuffle('x',0,'x','x') )
else:
# add bias term
if self.psi is None:
self.al = conv_out + self.b.dimshuffle('x',0,'x','x')
else:
self.al = self.psi( conv_out + self.b.dimshuffle('x',0,'x','x') )
return self.al
def CNN(x,c_l1,c_l2,f_l1,f_l2):
conv1=tensor.nnet.relu(conv2d(x,c_l1)) #default stride=1 --subsample=(1,1)
pool1=pool_2d(conv1,(2,2),st=(2,2),ignore_border=True) #default maxpool
conv2=tensor.nnet.relu(conv2d(pool1,c_l2))
pool2=pool_2d(conv2,(2,2),st=(2,2),ignore_border=True)
fpool2=tensor.flatten(pool2,outdim=2)
full1=tensor.nnet.relu(tensor.dot(fpool2,f_l1))
pyx=tensor.nnet.sigmoid(tensor.dot(full1,f_l2))
return c_l1, c_l2, f_l1, f_l2, pyx
def predict_custom_image(params, testImgFilename='own_0.png', activation= activation_convmlp, testImgFilenameDir = '../data/custom/'):
test_img_value = filter(str.isdigit, testImgFilename)
test_img = fli.processImg(testImgFilenameDir, testImgFilename)
nkerns = [20, 50]
batch_size = 1
poolsize = (2, 2)
layer0_input = test_img.reshape((batch_size, 1, 28, 28)).astype(numpy.float32)
conv_out_0 = conv2d(
input=layer0_input,
filters=params[6],
input_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5)
)
# downsample each feature map individually, using maxpooling
pooled_out_0 = downsample.max_pool_2d(
input=conv_out_0,
ds=poolsize,
ignore_border=True
)
output_0 = activation(pooled_out_0 + params[7].dimshuffle('x', 0, 'x', 'x'))
conv_out_1 = conv2d(
input=output_0,
filters=params[4],
input_shape=(batch_size, nkerns[0], 12, 12),
filter_shape=(nkerns[1], nkerns[0], 5, 5),
)
# downsample each feature map individually, using maxpooling
pooled_out_1 = downsample.max_pool_2d(
input=conv_out_1,
ds=poolsize,
ignore_border=True
)
output_1 = activation(pooled_out_1 + params[5].dimshuffle('x', 0, 'x', 'x'))
output_2 = activation(T.dot(output_1.flatten(2), params[2]) + params[3])
final_output = T.dot(output_2, params[0]) + params[1]
p_y_given_x = T.nnet.softmax(final_output)
y_pred = T.argmax(p_y_given_x, axis=1)
testfunc = theano.function([], [y_pred[0]])
prediction = testfunc()[0]
correct = (int(test_img_value) == prediction)
print('The prediction ' + str(testfunc()[0]) + ' for ' + testImgFilename + ' is ' + str(correct) + '.')
return correct
def _forward_negative_z(inputs, weights, bias=None):
inputs_plus = inputs * T.gt(inputs, 0)
weights_plus = weights * T.gt(weights, 0)
inputs_minus = inputs * T.lt(inputs, 0)
weights_minus = weights * T.lt(weights, 0)
negative_part_a = conv2d(inputs_plus, weights_minus)
negative_part_b = conv2d(inputs_minus, weights_plus)
together = negative_part_a + negative_part_b
if bias is not None:
bias_negative = bias * T.lt(bias, 0)
together += bias_negative.dimshuffle("x", 0, "x", "x")
return together
def conv2d_test_sets():
def _convert(input, kernel, output, kwargs):
return [theano.shared(floatX(input)), floatX(kernel), output, kwargs]
input = np.random.random((3, 1, 16, 16))
kernel = np.random.random((16, 1, 3, 3))
output = conv2d(input, kernel).eval()
yield _convert(input, kernel, output, {})
input = np.random.random((3, 3, 16, 16))
kernel = np.random.random((16, 3, 3, 3))
output = conv2d(input, kernel).eval()
yield _convert(input, kernel, output, {})
def modelFlow(X, params):
lconv1 = relu(conv2d(X, params[0][0], border_mode='full') +
params[0][1].dimshuffle('x', 0, 'x', 'x'))
lds1 = pool_2d(lconv1, (2, 2))
lconv2 = relu(conv2d(lds1, params[1][0]) +
params[1][1].dimshuffle('x', 0, 'x', 'x'))
lds2 = pool_2d(lconv2, (2, 2))
lconv3 = relu(conv2d(lds2, params[2][0]) +
params[2][1].dimshuffle('x', 0, 'x', 'x'))
lds3 = pool_2d(lconv3, (2, 2))
return X, lconv1, lds1, lconv2, lds2, lconv3, lds3
def __init__(self, input, filter_shape, image_shape, f_params_w, f_params_b, lrn=False, t_style=None, t_content=None, convstride=1, padsize =0, group=1, poolsize = 3, poolstride = 1):
self.input = input
#theano.shared(np.asarray(np.input))
if t_style is not None:
self.t_style = np.asarray(np.load(t_style),dtype=theano.config.floatX)
if t_content is not None:
self.t_content = np.asarray(np.load(t_content),dtype=theano.config.floatX)
if lrn is True:
self.lrn_func = CrossChannelNormalization()
#if padsize==(0,0):
#padsize='valid'
if group == 1:
self.W = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w)),(3,0,1,2)),dtype=theano.config.floatX), borrow=True)
self.b = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b)),dtype=theano.config.floatX), borrow=True)
conv_out = conv2d(input=self.input,filters=self.W,filter_shape=filter_shape,border_mode = padsize,subsample=(convstride, convstride),filter_flip=True)
#self.params = [self.W, self.b]
elif group == 2:
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[1] = self.filter_shape[1] / 2
#self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[1] = self.image_shape[1] / 2
self.W0 = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w[0])),(3,0,1,2)),dtype=theano.config.floatX), borrow=True)
self.W1 = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w[1])),(3,0,1,2)),dtype=theano.config.floatX), borrow=True)
self.b0 = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b[0])),dtype=theano.config.floatX), borrow=True)
self.b1 = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b[1])),dtype=theano.config.floatX), borrow=True)
conv_out0 = conv2d(input=self.input[:,:self.image_shape[1],:,:],filters=self.W0,filter_shape=tuple(self.filter_shape),border_mode = padsize,subsample=(convstride, convstride),filter_flip=True) + self.b0.dimshuffle('x', 0, 'x', 'x')
conv_out1 = conv2d(input=self.input[:,self.image_shape[1]:,:,:],filters=self.W1,filter_shape=tuple(self.filter_shape),border_mode = padsize,subsample=(convstride, convstride),filter_flip=True) + self.b1.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1],axis=1)
#self.params = [self.W0, self.b0, self.W1, self.b1]
else:
raise AssertionError()
relu_out = T.maximum(conv_out, 0)
if poolsize != 1:
self.output = pool.pool_2d(input=relu_out,ds=(poolsize,poolsize),ignore_border=True, st=(poolstride,poolstride),mode='average_exc_pad')
#self.output = downsample.max_pool_2d(input=relu_out,ds=(poolsize,poolsize),ignore_border=True, st=(poolstride,poolstride))
else:
self.output = relu_out
if lrn is True:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
def nin2(X, param, shape):
w1, w2 = param
map0 = []
for i in xrange(shape[0]):
map1 = []
for j in xrange(shape[1]):
Xj = X[:, j, :, :].dimshuffle(0, 'x', 1, 2)
w1ij = w1[i, j, :, :, :].dimshuffle(0, 'x', 1, 2)
w2ij = w2[i, j, :].dimshuffle('x', 0, 'x', 'x')
tmp = conv2d(Xj, w1ij, border_mode='valid')
tmp = relu(tmp, alpha=0)
map1.append(conv2d(tmp, w2ij, border_mode='valid'))
map0.append(relu(T.sum(map1, axis=0), alpha=0))
return T.concatenate(map0, axis=1)
def lecun_lcn(self, X, kernel_size=9, threshold = 1e-4, use_divisor=True, border=False):
"""
Yann LeCun's local contrast normalization
Orginal code in Theano by: Guillaume Desjardins
"""
filter_shape = (1, 1, kernel_size, kernel_size)
filters = gaussian_filter(kernel_size).reshape(filter_shape)
filters = shared(_asarray(filters, dtype=floatX), borrow=True)
mid = int(floor(kernel_size/2.))
if border:
r = (kernel_size-1)/2
up = X[:,:,0:1,:].repeat(r,axis=2)
down = X[:,:,-1:,:].repeat(r,axis=2)
X_ = T.concatenate([up,X,down],axis=2)
left = X_[:,:,:,0:1].repeat(r,axis=3)
right = X_[:,:,:,-1:].repeat(r,axis=3)
X_ = T.concatenate([left,X_,right],axis=3)
convout = conv2d(X_, filters=filters, filter_shape=filter_shape,
border_mode='valid')
centered_X = X - convout
else:
convout = conv2d(X, filters=filters, filter_shape=filter_shape,
border_mode='full')
# For each pixel, remove mean of kernel_sizexkernel_size neighborhood
centered_X = X - convout[:,:,mid:-mid,mid:-mid]
if use_divisor:
# Scale down norm of kernel_sizexkernel_size patch
sum_sqr_XX = conv2d(T.sqr(X), filters=filters,
filter_shape=filter_shape, border_mode='full')
sum_sqr_XX = sum_sqr_XX[:,:,mid:-mid,mid:-mid]
sum_sqr_XX = T.maximum(sum_sqr_XX, threshold)
denom = T.sqrt(sum_sqr_XX)
# denom = abs(centered_X)
per_img_mean = denom.mean(axis=[2,3])
divisor = T.largest(per_img_mean.dimshuffle(0,1,'x','x'), denom)
divisor = T.maximum(divisor, threshold)
new_X = centered_X / divisor
return new_X
else:
return centered_X
def conv(X, w, b, activation):
# z = dnn_conv(X, w, border_mode=int(np.floor(w.get_value().shape[-1]/2.)))
s = int(np.floor(w.get_value().shape[-1]/2.))
z = conv2d(X, w, border_mode='full')[:, :, s:-s, s:-s]
if b is not None:
z += b.dimshuffle('x', 0, 'x', 'x')
return activation(z)
def T_l2_cost_conv_dA(x,a,A,imshp,kshp,featshp,stride=(1,1),mask=True):
image_error, kernel, features = helper_T_l2_cost_conv(x=x,a=a,A=A,imshp=imshp,kshp=kshp,featshp=featshp,stride=stride,mask=mask)
if stride == (1,1):
image_error_rot = T.transpose(image_error,[1,0,2,3])[:,:,::-1,::-1]
imshp_rot = (imshp[1],imshp[0],imshp[2],imshp[3])
featshp_rot = (featshp[1],featshp[0],featshp[2],featshp[3])
features_rot = T.transpose(features,[1,0,2,3])
featshp_rot_logical = (featshp_rot[0],
featshp_rot[1],
imshp[2] - kshp[2] + 1,
imshp[3] - kshp[3] + 1)
kernel_grad_rot = -1.*conv2d(image_error_rot,features_rot,
image_shape=imshp_rot,filter_shape=featshp_rot,
imshp_logical=imshp_rot[1:],kshp_logical=featshp_rot_logical[2:])
kernel_grad = T.transpose(kernel_grad_rot,[1,0,2,3])
reshape_kernel_grad = T.transpose(T.reshape(kernel_grad,(kshp[0],kshp[1]*kshp[2]*kshp[3]),ndim=2))
return reshape_kernel_grad
else:
my_conv = MyConv_view(strides=stride,kshp=kshp)
kernel_grad = my_conv(image_error,features)
reshape_kernel_grad = T.transpose(T.reshape(kernel_grad, (kshp[0], kshp[1] * kshp[2] * kshp[3]), ndim=2))
return reshape_kernel_grad
def conv1d_mc0(input, filters, input_shape=None, filter_shape=None,
border_mode='valid', subsample=(1,)):
"""
Using conv2d with width == 1.
"""
image_shape = input_shape
if image_shape is None:
image_shape_mc0 = None
else:
# (b, c, i0) to (b, c, 1, i0)
image_shape_mc0 = (image_shape[0], image_shape[1], 1, image_shape[2])
if filter_shape is None:
filter_shape_mc0 = None
else:
filter_shape_mc0 = (filter_shape[0], filter_shape[1], 1,
filter_shape[2])
input_mc0 = input.dimshuffle(0, 1, 'x', 2)
filters_mc0 = filters.dimshuffle(0, 1, 'x', 2)
conved = conv2d(
input_mc0, filters_mc0, input_shape=image_shape_mc0,
filter_shape=filter_shape_mc0, subsample=(1, subsample[0]),
border_mode=border_mode)
return conved[:, :, 0, :] # drop the unused dimension
def test_conv_with_bias(self):
images = T.dtensor4('inputs')
weights = T.dtensor4('weights')
bias = T.dvector('bias')
ishape = [(8, 3, 256, 256), (16, 3, 256, 256), (32, 3, 256, 256), (64, 3, 256, 256)]
wshape = [(8, 3, 3, 3), (16, 3, 3, 3), (32, 3, 3, 3), (64, 3, 3, 3)]
for i, ish in enumerate(ishape):
wsh = wshape[i]
images_internal = U2IConv(imshp=ish, kshp=wsh)(images)
convOutBias_internal = Conv2D(imshp=ish, kshp=wsh, filter_flip=False)(images_internal, weights, bias)
convOutBias_user = I2U()(convOutBias_internal)
ival = numpy.random.rand(*ish).astype(numpy.float64)
wval = numpy.random.rand(*wsh).astype(numpy.float64)
bval = numpy.random.rand(wsh[0]).astype(numpy.float64)
fopt = theano.function(inputs=[images, weights, bias], outputs=convOutBias_user, mode=mode_with_mkl)
new_old = fopt(ival, wval, bval)
convOut = conv2d(images, weights, input_shape=ish, filter_shape=wsh, filter_flip=False)
convOutBias = convOut + bias.dimshuffle('x', 0, 'x', 'x')
fori = theano.function(inputs=[images, weights, bias], outputs=convOutBias, mode=mode_without_mkl)
old_out = fori(ival, wval, bval)
assert str(fopt.maker.fgraph.toposort()) != str(fori.maker.fgraph.toposort())
assert numpy.allclose(old_out, new_old)
def conv1d_sc(input, filters, input_shape=None, filter_shape=None,
border_mode='valid', subsample=(1,)):
"""
Using conv2d with a single input channel.
border_mode has to be 'valid' at the moment.
"""
if border_mode != 'valid':
log.error("Unsupported border_mode for conv1d_sc: "
"%s" % border_mode)
raise RuntimeError("Unsupported border_mode for conv1d_sc: "
"%s" % border_mode)
image_shape = input_shape
if image_shape is None:
image_shape_sc = None
else:
# (b, c, i0) to (b, 1, c, i0)
image_shape_sc = (image_shape[0], 1, image_shape[1], image_shape[2])
if filter_shape is None:
filter_shape_sc = None
else:
filter_shape_sc = (filter_shape[0], 1, filter_shape[1],
filter_shape[2])
input_sc = input.dimshuffle(0, 'x', 1, 2)
# We need to flip the channels dimension because it will be convolved over.
filters_sc = filters.dimshuffle(0, 'x', 1, 2)[:, :, ::-1, :]
conved = conv2d(input_sc, filters_sc, input_shape=image_shape_sc,
filter_shape=filter_shape_sc,
subsample=(1, subsample[0]))
return conved[:, :, 0, :] # drop the unused dimension
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2), activation=T.tanh):
assert image_shape[1] == filter_shape[1]
self.input = input
fan_in = np.prod(filter_shape[1:])
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) //
np.prod(poolsize))
W_bound = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(
np.asarray(
rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX
),
borrow=True
)
b_values = np.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
self.conv_out = conv2d(
input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape
)
self.output = activation(self.conv_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.params = [self.W, self.b]
self.outshape = (image_shape[0], filter_shape[0], image_shape[2]-filter_shape[2]+1, image_shape[3] - filter_shape[3] + 1)
def __init__(self, rng, input, layer_shape, input_shape, pool_size = (2,2)):
'''
:param rng: random number generator
:param input: 4D tensor with shape of input_shape
:param layer_shape: 4D matrix, out_put_num * input_num * kernel_rows * kernel_cols
:param input_shape: 4D matrix, batch_size * input_num * image_rows * image_cols
:param pool_size: pool_size
:return: Nothing
'''
assert input_shape[1] == layer_shape[1]
self.input = input
fan_in = np.prod(layer_shape[1:])
fan_out = (layer_shape[0] * np.prod(layer_shape[2:])) // np.prod(pool_size)
W_bound = np.sqrt(6.0 / (fan_out + fan_in))
self.W = theano.shared(np.array(rng.uniform(low = - W_bound, high= W_bound, size = layer_shape), dtype = theano.config.floatX),
borrow=True)
self.b = theano.shared(np.zeros(shape = (layer_shape[0], ), dtype = theano.config.floatX), borrow = True)
convolution_out = conv2d(input, self.W, filter_shape = layer_shape, input_shape = input_shape) #what will happen if I delete the last two parameters
pool_out = downsample.pool_2d(convolution_out, pool_size, ignore_border = True)
self.output = T.tanh(pool_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.params = [self.W, self.b]
def conv1d_mc1(input, filters, input_shape=None, filter_shape=None,
border_mode='valid', subsample=(1,)):
"""
Using conv2d with height == 1.
"""
image_shape = input_shape
if image_shape is None:
image_shape_mc1 = None
else:
# (b, c, i0) to (b, c, i0, 1)
image_shape_mc1 = (image_shape[0], image_shape[1], image_shape[2], 1)
if filter_shape is None:
filter_shape_mc1 = None
else:
filter_shape_mc1 = (filter_shape[0], filter_shape[1],
filter_shape[2], 1)
input_mc1 = input.dimshuffle(0, 1, 2, 'x')
filters_mc1 = filters.dimshuffle(0, 1, 2, 'x')
conved = conv2d(
input_mc1, filters_mc1, input_shape=image_shape_mc1,
filter_shape=filter_shape_mc1, subsample=(subsample[0], 1),
border_mode=border_mode)
return conved[:, :, :, 0] # drop the unused dimension
def __init__(self,rng,input,filter_shape,image_shape,poolsize=(2,2)):
assert image_shape[1] == filter_shape[1]
self.input = input
fan_in = np.prod(filter_shape)
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) //
np.prod(poolsize))
w_bound = np.sqrt(6./(fan_out + fan_in))
self.W = shared(np.asarray(
rng.uniform(low=-w_bound,
high=w_bound,
size = filter_shape),
dtype = th.config.floatX),
borrow = True)
b_values = np.zeros((filter_shape[0],),dtype=th.config.floatX)
self.b = th.shared(value = b_values, borrow=True)
conv_out = conv2d(input= input,
filters = self.W,
filter_shape =filter_shape,
input_shape = image_shape)
pooled_out = downsample.max_pool_2d(input = conv_out,
ds = poolsize,
ignore_border=True)
self.output = T.tanh(pooled_out + self.b.dimshuffle('x',0,'x','x'))
self.params = [self.W,self.b]
self.input = input
def test_conv_no_bias(self):
images = T.dtensor4('input_conv')
weights = T.dtensor4('weights')
images_internal = U2IConv(imshp=(12, 3, 256, 256), kshp=(12, 3, 3, 3))(images)
convOut = Conv2D(imshp=(12, 3, 256, 256), kshp=(12, 3, 3, 3), filter_flip=False)(images_internal, weights)
convOut_user = I2U()(convOut)
convOutLoss = T.mean(convOut_user)
conv_op_di = T.grad(convOutLoss, images)
conv_op_dk = T.grad(convOutLoss, weights)
convOutBack = [conv_op_di, conv_op_dk]
ival = numpy.random.rand(12, 3, 256, 256).astype(numpy.float64)
wval = numpy.random.rand(12, 3, 3, 3).astype(numpy.float64)
fopt = theano.function(inputs=[images, weights], outputs=convOutBack, mode=mode_with_mkl)
new_out = fopt(ival, wval)
convOut = conv2d(images, weights, input_shape=(12, 3, 256, 256), filter_shape=(12, 3, 3, 3), filter_flip=False)
convOutLoss = T.mean(convOut)
conv_op_di = T.grad(convOutLoss, images)
conv_op_dk = T.grad(convOutLoss, weights)
convOutBack = [conv_op_di, conv_op_dk]
fori = theano.function(inputs=[images, weights], outputs=convOutBack, mode=mode_without_mkl)
old_out = fori(ival, wval)
assert len(fopt.maker.fgraph.toposort()) != len(fori.maker.fgraph.toposort())
assert numpy.allclose(old_out[0], new_out[0])
assert new_out[0].dtype == 'float64'
def T_l2_cost_conv(x,a,A,imshp,kshp,mask=True):
"""
xsz*ysz*nchannels, nimages = x.shape
xsz*ysz*nfeat, nimages = a.shape
xsz*ysz*nchannels, nfeat = A.shape
"""
#imshp = num images, channels, szy, szx
#kshp = features, channels, szy, szx
#featshp = num images, features, szy, szx
featshp = (imshp[0],kshp[0],imshp[2] - kshp[2] + 1,imshp[3] - kshp[3] + 1) # num images, features, szy, szx
image = T.reshape(T.transpose(x),imshp)
kernel = T.reshape(T.transpose(A),kshp)
features = T.reshape(T.transpose(a),featshp)
# Need to transpose first two dimensions of kernel, and reverse index kernel image dims (for correlation)
kernel_rotated = T.transpose(kernel[:,:,::-1,::-1],axes=[1,0,2,3])
image_estimate = conv2d(features,kernel_rotated,border_mode='full')
if mask:
image_error_temp = image - image_estimate
image_error = T.zeros_like(image_error_temp)
image_error = T.set_subtensor(image_error[:,:,(kshp[2]-1):(imshp[2]-kshp[2]+1),(kshp[3]-1):(imshp[3]-kshp[3]+1)],
image_error_temp[:,:,(kshp[2]-1):(imshp[2]-kshp[2]+1),(kshp[3]-1):(imshp[3]-kshp[3]+1)])
else:
image_error = image - image_estimate
return .5*T.sum(image_error **2)
def convpool(X, W, b, poolsize=(2, 2)):
conv_out = conv2d(input=X, filters=W)
pooled_out = pool.pool_2d(input=conv_out, ws=poolsize, ignore_border=True)
return T.tanh(pooled_out + b.dimshuffle('x', 0, 'x', 'x'))
def conv(X, w, s = 2, b = None, activation = relu):
z = conv2d(X, w, border_mode='full')[:, :, s:-s, s:-s]
if b is not None:
z += b.dimshuffle('x', 0, 'x', 'x')
return activation(z)
def _train_fprop(self, state_below):
conv_out = conv2d(state_below,
self.W,
border_mode=self.border_mode,
subsample=self.stride)
return conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
def __init__(self,
l,
C_ls,
Wl,
Pl=None,
alm1=None,
c=None,
b=None,
activation=T.tanh,
rng=None):
""" Initialize the parameters for the `layer`
@type rng : numpy.random.RandomState
@param rng : random number generator used to initialize weights
@type l : (positive) integer
@param l : layer number label, l=0 (input),1,...L-1, L is "output layer"
@type Cl : tuple of (positive) integers of size (length) 2, only
@param Cls : "matrix" size dimensions of Theta or weight mmatrix, of size dims. (s_l, s_lp1) (this is important)
Bottom line: s_ls = (s_l, s_lp1)
@type alm1 : theano shared variable or vector of size dims. (m, s_l), m=1,2..., number of training examples
@oaram alm1 : "nodes" or "units" of layer l
@type c : theano shared; of size dims. (s_l,s_{l+1}), i.e. matrix dims. (s_{l}x(s_{l+1})), i.e. \Theta \in \text{Mat}_{\mathbb{R}}(s_l,s_{l+1})
@param c : "weights" or parameters for l, l=1,2, ... L-1
@type b : theano shared of dim. s_{l+1} or s_lp1; it's now a "row" array of that length s_lp1
@param b : intercepts
@type activation : theano.Op or function
@param activation : Non linearity to be applied in the layer(s)
"""
C_lm1, C_l = C_ls
if rng is None:
rng = np.random.RandomState(1234)
# num input feature maps * filter height * filter width
fan_in = C_lm1 * np.prod(Wl)
# num output feature maps * filter height * filter width / pooling size
if Pl is not None:
fan_out = C_l * np.prod(Wl) // np.prod(Pl)
else:
fan_out = C_l * np.prod(Wl)
# make the filter size out of C_ls and Wl
filter_size = (C_l, C_lm1) + Wl
assert len(filter_size) == (2 + len(Wl))
if c is None:
try:
c_values = np.asarray(rng.uniform(
low=-np.sqrt(6. / (fan_in + fan_out)),
high=np.sqrt(6. / (fan_in + fan_out)),
size=filter_size),
dtype=theano.config.floatX)
except MemoryError:
c_values = np.zeros(filter_size).astype(theano.config.floatX)
if activation == T.nnet.sigmoid:
c_values *= np.float32(4)
c = theano.shared(c_values, name="c" + str(l), borrow=True)
if b is None:
b_values = np.zeros((C_l, )).astype(theano.config.floatX)
b = theano.shared(value=b_values, name='b' + str(l), borrow=True)
if alm1 is None:
alm1 = T.tensor4(name='a' + str(l) + 'm1',
dtype=theano.config.floatX)
self.c = c # size dims. (C_l,C_lm1,W_1,...W_d) i.e. C_l x C_lm1 x W_1,x ... x W_d
self.b = b # dims. C_l
self.alm1 = alm1 # dims. (m,C_lm1,L_1,...L_d)
self.C_ls = C_ls
self.Wl = Wl
self.Pl = Pl
self.l = l
if activation is None:
self.psi = None
else:
self.psi = activation
# do a "basic" convolution, do a "basic" connect through
conv_out = conv2d(self.alm1, self.c)
if Pl is not None:
pooled_out = pool.pool_2d(conv_out, self.Pl, ignore_border=True)
if self.psi is None:
self.al = pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')
else:
self.al = self.psi(pooled_out +
self.b.dimshuffle('x', 0, 'x', 'x'))
else:
if self.psi is None:
self.al = conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
else:
self.al = self.psi(conv_out +
self.b.dimshuffle('x', 0, 'x', 'x'))
def __init__(self,
rng,
input,
filter_shape,
image_shape,
poolsize=(2, 2),
dropout_percent=0.5,
stride=None,
pool_method="max",
amp_value=None,
relu=False,
batch_norm=False,
precalculated_batchnorm_values=None,
batchnorm_slide_percent=0.):
"""
This is a convolutional layer that accepts 4D data and
convolutes the last 2 dimensions
Args:
filter_shape: int array (#conv layers, #channels, width,height)
dropout_percent: float that randomly disables inputs, prevents
overfitting
relu: Boolean that determines if output should be rectified
batch_norm: Boolean that determines if batch normalization
should be used
precalculated_batchnorm_values: float array that will replace
per-batch calculated
sliding_batchnorm_values: float, if nonzero, this will establish a
separate update to modify the means and standard deviation
after each batch, but not completely
"""
try:
assert image_shape[1] == filter_shape[1]
except AssertionError:
print 'Image shape is ' + str(image_shape[1])
print 'Filter shape is ' + str(filter_shape[1])
print "Poolsize is " + str(poolsize)
raise AssertionError
self.input = input
self.precalculated_batchnorm_values = precalculated_batchnorm_values
self.batchnorm_slide_percent = batchnorm_slide_percent
#this is the size of the number of inputs to each
#convolution hidden unit
fan_in = np.prod(filter_shape[1:])
#this is the number of output weights per channel divided by the
#pooling size
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]))/\
np.prod(poolsize)
W_bound = np.sqrt(6. / (fan_in + fan_out))
#initializes weights with appropriate values and shape
self.W = theano.shared(np.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
dropout_matrix = theano.shared(np.asarray(rng.binomial(
1, 1 - dropout_percent, filter_shape),
dtype=theano.config.floatX),
name='d',
borrow=True)
self.dropout_matrix = dropout_matrix
#initializes biases to zero
#(important to avoid presumptions about content of image)
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
#creates a shared variable instance (instead of a deep copy)
self.b = theano.shared(value=b_values, borrow=True)
self.dropout_weight = theano.shared(
np.asarray(1., dtype=theano.config.floatX))
#convolve; dropout matrix should work, but keep an eye on it
print 'conv image shape: ' + str(image_shape)
self.params = [self.W, self.b]
#track input (no longer redundant with batchnorm)
self.raw_input = input
if not batch_norm:
self.input = input
else:
print 'implementing batch normalization'
rn_input = range(image_shape[1])
#GAMMA is 0 because a constant of 1 is added to the transform
#to avoid problems with the L2 norm
self.GAMMA = theano.shared(
np.float32([
0. + rng.uniform(low=-0.001, high=0.001) for _ in rn_input
]))
self.BETA = theano.shared(np.float32([0 for _ in rn_input]))
self.params += [self.GAMMA, self.BETA]
if self.precalculated_batchnorm_values <> None:
self.sd_input = self.precalculated_batchnorm_values[0]
self.means = self.precalculated_batchnorm_values[1]
elif self.batchnorm_slide_percent == 0:
self.sd_input = T.sqrt(T.var(input, (0, 2, 3)) +
0.00001).dimshuffle('x', 0, 'x', 'x')
self.means = T.mean(input,
(0, 2, 3)).dimshuffle('x', 0, 'x', 'x')
else:
#set old values to initialized theano value
self.sd_input_old = theano.shared(
np.float32(np.ones((1, image_shape[1], 1, 1))),
broadcastable=(True, False, True, True))
self.means_old = theano.shared(
np.float32(np.zeros((1, image_shape[1], 1, 1))),
broadcastable=(True, False, True, True))
sbsp = self.batchnorm_slide_percent
self.sd_input = sbsp * self.sd_input_old + \
(1.-sbsp)*T.sqrt(T.var(input,(0,2,3))+0.00001).\
dimshuffle('x',0,'x','x')
self.means = sbsp * self.means_old + \
(1-sbsp) * T.mean(input,(0,2,3)).\
dimshuffle('x',0,'x','x')
self.input_normalized = (input - self.means) / self.sd_input
self.input = self.input_normalized * (np.float32(1.) + \
self.GAMMA.dimshuffle('x',0,'x','x')) + \
self.BETA.dimshuffle('x',0,'x','x')
if amp_value == None:
conv_out = conv2d(
input=self.input,
filters=self.W * dropout_matrix,
filter_shape=filter_shape,
input_shape=image_shape #keyword changed from "image_shape"
)
else:
conv_out = conv2d(
input=self.input,
filters=self.W * dropout_matrix * amp_value,
filter_shape=filter_shape,
input_shape=image_shape #keyword changed from "image_shape"
)
#pool (max pooling in this case)
pooled_out = downsample.max_pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True,
st=stride,
mode=pool_method)
self.stride = stride
self.pool_method = pool_method
self.relu = relu
x = T.tensor4('x', dtype=theano.config.floatX)
linmax = function([x], T.maximum(x, 0))
if not self.relu:
self.output = T.tanh(pooled_out +
self.b.dimshuffle('x', 0, 'x', 'x'))
else:
self.output = linmax(
T.Tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')))
#store parameters
#extra params to pass to other methods
self.rng = rng
self.filter_shape = filter_shape
self.dropout_percent = dropout_percent
print 'conv filter shape: %s' % str(self.filter_shape)
def __init__(self,
rng,
input,
image_shape,
filter_shape,
dropout,
pool=True,
poolsize=(2, 2),
border_mode='valid',
act='absTanh',
bch_norm=False,
param_seed=54621,
use_params=False,
conv_params=None):
"""
Allocate a ConvPoolLayer with shared variable internal parameters.
:type rng: np.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type pool: boolean
:param pool: indicates whether pooling should be used after convolution or not
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
if use_params:
# use previously defined parameters
self.W = conv_params[0]
self.b = conv_params[1]
else:
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = np.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) /
np.prod(poolsize))
# initialize weights with random weights
W_bound = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(np.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
name='Conv_W',
borrow=True)
# the bias is a 1D tensor -- one bias per output feature map
b_values = np.zeros((filter_shape[0], ),
dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, name='Conv_b', borrow=True)
# convolve input feature maps with filters
conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape,
border_mode=border_mode)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
if bch_norm:
np_g = np.ones(filter_shape[0]).astype(theano.config.floatX)
g = theano.shared(np_g, name='conv_bn_g', borrow=True)
self.g = g
normed = (conv_out - conv_out.mean(axis=(0, 2, 3), keepdims=True)
) / (conv_out.std(axis=(0, 2, 3), keepdims=True) + 1E-6)
pre_act = self.g.dimshuffle('x', 0, 'x',
'x') * normed + self.b.dimshuffle(
'x', 0, 'x', 'x')
else:
pre_act = conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
conv_func_out = apply_act(pre_act, act)
# if pooling should be done, downsample the images
if pool:
# downsample each feature map individually, using maxpooling
pooled_out = pool_2d(input=conv_func_out,
ds=poolsize,
ignore_border=True)
else:
pooled_out = conv_func_out
# use dropout=1 in train and dropout=0 in test
self.dropout = dropout
srng = RandomStreams(seed=param_seed)
pooled_out = T.switch(
T.gt(self.dropout, 0),
pooled_out * srng.normal(
size=theano.tensor.shape(pooled_out), avg=1.0, std=1.0),
pooled_out)
self.output = pooled_out
# store parameters of this layer
if bch_norm:
self.params = [self.W, self.b, self.g]
else:
self.params = [self.W, self.b]
def forward(self, X):
conv = conv2d(X,self.W) #Convolution
max_pool = pool.pool_2d(conv, ws=self.poolsz, ignore_border=True) #Max-pooling
return T.nnet.relu(max_pool + self.b.dimshuffle('x', 0, 'x', 'x')) #Non-linearity
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receive a gradient from
# number out feature maps * filter height * fitler width / poolsize. because stride = 1
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) //
numpy.prod(poolsize))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
self.Wv = theano.shared(numpy.zeros(filter_shape,
dtype=theano.config.floatX),
borrow=True)
# bias
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
self.bv = theano.shared(numpy.zeros((filter_shape[0], ),
dtype=theano.config.floatX),
borrow=True)
# Convolve input feature maps with filters
conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape)
# downsaple each feature map individually, using maxpooling
pooled_out = downsample.max_pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True)
# add the bias term. We first reshape it to a tensor of shape(1, n_filters, 1, 1). Each bias
# will thus be broadcasted across mini-batches and feature map width& height
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layers
self.params = [self.W, self.b]
self.velocity = [self.Wv, self.bv]
# ?? keep track of model input
self.input = input
else:
high = numpy.sqrt(6. / (numpy.sum(shape[:2]) * numpy.prod(shape[2:])))
shape = shape if n is None else (n, ) + shape
return theano.shared(numpy.asarray(numpy.random.uniform(low=-high,
high=high,
size=shape),
dtype=dtype),
name=name)
n_conv1 = 20
W1 = shared_glorot_uniform((n_conv1, 1, 5, 5))
conv1_out = conv2d(input=layer0_input,
filters=W1,
filter_shape=(n_conv1, 1, 5, 5),
input_shape=(batch_size, 1, 28, 28))
from theano.tensor.signal import pool
pooled_out = pool.pool_2d(input=conv1_out, ws=(2, 2), ignore_border=True)
n_conv2 = 50
W2 = shared_glorot_uniform((n_conv2, n_conv1, 5, 5))
conv2_out = conv2d(input=pooled_out,
filters=W2,
filter_shape=(n_conv2, n_conv1, 5, 5),
input_shape=(batch_size, n_conv1, 12, 12))
pooled2_out = pool.pool_2d(input=conv2_out, ws=(2, 2), ignore_border=True)
# initialize shared variable for weights.
# 16 random filters
rng = np.random.RandomState(23455)
w_shp = (24, 3, 9, 9)
w_bound = np.sqrt(3 * 9 * 9)
W_real = np.asarray(rng.uniform(low=-1.0 / w_bound,
high=1.0 / w_bound,
size=w_shp),
dtype=input.dtype)
plot_filters(W_real, sz=(6,4), title='Random filters', show=True)
W = theano.shared(W_real, name ='W')
# convolution, max-pooling and ReLU
f1 = theano.function([input], conv2d(input, W))
f2 = theano.function([input], pool.pool_2d(input, (16, 16), ignore_border=True))
feat_maps = relu(f2(f1(np.asarray(image_list, dtype='float32').transpose(0,3,1,2))))
feat_maps = feat_maps.reshape(feat_maps.shape[0],np.prod(feat_maps.shape[1:]))
if PCA_dim > 0:
pca = PCA(n_components=PCA_dim)
# 10-fold cross-validation
print('performing cross-validation using SVM regression')
n_folds = 10
# shuffle samples
ids = np.random.permutation(feat_maps.shape[0])
feat_maps = feat_maps[ids,:]
labels_ = labels[ids]
def __init__(self, rng, filter_shape, image_shape, poolsize=2, xin=None):
assert image_shape[1] == filter_shape[1]
self.image_shape = theano.shared(value=np.asarray(image_shape,
dtype='int16'),
borrow=True)
self.poolsize = (poolsize, poolsize)
#self.input = input
if xin:
self.x = xin
else:
self.x = T.matrix(name='input')
self.x1 = self.x.reshape(self.image_shape, ndim=4)
self.filter_shape = filter_shape
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = np.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) /
np.prod(self.poolsize))
# initialize weights with random weights
W_bound = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(np.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
self.W_prime = self.W[:, :, ::-1, ::-1]
self.W_prime = self.W_prime.dimshuffle(1, 0, 2, 3)
#self.W_prime=self.W_prime[:,::-1]
#print self.W.get_value()
#print self.W_prime.eval()
# the bias is a 1D tensor -- one bias per output feature map
b_values = np.zeros((filter_shape[0], ), dtype=theano.config.floatX)
bp_values = np.zeros((filter_shape[1], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
self.b_prime = theano.shared(value=bp_values, borrow=True)
if poolsize < -1:
self.x1 = self.x1.repeat(int(-poolsize),
axis=2).repeat(int(-poolsize), axis=3)
# convolve input feature maps with filters
conv_out = conv2d(
input=self.x1,
filters=self.W,
filter_shape=filter_shape,
#image_shape=self.image_shape.eval(),
border_mode='full')
bp = (filter_shape[2] - 1) / 2
conv_out = conv_out[:, :, bp:-bp, bp:-bp]
# downsample each feature map individually, using maxpooling
if poolsize > 1:
try:
self.pooled_out = pool.pool_2d(input=conv_out,
ws=self.poolsize,
ignore_border=True)
except:
self.pooled_out = pool.pool_2d(input=conv_out,
ds=self.poolsize,
ignore_border=True)
else:
self.pooled_out = conv_out
self.hidden = T.maximum(
0, (self.pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')))
# store parameters of this layer
self.params = [self.W, self.b]
def __init__(self, input, filter_shape, image_shape, poolsize=(2, 2)):
"""
Allocate a NetConvPoolLayer with shared variable internal parameters.
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = np.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) //
np.prod(poolsize))
# initialize weights with random weights
W_bound = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(np.random.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX,
name='W')
# the bias is a 1D tensor -- one bias per output feature map
b_values = np.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
self.conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
image_shape=image_shape)
# downsample each feature map individually, using maxpooling
self.pooled_out = downsample.max_pool_2d(input=self.conv_out,
ds=poolsize,
ignore_border=True)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.tanh(self.pooled_out +
self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
# keep track of model input
self.input = input
def forward(self, X):
conv_out = conv2d(input=X, filters=self.W)
pooled_out = downsample.max_pool_2d(input=conv_out,
ds=self.poolsz,
ignore_border=True)
return T.nnet.relu(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2),
pool_ignore_border=True, normal=False, std_normal=2, binary=True, normalization=True, eps=1e-4):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
:type binary: boolean
:param binary: use binarized weights for output or not
:type epsilon: float
:param epsilon: normalization variable
:type normalization: boolean
:param normalization: normalization output or not
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) //
numpy.prod(poolsize))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
if normal:
self.W = theano.shared(
numpy.asarray(
rng.normal(loc=0, scale=std_normal, size=filter_shape),
dtype=theano.config.floatX
),
borrow=True
)
else:
self.W = theano.shared(
numpy.asarray(
# rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
2*rng.binomial(n=1, p=.5, size=filter_shape) - 1,
dtype=theano.config.floatX
),
borrow=True
)
# Generate binarized weights
self.W_bin = self.binarize()
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
if binary:
conv_out = conv2d(
input=input,
filters=self.W_bin,
filter_shape=filter_shape,
image_shape=image_shape
)
else:
conv_out = conv2d(
input=input,
filters=self.W,
filter_shape=filter_shape,
image_shape=image_shape
)
if normalization:
conv_out = (conv_out - T.mean(conv_out))/T.sqrt(T.var(conv_out) + eps)
# downsample each feature map individually, using maxpooling
pooled_out = downsample.max_pool_2d(
input=conv_out,
ds=poolsize,
ignore_border=pool_ignore_border
)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
self.params_bin = [self.W_bin, self.b]
# keep track of model input
self.input = input
def __init__(self,
rng,
is_train,
input_data,
filter_shape,
image_shape,
ssample=(1, 1),
bordermode='valid',
p=0.5):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
assert image_shape[1] == filter_shape[1]
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) //
numpy.prod(ssample))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
gamma_values = numpy.ones((filter_shape[0], ),
dtype=theano.config.floatX)
self.gamma = theano.shared(value=gamma_values, borrow=True)
beta_values = numpy.zeros((filter_shape[0], ),
dtype=theano.config.floatX)
self.beta = theano.shared(value=beta_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv2d(input=input_data,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape,
subsample=ssample,
border_mode=bordermode)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
lin_output = conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
bn_output = batch_normalization(
inputs=lin_output,
gamma=self.gamma.dimshuffle('x', 0, 'x', 'x'),
beta=self.beta.dimshuffle('x', 0, 'x', 'x'),
mean=lin_output.mean((0, ), keepdims=True),
std=lin_output.std((0, ), keepdims=True),
mode='low_mem')
activated_output = T.nnet.relu(bn_output)
dropped_output = drop(activated_output, p)
self.output = T.switch(T.neq(is_train, 0), dropped_output,
p * activated_output)
# store parameters of this layer
self.params = [self.W, self.b, self.gamma, self.beta]
# keep track of model input
self.input = input_data
def __init__(self,
emb,
pos,
nc=2,
de=100,
disc_h=250,
fs=[3, 4, 5],
nf=300,
emb_reg=False,
pos_reg=False,
longhist=True):
'''
emb :: Embedding Matrix
nh :: hidden layer size
nc :: Number of classes
de :: Dimensionality of word embeddings
p_drop :: Dropout probability
'''
# Source Embeddings
self.emb = theano.shared(name='Words', value=emb.astype('float32'))
self.target_emb = theano.shared(name='Wordst',
value=emb.astype('float32'))
self.avg_emb = theano.shared(name='Wordsa',
value=emb.astype('float32'))
self.pos = theano.shared(name='Pos', value=pos.astype('float32'))
self.target_pos = theano.shared(name='Post',
value=pos.astype('float32'))
self.avg_pos = theano.shared(name='Posta', value=pos.astype('float32'))
# Targt Embeddings
# Source Output Weights
self.w_o = theano.shared(name='w_o',
value=he_normal(
(nf * len(fs), 1)).astype('float32'))
self.b_o = theano.shared(name='b_o',
value=np.zeros((1, )).astype('float32'))
# Discriminator Weights
self.w_h_1 = theano.shared(name='w_h_1',
value=he_normal((nf * len(fs),
disc_h)).astype('float32'))
self.b_h_1 = theano.shared(name='b_h_1',
value=np.zeros(
(disc_h, )).astype('float32'))
self.w_h_2 = theano.shared(name='w_h_2',
value=he_normal(
(disc_h, disc_h)).astype('float32'))
self.b_h_2 = theano.shared(name='b_h_2',
value=np.zeros(
(disc_h, )).astype('float32'))
self.w_adv = theano.shared(name='w_adv',
value=he_normal(
(disc_h, 1)).astype('float32'))
self.b_adv = theano.shared(name='b_adv',
value=np.zeros((1, )).astype('float32'))
# Update these parameters
self.params_source = [self.w_o, self.b_o, self.emb, self.pos]
self.params_target = [self.target_emb, self.target_pos]
#self.params_discriminator = [self.w_h_1, self.b_h_1, self.w_h_3, self.b_h_3,
self.params_discriminator = [
self.w_h_1, self.b_h_1, self.w_h_2, self.b_h_2, self.w_adv,
self.b_adv
]
source_idxs = T.matrix()
target_idxs = T.matrix()
source_e1_pos_idxs = T.matrix()
source_e2_pos_idxs = T.matrix()
target_e1_pos_idxs = T.matrix()
target_e2_pos_idxs = T.matrix()
source_Y = T.ivector()
# get word embeddings based on indicies
source_x_word = self.emb[T.cast(source_idxs, 'int32')]
source_x_e1_pos = self.pos[T.cast(source_e1_pos_idxs, 'int32')]
source_x_e2_pos = self.pos[T.cast(source_e2_pos_idxs, 'int32')]
source_x_word = T.concatenate(
[source_x_word, source_x_e1_pos, source_x_e2_pos], axis=2)
mask = T.neq(source_idxs, 0) * 1
source_x_word = source_x_word * mask.dimshuffle(0, 1, 'x')
source_x_word = source_x_word.reshape(
(source_x_word.shape[0], 1, source_x_word.shape[1],
source_x_word.shape[2]))
target_x_word = self.target_emb[T.cast(target_idxs, 'int32')]
target_x_e1_pos = self.target_pos[T.cast(target_e1_pos_idxs, 'int32')]
target_x_e2_pos = self.target_pos[T.cast(target_e2_pos_idxs, 'int32')]
target_x_word = T.concatenate(
[target_x_word, target_x_e1_pos, target_x_e2_pos], axis=2)
mask2 = T.neq(target_idxs, 0) * 1
target_x_word = target_x_word * mask2.dimshuffle(0, 1, 'x')
target_x_word = target_x_word.reshape(
(target_x_word.shape[0], 1, target_x_word.shape[1],
target_x_word.shape[2]))
de = de + 2 * pos.shape[1]
source_cnn_w, source_cnn_b = cnn_weights(de, fs, nf)
target_cnn_w, target_cnn_b = cnn_weights(de, fs, nf)
avg_cnn_w, avg_cnn_b = cnn_weights(de, fs, nf)
self.params_source += source_cnn_w + source_cnn_b
self.params_target += target_cnn_w + target_cnn_b
self.params_avg = avg_cnn_w + avg_cnn_b
dropout_switch = T.scalar()
real_attention = T.scalar()
source_l1_w_all = []
for w, b, width in zip(source_cnn_w, source_cnn_b, fs):
l1_w = conv2d(source_x_word,
w,
image_shape=(None, 1, None, de),
filter_shape=(nf, 1, width, de))
l1_w = rectify(l1_w + b.dimshuffle('x', 0, 'x', 'x'))
l1_w = T.max(l1_w, axis=2).flatten(2)
source_l1_w_all.append(l1_w)
target_l1_w_all = []
for w, b, width in zip(target_cnn_w, target_cnn_b, fs):
l1_w = conv2d(target_x_word,
w,
image_shape=(None, 1, None, de),
filter_shape=(nf, 1, width, de))
l1_w = rectify(l1_w + b.dimshuffle('x', 0, 'x', 'x'))
l1_w = T.max(l1_w, axis=2).flatten(2)
target_l1_w_all.append(l1_w)
source_h = T.concatenate(source_l1_w_all, axis=1)
source_h = dropout(source_h, dropout_switch, 0.5)
target_h = T.concatenate(target_l1_w_all, axis=1)
target_h = dropout(target_h, dropout_switch, 0.5)
pyx_source = T.nnet.nnet.sigmoid(
T.dot(source_h, self.w_o) + self.b_o.dimshuffle('x', 0))
pyx_source = T.clip(pyx_source, 1e-5, 1 - 1e-5)
self.step = theano.shared(np.float32(0))
source_h2 = rectify(T.dot(source_h, self.w_h_1) + self.b_h_1)
source_h2 = dropout(source_h2, dropout_switch, 0.5)
source_h2 = rectify(T.dot(source_h2, self.w_h_2) + self.b_h_2)
source_h2 = dropout(source_h2, dropout_switch, 0.5)
target_h2 = rectify(T.dot(target_h, self.w_h_1) + self.b_h_1)
target_h2 = dropout(target_h2, dropout_switch, 0.5)
target_h2 = rectify(T.dot(target_h2, self.w_h_2) + self.b_h_2)
target_h2 = dropout(target_h2, dropout_switch, 0.5)
pyx_adv_source = T.nnet.nnet.sigmoid(
T.dot(source_h2, self.w_adv) + self.b_adv.dimshuffle('x', 0))
pyx_adv_source = T.clip(pyx_adv_source, 1e-5, 1 - 1e-5)
pyx_adv_target = T.nnet.nnet.sigmoid(
T.dot(target_h2, self.w_adv) + self.b_adv.dimshuffle('x', 0))
pyx_adv_target = T.clip(pyx_adv_target, 1e-5, 1 - 1e-5)
pyx_test = T.nnet.nnet.sigmoid(
T.dot(target_h, self.w_o) + self.b_o.dimshuffle('x', 0))
# Generator Loss
#L_adv_generator = -.9*T.log(pyx_adv_target).mean() - .1*T.log(1.-pyx_adv_target).mean()
L_adv_generator = -T.log(pyx_adv_target).mean()
num_updates = theano.shared(as_floatX(1.).astype("float32"))
if emb_reg:
L_adv_generator += .5 * ((self.avg_emb - self.target_emb)**2).sum()
if pos_reg:
L_adv_generator += .5 * ((self.avg_pos - self.target_pos)**2).sum()
if True:
L_adv_generator += .5 * sum(
[((s - t)**2).sum()
for s, t in zip(self.params_avg, self.params_target[2:])])
updates_generator, _ = Adam(L_adv_generator,
self.params_target,
lr2=0.0002)
if not longhist:
updates_generator.append(
(self.avg_emb, 0.9 * self.avg_emb + 0.1 * self.target_emb))
updates_generator.append(
(self.avg_pos, 0.9 * self.avg_pos + 0.1 * self.target_pos))
for p, t in zip(self.params_avg, self.params_target[2:]):
updates_generator.append((p, 0.9 * p + 0.1 * t))
else:
updates_generator.append(
(self.avg_emb, self.avg_emb + self.target_emb))
updates_generator.append(
(self.avg_pos, self.avg_pos + self.target_pos))
updates_generator.append((num_updates, num_updates + 1.))
self.train_batch_generator = theano.function([target_idxs, target_e1_pos_idxs, target_e2_pos_idxs,\
dropout_switch],
L_adv_generator, updates=updates_generator, allow_input_downcast=True, on_unused_input='ignore')
L_adv_discriminator = -T.log(1 - pyx_adv_target).mean() - (
(srng2.uniform(low=0.7, high=1., size=pyx_adv_target.shape)) *
T.log(pyx_adv_source)).mean()
#L_adv_discriminator = -0.9*T.log(1.-pyx_adv_target).mean() - .9*T.log(pyx_adv_source).mean() - .1*T.log(pyx_adv_target).mean() - .1*T.log(1.-pyx_adv_source).mean()
#L_adv_discriminator = -T.log(1.-pyx_adv_target).mean() - T.log(pyx_adv_source).mean()
#L_adv_discriminator += 1e-2*sum([(x**2).sum() for x in self.params_discriminator])
updates_discriminator, self.disc_lr = Adam(L_adv_discriminator,
self.params_discriminator,
lr2=0.0002)
#L_source = T.nnet.binary_crossentropy(pyx_source.flatten(), source_Y).mean() + 1e-4 * sum([(x**2).sum() for x in self.params_source])
L_source = T.nnet.binary_crossentropy(
pyx_source.flatten(),
source_Y).mean() + 1e-4 * sum([(x**2).sum()
for x in self.params_source])
updates_source, _ = Adam(L_source, self.params_source, lr2=0.001)
self.train_batch_source = theano.function([source_idxs, source_e1_pos_idxs, source_e2_pos_idxs, source_Y,\
dropout_switch],
L_source, updates=updates_source, allow_input_downcast=True, on_unused_input='ignore')
self.train_batch_discriminator = theano.function([target_idxs, source_idxs, target_e1_pos_idxs, target_e2_pos_idxs,\
source_e1_pos_idxs, source_e2_pos_idxs, dropout_switch],
L_adv_discriminator, updates=updates_discriminator, allow_input_downcast=True, on_unused_input='ignore')
self.features = theano.function([target_idxs, target_e1_pos_idxs, target_e2_pos_idxs, dropout_switch],\
target_h, allow_input_downcast=True, on_unused_input='ignore')
self.predict_proba = theano.function([target_idxs, target_e1_pos_idxs, target_e2_pos_idxs, dropout_switch],\
pyx_test.flatten(), allow_input_downcast=True, on_unused_input='ignore')
self.predict_src_proba = theano.function([source_idxs, source_e1_pos_idxs, source_e2_pos_idxs, dropout_switch],\
pyx_source.flatten(), allow_input_downcast=True, on_unused_input='ignore')
def __init__(self,
rng,
inputVar,
cfgParams,
copyLayer=None,
layerNum=None):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type inputVar: theano.tensor.dtensor4
:param inputVar: symbolic image tensor, of shape image_shape
:type cfgParams: ConvPoolLayerParams
"""
import theano
import theano.tensor as T
from theano.tensor.signal.pool import pool_2d
from theano.tensor.nnet import conv2d
super(ConvPoolLayer, self).__init__(rng)
assert isinstance(cfgParams, ConvPoolLayerParams)
floatX = theano.config.floatX # @UndefinedVariable
filter_shape = cfgParams.filter_shape
image_shape = cfgParams.image_shape
filter_stride = cfgParams.stride
poolsize = cfgParams.poolsize
poolType = cfgParams.poolType
activation = cfgParams.activation
inputDim = cfgParams.inputDim
border_mode = cfgParams.border_mode
self.cfgParams = cfgParams
self.layerNum = layerNum
assert image_shape[1] == filter_shape[1]
self.inputVar = inputVar
# there are "num inputVar feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" / pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) /
numpy.prod(filter_stride) / numpy.prod(poolsize))
if not (copyLayer is None):
self.W = copyLayer.W
else:
wInitVals = self.getInitVals(filter_shape,
'conv',
act_fn=cfgParams.activation_str,
orthogonal=False,
method=cfgParams._init_method)
self.W = theano.shared(wInitVals,
borrow=True,
name='convW{}'.format(layerNum))
# the bias is a 1D tensor -- one bias per output feature map
if self.cfgParams.hasBias is True:
if not (copyLayer is None):
self.b = copyLayer.b
else:
b_values = numpy.zeros((filter_shape[0], ), dtype=floatX)
self.b = theano.shared(value=b_values,
borrow=True,
name='convB{}'.format(layerNum))
if border_mode == 'same':
# convolve inputVar feature maps with filters
conv_out = conv2d(input=inputVar,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape,
subsample=filter_stride,
border_mode='full')
# perform full convolution and crop output of input size
offset_2 = filter_shape[2] // 2
offset_3 = filter_shape[3] // 2
conv_out = conv_out[:, :, offset_2:offset_2 + image_shape[2],
offset_3:offset_3 + image_shape[3]]
else:
# convolve inputVar feature maps with filters
conv_out = conv2d(input=inputVar,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape,
subsample=filter_stride,
border_mode=border_mode)
# downsample each feature map individually, using maxpooling
if poolType == 0:
# use maxpooling
pooled_out = pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True,
mode='max')
elif poolType == 1:
# use average pooling
pooled_out = pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True,
mode='average_inc_pad')
elif poolType == 3:
# use subsampling and ignore border
pooled_out = conv_out[:, :, :(inputDim[2] // poolsize[0]) *
poolsize[0], :(inputDim[3] // poolsize[1]) *
poolsize[1]][:, :, ::2, ::2]
elif poolType == -1:
# no pooling at all
pooled_out = conv_out
else:
raise NotImplementedError()
# add the bias term. Since the bias is a vector (1D array), we first reshape it to a tensor of shape
# (1,n_filters,1,1). Each bias will thus be broadcasted across mini-batches and feature map width & height
if self.cfgParams.hasBias is True:
lin_output = pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')
else:
lin_output = pooled_out
self.output_pre_act = lin_output
self.output = (lin_output
if activation is None else activation(lin_output))
self.output.name = 'output_layer_{}'.format(self.layerNum)
# store parameters of this layer
self.params = [self.W, self.b] if self.cfgParams.hasBias else [self.W]
self.weights = [self.W]
def __init__(self,
rng,
inputVar,
cfgParams,
copyLayer=None,
layerNum=None):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type inputVar: theano.tensor.dtensor4
:param inputVar: symbolic image tensor, of shape image_shape
:type cfgParams: ConvPoolLayerParams
"""
import theano
import theano.tensor as T
from theano.tensor.nnet import conv2d
super(ConvLayer, self).__init__(rng)
assert isinstance(cfgParams, ConvLayerParams)
floatX = theano.config.floatX # @UndefinedVariable
filter_shape = cfgParams.filter_shape
image_shape = cfgParams.image_shape
filter_stride = cfgParams.stride
activation = cfgParams.activation
inputDim = cfgParams.inputDim
border_mode = cfgParams.border_mode
self.cfgParams = cfgParams
self.layerNum = layerNum
assert image_shape[1] == filter_shape[1]
self.inputVar = inputVar
if not (copyLayer is None):
self.W = copyLayer.W
else:
wInitVals = self.getInitVals(filter_shape,
'conv',
act_fn=cfgParams.activation_str,
orthogonal=False,
method=cfgParams._init_method)
self.W = theano.shared(wInitVals,
borrow=True,
name='convW{}'.format(layerNum))
# the bias is a 1D tensor -- one bias per output feature map
if self.cfgParams.hasBias is True:
if not (copyLayer is None):
self.b = copyLayer.b
else:
b_values = numpy.zeros((filter_shape[0], ), dtype=floatX)
self.b = theano.shared(value=b_values,
borrow=True,
name='convB{}'.format(layerNum))
# convolve inputVar feature maps with filters
conv_out = conv2d(input=inputVar,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape,
subsample=filter_stride,
border_mode=border_mode)
# add the bias term. Since the bias is a vector (1D array), we first reshape it to a tensor of shape
# (1,n_filters,1,1). Each bias will thus be broadcasted across mini-batches and feature map width & height
if self.cfgParams.hasBias is True:
lin_output = conv_out + self.b.dimshuffle('x', 0, 'x', 'x')
else:
lin_output = conv_out
self.output_pre_act = lin_output
self.output = (lin_output
if activation is None else activation(lin_output))
self.output.name = 'output_layer_{}'.format(self.layerNum)
# store parameters of this layer
self.params = [self.W, self.b] if self.cfgParams.hasBias else [self.W]
self.weights = [self.W]
size=w_shp),
dtype=input.dtype),
name='W')
# initialize shared variable for bias (1D tensor) with random values
# IMPORTANT: biases are usually initialized to zero. However in this
# particular application, we simply apply the convolutional layer to
# an image without learning the parameters. We therefore initialize
# them to random values to "simulate" learning.
b_shp = (2, )
b = theano.shared(numpy.asarray(rng.uniform(low=-.5, high=.5, size=b_shp),
dtype=input.dtype),
name='b')
# build symbolic expression that computes the convolution of input with filters in w
conv_out = conv2d(input, W)
# build symbolic expression to add bias and apply activation function, i.e. produce neural net layer output
# A few words on ``dimshuffle`` :
# ``dimshuffle`` is a powerful tool in reshaping a tensor;
# what it allows you to do is to shuffle dimension around
# but also to insert new ones along which the tensor will be
# broadcastable;
# dimshuffle('x', 2, 'x', 0, 1)
# This will work on 3d tensors with no broadcastable
# dimensions. The first dimension will be broadcastable,
# then we will have the third dimension of the input tensor as
# the second of the resulting tensor, etc. If the tensor has
# shape (20, 30, 40), the resulting tensor will have dimensions
# (1, 40, 1, 20, 30). (AxBxC tensor is mapped to 1xCx1xAxB tensor)
# More examples:
def __init__(self,
rng,
input,
filter_shape,
p1,
p2,
image_shape,
poolsize=(2, 2),
W=None,
b=None):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type p1: int
:param p1: padding for input image in x direction
:type p2: int
:param p2: padding for input image in y direction
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) //
numpy.prod(poolsize))
if (W is None):
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(
low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
else:
self.W = theano.shared(W)
if (b is None):
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0], ),
dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
else:
self.b = theano.shared(b)
# convolve input feature maps with filters
conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
border_mode=(p1, p2),
input_shape=image_shape,
subsample=(1, 1))
# pool each feature map individually, using maxpooling
pooled_out = pool.pool_2d(input=conv_out,
ws=poolsize,
ignore_border=True)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.nnet.relu(pooled_out +
self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
# keep track of model input
self.input = input
def __init__(self,
input,
filter_shape,
image_shape,
f_params_w,
f_params_b,
lrn=False,
t_style=None,
t_content=None,
convstride=1,
padsize=0,
group=1,
poolsize=3,
poolstride=1):
self.input = input
if t_style is not None:
self.t_style = np.asarray(np.load(t_style),
dtype=theano.config.floatX)
if t_content is not None:
self.t_content = np.asarray(np.load(t_content),
dtype=theano.config.floatX)
if lrn is True:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = theano.shared(np.asarray(np.transpose(
np.load(os.path.join(params_path, f_params_w)), (3, 0, 1, 2)),
dtype=theano.config.floatX),
borrow=True)
self.b = theano.shared(np.asarray(np.load(
os.path.join(params_path, f_params_b)),
dtype=theano.config.floatX),
borrow=True)
conv_out = conv2d(input=self.input,
filters=self.W,
filter_shape=filter_shape,
border_mode=padsize,
subsample=(convstride, convstride),
filter_flip=True)
elif group == 2:
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[1] = self.filter_shape[1] / 2
self.image_shape[1] = self.image_shape[1] / 2
self.W0 = theano.shared(np.asarray(np.transpose(
np.load(os.path.join(params_path, f_params_w[0])),
(3, 0, 1, 2)),
dtype=theano.config.floatX),
borrow=True)
self.W1 = theano.shared(np.asarray(np.transpose(
np.load(os.path.join(params_path, f_params_w[1])),
(3, 0, 1, 2)),
dtype=theano.config.floatX),
borrow=True)
self.b0 = theano.shared(np.asarray(np.load(
os.path.join(params_path, f_params_b[0])),
dtype=theano.config.floatX),
borrow=True)
self.b1 = theano.shared(np.asarray(np.load(
os.path.join(params_path, f_params_b[1])),
dtype=theano.config.floatX),
borrow=True)
conv_out0 = conv2d(input=self.input[:, :self.image_shape[1], :, :],
filters=self.W0,
filter_shape=tuple(self.filter_shape),
border_mode=padsize,
subsample=(convstride, convstride),
filter_flip=True) + self.b0.dimshuffle(
'x', 0, 'x', 'x')
conv_out1 = conv2d(input=self.input[:, self.image_shape[1]:, :, :],
filters=self.W1,
filter_shape=tuple(self.filter_shape),
border_mode=padsize,
subsample=(convstride, convstride),
filter_flip=True) + self.b1.dimshuffle(
'x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
else:
raise AssertionError()
relu_out = T.maximum(conv_out, 0)
if poolsize != 1:
self.output = pool.pool_2d(input=relu_out,
ds=(poolsize, poolsize),
ignore_border=True,
st=(poolstride, poolstride),
mode='average_exc_pad')
else:
self.output = relu_out
if lrn is True:
self.output = self.lrn_func(self.output)
def ResidualUnit(name,
left_b,
right_b,
load_params,
cur_dim,
down_dim,
up_dim,
stride=(1, 1),
left_convolve=False):
# param key names
left_res = 'res%s_branch1' % (name)
left_bn = 'bn%s_branch1' % (name)
right_res_a = 'res%s_branch2a' % (name)
right_bn_a = 'bn%s_branch2a' % (name)
right_res_b = 'res%s_branch2b' % (name)
right_bn_b = 'bn%s_branch2b' % (name)
right_res_c = 'res%s_branch2c' % (name)
right_bn_c = 'bn%s_branch2c' % (name)
# init theano weights
# NOTE: the np.loaded params do not have any bias vectors
# except for the final dense layer, so no bias additions here
params = []
if left_convolve:
left_res_W = theano.shared(value=np.transpose(
load_params[left_res + '/W'], (3, 2, 0, 1)),
borrow=True,
name=left_res + '_W')
left_bn_beta = theano.shared(load_params[left_bn + '/beta'],
borrow=True,
name=left_bn + '_beta')
left_bn_gamma = theano.shared(load_params[left_bn + '/gamma'],
borrow=True,
name=left_bn + '_gamma')
left_bn_mean = theano.shared(load_params[left_bn + '/mean/EMA'],
borrow=True,
name=left_bn + '_mean')
left_bn_std = theano.shared(np.sqrt(load_params[left_bn +
'/variance/EMA']),
borrow=True,
name=left_bn + '_std-dev')
params += [
left_res_W, left_bn_gamma, left_bn_beta, left_bn_mean, left_bn_std
]
right_res_a_W = theano.shared(value=np.transpose(
load_params[right_res_a + '/W'], (3, 2, 0, 1)),
borrow=True,
name=right_res_a + '_W')
right_bn_a_beta = theano.shared(load_params[right_bn_a + '/beta'],
borrow=True,
name=right_bn_a + '_beta')
right_bn_a_gamma = theano.shared(load_params[right_bn_a + '/gamma'],
borrow=True,
name=right_bn_a + '_gamma')
right_bn_a_mean = theano.shared(load_params[right_bn_a + '/mean/EMA'],
borrow=True,
name=right_bn_a + '_mean')
right_bn_a_std = theano.shared(np.sqrt(load_params[right_bn_a +
'/variance/EMA']),
borrow=True,
name=right_bn_a + '_std-dev')
right_res_b_W = theano.shared(value=np.transpose(
load_params[right_res_b + '/W'], (3, 2, 0, 1)),
borrow=True,
name=right_res_b + '_W')
right_bn_b_beta = theano.shared(load_params[right_bn_b + '/beta'],
borrow=True,
name=right_bn_b + '_beta')
right_bn_b_gamma = theano.shared(load_params[right_bn_b + '/gamma'],
borrow=True,
name=right_bn_b + '_gamma')
right_bn_b_mean = theano.shared(load_params[right_bn_b + '/mean/EMA'],
borrow=True,
name=right_bn_b + '_mean')
right_bn_b_std = theano.shared(np.sqrt(load_params[right_bn_b +
'/variance/EMA']),
borrow=True,
name=right_bn_b + '_std-dev')
right_res_c_W = theano.shared(value=np.transpose(
load_params[right_res_c + '/W'], (3, 2, 0, 1)),
borrow=True,
name=right_res_c + '_W')
right_bn_c_beta = theano.shared(load_params[right_bn_c + '/beta'],
borrow=True,
name=right_bn_c + '_beta')
right_bn_c_gamma = theano.shared(load_params[right_bn_c + '/gamma'],
borrow=True,
name=right_bn_c + '_gamma')
right_bn_c_mean = theano.shared(load_params[right_bn_c + '/mean/EMA'],
borrow=True,
name=right_bn_c + '_mean')
right_bn_c_std = theano.shared(np.sqrt(load_params[right_bn_c +
'/variance/EMA']),
borrow=True,
name=right_bn_c + '_std-dev')
params += [
right_res_a_W, right_bn_a_gamma, right_bn_a_beta, right_bn_a_mean,
right_bn_a_std, right_res_b_W, right_bn_b_gamma, right_bn_b_beta,
right_bn_b_mean, right_bn_b_std, right_res_c_W, right_bn_c_gamma,
right_bn_c_beta, right_bn_c_mean, right_bn_c_std
]
# make tensor graph
if left_convolve:
left_conv_out = conv2d(input=left_b,
filters=left_res_W,
filter_shape=(up_dim, cur_dim, 1, 1),
subsample=stride)
left_out = BN(inputs=left_conv_out.dimshuffle(0, 2, 3, 1),
gamma=left_bn_gamma,
beta=left_bn_beta,
mean=left_bn_mean,
std=left_bn_std).dimshuffle(0, 3, 1, 2)
else:
left_out = left_b
right_conv_a_out = conv2d(input=right_b,
filters=right_res_a_W,
filter_shape=(down_dim, cur_dim, 1, 1),
subsample=stride)
right_a_out = T.nnet.relu(
BN(inputs=right_conv_a_out.dimshuffle(0, 2, 3, 1),
gamma=right_bn_a_gamma,
beta=right_bn_a_beta,
mean=right_bn_a_mean,
std=right_bn_a_std).dimshuffle(0, 3, 1, 2))
right_conv_b_out = conv2d(input=right_a_out,
filters=right_res_b_W,
filter_shape=(down_dim, down_dim, 3, 3),
border_mode=(1, 1))
right_b_out = T.nnet.relu(
BN(inputs=right_conv_b_out.dimshuffle(0, 2, 3, 1),
gamma=right_bn_b_gamma,
beta=right_bn_b_beta,
mean=right_bn_b_mean,
std=right_bn_b_std).dimshuffle(0, 3, 1, 2))
right_c_conv_out = conv2d(input=right_b_out,
filters=right_res_c_W,
filter_shape=(up_dim, down_dim, 1, 1))
right_out = BN(inputs=right_c_conv_out.dimshuffle(0, 2, 3, 1),
gamma=right_bn_c_gamma,
beta=right_bn_c_beta,
mean=right_bn_c_mean,
std=right_bn_c_std).dimshuffle(0, 3, 1, 2)
output = T.nnet.relu(left_out + right_out)
return output, params, 4 * down_dim
def __init__(self,
rng,
input,
filter_shape,
image_shape,
poolsize=(2, 2),
needPool=True):
"""
根据传入的参数初始化卷基层
:tpye rng: numpy.random.RandomState
:param rng: 用于初始化权值的随机种子
:tpye input: theano.tensor.dtensor4
:param input: 用于表征image_shape的形状
:type filter_shape: 长度为4的tuple或者list
:param filter_shape: (number_of_filters, number of input feature maps,
filter_height, filter_width)
:type image_shape: 长度为4的tuple或者list
:param image_shape:(batch_size, num of input feature maps,
image height, image width)
:type poolsize: 长度为2的tuple或者list
:param poolsize: （x,y）,x,y分别表示下采样的规模
"""
assert image_shape[1] == filter_shape[1]
self.input = input
fan_in = numpy.prod(filter_shape[1:])
#fan_out的值为：filter的个数 × filter的宽 × filter的高 ÷
# 下采样核心的宽 × 下采样核心的高
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) //
numpy.prod(poolsize))
#用随机种子初始化权值参数--Weights矩阵
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
#将全部bias赋值为0
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
#调用theano的conv2d函数，对输入数据结构进行卷积构造
conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape)
# if needPool == True:
#进行下采样操作
pooled_out = downsample.max_pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True)
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
self.params = [self.W, self.b]
self.input = input
def deconv(X, w, s=2, b=None):
z = conv2d(X, w, border_mode='full')[:, :, s:-s, s:-s]
if b is not None:
z += b.dimshuffle('x', 0, 'x', 'x')
return z
def __init__(self,
rng,
trainmode,
running_average_factor,
input,
filter_shape,
image_shape,
poolsize=(2, 2),
name='convlayer'):
self.input = input
self.trainmode = trainmode
self.running_average_factor = running_average_factor
fan_in = np.prod(filter_shape[1:])
fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) //
np.prod(poolsize))
# initialize weights with random weights
W_bound = np.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(np.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
name='W_' + name,
borrow=True)
# b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX)
# self.b = theano.shared(value=b_values, name='b_'+name,borrow=True)
self.conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape)
# pool each feature map individually, using maxpooling
self.pooled_out = pool.pool_2d(input=self.conv_out,
ws=poolsize,
ignore_border=True)
self.gamma = theano.shared(np.asarray(rng.uniform(
low=-np.sqrt(6. / (filter_shape[0])),
high=np.sqrt(6. / (filter_shape[0])),
size=(filter_shape[0], )),
dtype=theano.config.floatX),
name='gamma_' + name,
borrow=True)
self.beta = theano.shared(np.zeros((filter_shape[0], ),
dtype=theano.config.floatX),
name='beta_' + name,
borrow=True)
self.new_running_mean = theano.shared(np.zeros(
(filter_shape[0]), dtype=theano.config.floatX),
name='rmean_' + name,
borrow=True)
self.new_running_var = theano.shared(np.zeros(
(filter_shape[0]), dtype=theano.config.floatX),
name='rvar_' + name,
borrow=True)
mean = T.mean(self.pooled_out,
axis=[0, 2, 3]) #.dimshuffle('x', 0, 'x', 'x')
var = T.mean(T.sqr(self.pooled_out -
mean.dimshuffle('x', 0, 'x', 'x')),
axis=[0, 2, 3]) #.dimshuffle('x', 0, 'x', 'x')
self.bnTr = (self.pooled_out - mean.dimshuffle('x', 0, 'x', 'x')) / (
T.sqrt(var.dimshuffle('x', 0, 'x', 'x') + 0.0001))
self.bnTe = (self.pooled_out - self.new_running_mean.dimshuffle(
'x', 0, 'x', 'x')) / (T.sqrt(
self.new_running_var.dimshuffle('x', 0, 'x', 'x') + 0.0001))
self.lbnOut = T.switch(self.trainmode, self.bnTr, self.bnTe)
self.actInput = self.gamma.dimshuffle(
'x', 0, 'x', 'x') * self.lbnOut + self.beta.dimshuffle(
'x', 0, 'x', 'x')
# activation = lambda x: T.switch(T.gt(x,6), 1+T.mul(x,0.1) ,
# T.switch(T.lt(x,-6),T.mul(x,0.1),(1/12.)*x+0.5))
self.output = T.nnet.relu(self.actInput)
# self.output = activation(self.actInput)
self.params = [self.W, self.gamma, self.beta]
# self.params = [self.W, self.beta]
self.raupdates = [
(self.new_running_mean,
self.new_running_mean * self.running_average_factor + mean *
(1 - self.running_average_factor)),
(self.new_running_var,
self.new_running_var * self.running_average_factor + var *
(1 - self.running_average_factor))
]
self.raparams = [self.new_running_mean, self.new_running_var]
def __init__(self, rng, input, filter_shape):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: np.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
W_bound = 1.0 / np.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
self.W = theano.shared(
np.asarray(
rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX
),
borrow=True
)
# the bias is a 1D tensor -- one bias per output feature map
b_values = np.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv2d(
input=input,
filters=self.W
)
# pool each feature map individually, using maxpooling
# pooled_out = pool.pool_2d(
# input=conv_out,
# ds=poolsize,
# ignore_border=True
# )
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.tanh(conv_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
# keep track of model input
self.input = input
out = [out[i].flatten(2) for i in range(len(out))]
vid_ = T.concatenate(out, axis=1)
# vid_ = var_norm(vid_,axis=1)
# traject convolution
# ------------------------------------------------------------------------------
if use.trajconv:
t_conv = t.reshape(
(batch.micro * prod(traj_shape[1:-1]), 1, 1, traj_shape[-1]))
t_filt_sh = (1, 1, 1, trajconv.filter_size)
n_out = traj_shape[-1]
for i in xrange(trajconv.layers):
t_W.append(_shared(rng.normal(loc=0, scale=0.01, size=t_filt_sh)))
t_conv = conv2d(t_conv,
filters=t_W[-1],
filter_shape=t_filt_sh,
border_mode='valid')
n_out -= trajconv.filter_size - 1
t_b.append(_shared(ones((n_out, ), dtype=floatX) * 0.1))
t_conv = t_conv + t_b[-1].dimshuffle('x', 0)
t_conv = activation(t_conv)
conv_length = prod(traj_shape[1:-1]) * trajconv.res_shape
t_conv = t_conv.reshape((batch.micro, conv_length))
if trajconv.append:
traj_ = T.concatenate([t.flatten(2), t_conv.flatten(2)], axis=1)
else:
traj_ = t_conv.flatten(2)
n_in_MLP -= traj_size
n_in_MLP += conv_length
def __init__(self, input1, input2, input3, word_embeddings, batch_size,
sequence_len, embedding_size, filter_sizes, num_filters,
keep_prob):
rng = np.random.RandomState(23455)
self.params = []
lookup_table = theano.shared(word_embeddings)
self.params += [lookup_table]
#input1-问题, input2-正向答案, input3-负向答案
#将每个字替换成字向量
input_matrix1 = lookup_table[T.cast(input1.flatten(), dtype="int32")]
input_matrix2 = lookup_table[T.cast(input2.flatten(), dtype="int32")]
input_matrix3 = lookup_table[T.cast(input3.flatten(), dtype="int32")]
#CNN的输入是4维矩阵，这里只是增加了一个维度而已
input_x1 = input_matrix1.reshape(
(batch_size, 1, sequence_len, embedding_size))
input_x2 = input_matrix2.reshape(
(batch_size, 1, sequence_len, embedding_size))
input_x3 = input_matrix3.reshape(
(batch_size, 1, sequence_len, embedding_size))
#print(input_x1.shape.eval())
self.dbg_x1 = input_x1
outputs_1, outputs_2, outputs_3 = [], [], []
#设置多种大小的filter
for filter_size in filter_sizes:
#每种大小的filter的数量是num_filters
filter_shape = (num_filters, 1, filter_size, embedding_size)
image_shape = (batch_size, 1, sequence_len, embedding_size)
fan_in = np.prod(filter_shape[1:])
fan_out = filter_shape[0] * np.prod(filter_shape[2:])
W_bound = np.sqrt(6. / (fan_in + fan_out))
W = theano.shared(np.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
b_values = np.zeros((filter_shape[0], ),
dtype=theano.config.floatX)
b = theano.shared(value=b_values, borrow=True)
#卷积+max_pooling
conv_out = conv2d(input=input_x1,
filters=W,
filter_shape=filter_shape,
input_shape=image_shape)
#卷积后的向量的长度为ds
pooled_out = pool.pool_2d(input=conv_out,
ds=(sequence_len - filter_size + 1, 1),
ignore_border=True,
mode='max')
pooled_active = T.tanh(pooled_out + b.dimshuffle('x', 0, 'x', 'x'))
outputs_1.append(pooled_active)
conv_out = conv2d(input=input_x2,
filters=W,
filter_shape=filter_shape,
input_shape=image_shape)
pooled_out = pool.pool_2d(input=conv_out,
ds=(sequence_len - filter_size + 1, 1),
ignore_border=True,
mode='max')
pooled_active = T.tanh(pooled_out + b.dimshuffle('x', 0, 'x', 'x'))
outputs_2.append(pooled_active)
conv_out = conv2d(input=input_x3,
filters=W,
filter_shape=filter_shape,
input_shape=image_shape)
pooled_out = pool.pool_2d(input=conv_out,
ds=(sequence_len - filter_size + 1, 1),
ignore_border=True,
mode='max')
pooled_active = T.tanh(pooled_out + b.dimshuffle('x', 0, 'x', 'x'))
outputs_3.append(pooled_active)
self.params += [W, b]
self.dbg_conv_out = conv_out.shape
num_filters_total = num_filters * len(filter_sizes)
self.dbg_outputs_1 = outputs_1[0].shape
#每一个句子的语义表示向量的长度为num_filters_total
output_flat1 = T.reshape(T.concatenate(outputs_1, axis=1),
[batch_size, num_filters_total])
output_flat2 = T.reshape(T.concatenate(outputs_2, axis=1),
[batch_size, num_filters_total])
output_flat3 = T.reshape(T.concatenate(outputs_3, axis=1),
[batch_size, num_filters_total])
#dropout, keep_prob为1表示不进行dropout
output_drop1 = self._dropout(rng, output_flat1, keep_prob)
output_drop2 = self._dropout(rng, output_flat2, keep_prob)
output_drop3 = self._dropout(rng, output_flat3, keep_prob)
#计算问题和答案之前的向量夹角
#计算向量的长度
len1 = T.sqrt(T.sum(output_drop1 * output_drop1, axis=1))
len2 = T.sqrt(T.sum(output_drop2 * output_drop2, axis=1))
len3 = T.sqrt(T.sum(output_drop3 * output_drop3, axis=1))
#计算向量之间的夹角
cos12 = T.sum(output_drop1 * output_drop2, axis=1) / (len1 * len2)
self.cos12 = cos12
cos13 = T.sum(output_drop1 * output_drop3, axis=1) / (len1 * len3)
self.cos13 = cos13
zero = theano.shared(np.zeros(batch_size, dtype=theano.config.floatX),
borrow=True)
margin = theano.shared(np.full(batch_size,
0.05,
dtype=theano.config.floatX),
borrow=True)
#Loss损失函数
diff = T.cast(T.maximum(zero, margin - cos12 + cos13),
dtype=theano.config.floatX)
self.cost = T.sum(diff, acc_dtype=theano.config.floatX)
#mini-batch数据的准确率(如果正向答案和问题之间的cosine大于负向答案和问题的cosine，则认为正确，
#否则是错误的)
#Loss和Accuracy是用来评估训练中模型时候收敛的两个很重要的指标
self.accuracy = T.sum(T.cast(T.eq(zero, diff),
dtype='int32')) / float(batch_size)
self.W = theano.shared(
np.asarray(
rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX
),
borrow=True
)
# the bias is a 1D tensor -- one bias per output feature map
b_values = np.zeros((filter_shape[0],), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
conv_out = conv2d(
input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape
)
# pool each feature map individually, using maxpooling
pooled_out = pool.pool_2d(
input=conv_out,
ds=poolsize,
ignore_border=True
)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
def __init__(self, rng, input, filter_shape, image_shape, W=None, b=None):
"""
Allocate a convulutional layer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]))
if (W is None):
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(
low=-W_bound, high=W_bound, size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
else:
self.W = theano.shared(W)
if (b is None):
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0], ),
dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
else:
self.b = theano.shared(b)
# convolve input feature maps with filters
conv_out = conv2d(input=input,
filters=self.W,
filter_shape=filter_shape,
input_shape=image_shape)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.nnet.relu(conv_out +
self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
# keep track of model input
self.input = input
# L2_sqr for L2 regularization
self.L2_sqr = ((self.b**2).sum() + (self.W**2).sum())
