def forward(self, x):
if self.W.array is None:
self._initialize_params(x.shape[1])
# Standard call
y = super(VerifiableConvolution2D, self).forward(x)
if not (hasattr(x, 'lower') and hasattr(x, 'upper')):
return y
# Call with bounds
lower, upper = x.lower, x.upper
c = (lower + upper) / 2.
r = (upper - lower) / 2.
c = F.convolution_2d(c,
self.W,
b=self.b,
stride=self.stride,
pad=self.pad)
r = F.convolution_2d(r,
abs(self.W),
b=None,
stride=self.stride,
pad=self.pad)
y.lower = c - r
y.upper = c + r
if _DO_TEST:
self.xp.testing.assert_array_less(y.lower.array, y.array + _TEST_E)
self.xp.testing.assert_array_less(y.array, y.upper.array + _TEST_E)
return y
def test_forward_consistency(self, nobias=False):
x_cpu = chainer.Variable(self.x)
W_cpu = chainer.Variable(self.W)
b_cpu = None if nobias else chainer.Variable(self.b)
y_cpu = functions.convolution_2d(x_cpu,
W_cpu,
b_cpu,
stride=self.stride,
pad=self.pad,
use_cudnn=self.use_cudnn,
cover_all=self.cover_all)
x_gpu = chainer.Variable(cuda.to_gpu(self.x))
W_gpu = chainer.Variable(cuda.to_gpu(self.W))
b_gpu = None if nobias else chainer.Variable(cuda.to_gpu(self.b))
y_gpu = functions.convolution_2d(x_gpu,
W_gpu,
b_gpu,
stride=self.stride,
pad=self.pad,
use_cudnn=self.use_cudnn,
cover_all=self.cover_all)
testing.assert_allclose(y_cpu.data, y_gpu.data.get(),
**self.check_forward_options)
def test_forward_consistency(self, nobias=False):
x_cpu = chainer.Variable(self.x)
W_cpu = chainer.Variable(self.W)
b_cpu = None if nobias else chainer.Variable(self.b)
with chainer.using_config('cudnn_deterministic',
self.cudnn_deterministic):
y_cpu = F.convolution_2d(x_cpu,
W_cpu,
b_cpu,
stride=self.stride,
pad=self.pad,
cover_all=self.cover_all)
x_gpu = chainer.Variable(cuda.to_gpu(self.x))
W_gpu = chainer.Variable(cuda.to_gpu(self.W))
b_gpu = None if nobias else chainer.Variable(cuda.to_gpu(self.b))
with chainer.using_config('use_cudnn', self.use_cudnn):
with chainer.using_config('cudnn_deterministic',
self.cudnn_deterministic):
y_gpu = F.convolution_2d(x_gpu,
W_gpu,
b_gpu,
stride=self.stride,
pad=self.pad,
cover_all=self.cover_all)
testing.assert_allclose(y_cpu.data, y_gpu.data.get(),
**self.check_forward_options)
def __call__(self, x):
# ch: canvas, ref, prev_pen
pred = self.calc(x) # b, 1, w, h
shape = pred.shape
b, ch, h, w = shape
pred = F.reshape(pred, (b, -1))
# pred = F.softmax(pred)
pred = E.gumbel_softmax(pred, tau=self.tau)
pred = F.reshape(pred, (b, 1) + shape[2:])
self.current_pos = pred # pen position
# mx, my = np.meshgrid(np.arange(w), np.arange(h))
# bmx, bmy, pos = F.broadcast(
# mx.reshape((1, 1, h, w)),
# my.reshape((1, 1, h, w)),
# self.current_pos)
# px, py = pos*mx, pos*my
# prex, prey = np.sum(mx*x[:, 2, :, :]), np.sum(my*x[:, 2, :, :])
# dx = F.sqrt((F.sum(px)-prex)**2+(F.sum(py)-prey)**2)
mv_cost = F.sum(
0.5 * self.current_pos *
(F.convolution_2d(x[:, 2:3, :, :], self.move_cost, pad=2) + 4))
# mv_cost = 0.3*F.relu(dx-1.5)
# print(mv_cost.data)
draw = F.convolution_2d(pred, self.pen, pad=1) # pen stroke
strength, draw = F.broadcast(self.strength, draw[:, 0, :, :])
self.draw = strength * draw
canvas = x[:, 0, :, :] + self.draw
self.canvas = E.leaky_clip(canvas[0, :, :], 0., 1., leak=0.001)
ref = x[:, 1, :, :]
diff = F.sum((canvas - ref)**2)
self.loss = diff + mv_cost
return self.loss
def forward(self, x):
"""
h1 : (1, 64, 112, 112)
h2 : (1, 128, 56, 56)
h3 : (1, 256, 28, 28)
h4 : (1, 512, 14, 14)
h5 : (1, 512, 7, 7)
:param x:
:return:
"""
h = x
h = F.relu((self.conv1_1(h)))
h = F.relu((self.conv1_2(h)))
pool1 = F.max_pooling_2d(h, 2, stride=2)
h = F.relu((self.conv2_1(pool1)))
h = F.relu((self.conv2_2(h)))
pool2 = F.max_pooling_2d(h, 2, stride=2)
h = F.relu((self.conv3_1(pool2)))
h = F.relu((self.conv3_2(h)))
h = F.relu((self.conv3_3(h)))
pool3 = F.max_pooling_2d(h, 2, stride=2)
h = F.relu((self.conv4_1(pool3)))
h = F.relu((self.conv4_2(h)))
h = F.relu((self.conv4_3(h)))
pool4 = F.max_pooling_2d(h, 2, stride=2)
if self.texture:
h = {
'pool1': pool1,
'pool2': pool2,
'pool3': pool3,
'pool4': pool4
}[self.texture_layer]
if self.cbp:
h = F.convolution_2d(h, self.W1) * F.convolution_2d(h, self.W2)
h = global_average_pooling_2d(h)
if self.normalize:
h = power_normalize(h)
h = F.normalize(h)
h = self.fc8(F.dropout(h, 0.2))
return h
else:
b, ch, height, width = h.data.shape
h = F.reshape(h, (b, ch, width * height))
h = F.batch_matmul(h, h, transb=True) / self.xp.float32(
width * height)
h = self.fc8(F.dropout(h, 0.4))
return h
else:
h = F.relu((self.conv5_1(pool4)))
h = F.relu((self.conv5_2(h)))
h = F.relu((self.conv5_3(h)))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.dropout(F.relu(self.fc6(h)), ratio=0.5)
h = F.dropout(F.relu(self.fc7(h)), ratio=0.5)
h = self.fc8(h)
return h
def propdown(self, hid):
""" This function propagates the hidden units activation downwords to the visible units
:param hid: Variable Matrix(batch_size, out_channels, image_height_out, image_width_out) - given h_sample
:return: Variable Matrix(batch_size, in_channels, image_height, image_width) - probability for each visible units to be v_j = 1
"""
batch_size = hid.data.shape[0]
if self.real == 0:
W_flipped = F.swapaxes(CF.flip(self.conv.W, axes=(2, 3)), axis1=0, axis2=1)
pre_sigmoid_activation = F.convolution_2d(hid, W_flipped, self.conv.a, pad=self.ksize-1)
# F.matmul(hid, self.l.W) + F.broadcast_to(self.l.a, (batch_size, self.n_visible))
v_mean = F.sigmoid(pre_sigmoid_activation)
#print('W info ', self.conv.W.data.shape, 'W_flipped info ', W_flipped.data.shape)
#print('W info ', self.conv.W.data[3, 0, 2, 3], 'W_flipped info ', W_flipped.data[0, 3, 8, 7])
#print('W info ', self.conv.W.data[3, 0, 8, 7], 'W_flipped info ', W_flipped.data[0, 3, 2, 3])
#print('W info ', self.conv.W.data[19, 0, 4, 0], 'W_flipped info ', W_flipped.data[0, 19, 6, 10])
#print('pre_sigmoidactivation', F.sum(pre_sigmoid_activation).data)
#print('v_mean', v_mean.data.shape)
#print('v_mean sum', F.sum(v_mean).data)
#print('hid', hid.data.shape)
else:
# TODO: check
W_flipped = F.swapaxes(CF.flip(self.conv.W, axes=(2, 3)), axis1=0, axis2=1)
v_mean = F.convolution_2d(hid, W_flipped, self.conv.a, pad=self.ksize-1)
return v_mean
def test_forward_consistency(self, nobias=False):
x_cpu = chainer.Variable(self.x)
W_cpu = chainer.Variable(self.W)
b_cpu = None if nobias else chainer.Variable(self.b)
with chainer.using_config('cudnn_deterministic',
self.cudnn_deterministic):
y_cpu = F.convolution_2d(x_cpu,
W_cpu,
b_cpu,
stride=self.stride,
pad=self.pad,
cover_all=self.cover_all,
dilate=self.dilate,
group=self.group)
x_gpu = chainer.Variable(cuda.to_gpu(self.x))
W_gpu = chainer.Variable(cuda.to_gpu(self.W))
b_gpu = None if nobias else chainer.Variable(cuda.to_gpu(self.b))
with chainer.using_config('use_cudnn', self.use_cudnn):
with chainer.using_config('cudnn_deterministic',
self.cudnn_deterministic):
with chainer.using_config('autotune', self.autotune):
y_gpu = F.convolution_2d(x_gpu,
W_gpu,
b_gpu,
stride=self.stride,
pad=self.pad,
cover_all=self.cover_all,
dilate=self.dilate,
group=self.group)
testing.assert_allclose(y_cpu.data,
y_gpu.data.get(),
atol=5e-4,
rtol=5e-3)
def propdown(self, hid):
""" This function propagates the hidden units activation downwords to the visible units
:param hid: Variable Matrix(batch_size, out_channels, image_height_out, image_width_out) - given h_sample
:return: Variable Matrix(batch_size, in_channels, image_height, image_width) - probability for each visible units to be v_j = 1
"""
batch_size = hid.data.shape[0]
if self.real == 0:
W_flipped = F.swapaxes(CF.flip(self.conv.W, axes=(2, 3)), axis1=0, axis2=1)
pre_sigmoid_activation = F.convolution_2d(hid, W_flipped, self.conv.a, pad=self.ksize-1)
# F.matmul(hid, self.l.W) + F.broadcast_to(self.l.a, (batch_size, self.n_visible))
v_mean = F.sigmoid(pre_sigmoid_activation)
#print('W info ', self.conv.W.data.shape, 'W_flipped info ', W_flipped.data.shape)
#print('W info ', self.conv.W.data[3, 0, 2, 3], 'W_flipped info ', W_flipped.data[0, 3, 8, 7])
#print('W info ', self.conv.W.data[3, 0, 8, 7], 'W_flipped info ', W_flipped.data[0, 3, 2, 3])
#print('W info ', self.conv.W.data[19, 0, 4, 0], 'W_flipped info ', W_flipped.data[0, 19, 6, 10])
#print('pre_sigmoidactivation', F.sum(pre_sigmoid_activation).data)
#print('v_mean', v_mean.data.shape)
#print('v_mean sum', F.sum(v_mean).data)
#print('hid', hid.data.shape)
else:
# TODO: check
W_flipped = F.swapaxes(CF.flip(self.conv.W, axes=(2, 3)), axis1=0, axis2=1)
v_mean = F.convolution_2d(hid, W_flipped, self.conv.a, pad=self.ksize-1)
return v_mean
def forward_prop(x, local_time, sequence, isFirst, timestamp, satellite_name):
s = cp.empty([local_time, distance_forward, channels_hidden, M, N])
e = cp.empty([local_time, distance_forward])
alpha = cp.empty([local_time, distance_forward])
p = cp.empty([local_time, channels_p, M, N])
# Hidden Unit
h = cp.empty([local_time + 1, channels_hidden, M, N])
h[-1] = cp.zeros([channels_hidden, M, N])
# LSTM FORWARD PROPAGATION
for t in np.arange(local_time):
# Attention Network
for z in range(
timestamp + t - (distance + learning_window), timestamp +
distance_forward + t - (distance + learning_window)):
temp = cp.concatenate(
(cp.asarray(satellite_images[sequence][z]), h[t - 1]), axis=0)
s[t][z - (timestamp + t - (distance + learning_window))] = tanh(
cp.asarray(
F.convolution_2d(temp.reshape(
1, channels_img + channels_hidden, M, N),
e_kernel,
b=None,
pad=pad_constant)[0].data) + bias_e)
s_temp = s[t][z - (timestamp + t -
(distance + learning_window))].reshape(
M * N * channels_hidden)
e[t][z - (timestamp + t - (distance + learning_window))] = cp.dot(
v_connected_weights,
s_temp) + bias_v[z - (timestamp + t -
(distance + learning_window))]
xtemp = satellite_images[sequence][timestamp - distance:timestamp -
distance + distance_forward, 0]
alpha[t] = softmax(e[t])
p[t] = cp.tensordot(alpha[t], cp.asarray(xtemp), axes=1).reshape(
1, M, N) # Sum all x arrays up, weighted array
temporary = cp.concatenate((x[t], p[t], h[t - 1]), axis=0)
temporary = temporary.reshape(
1, channels_img + channels_p + channels_hidden, M, N)
h[t] = tanh(
cp.asarray(
F.convolution_2d(temporary, main_kernel, b=None, pad=2)
[0].data) + bias_h)
# 1 x 1 convolution
output = cp.matmul(connected_weights, h[local_time - 1].reshape(
channels_hidden, M * N)).reshape(M, N) + bias_y[0]
true_output = rect_linear(output)
return true_output, output, cp.reshape(
h[local_time - 1], (channels_hidden, M * N)), p, h, s, e, alpha, xtemp
def gradimg(img):
grad = xp.tile(
xp.asarray([[[[1, 0, -1], [2, 0, -2], [1, 0, -1]]]], dtype=img.dtype),
(img.array.shape[1], 1, 1))
dx = F.convolution_2d(img, grad)
dy = F.convolution_2d(img, xp.transpose(grad, (0, 1, 3, 2)))
return (F.sqrt(dx**2 + dy**2))
def compact_bilinear_pooling(x, randweight):
h = F.convolution_2d(x, randweight['W1']) * F.convolution_2d(
x, randweight['W2'])
h = global_average_pooling_2d(h)
h = power_normalize(h)
h = F.normalize(h)
return h
def _2d_ssim(img1,
img2,
window,
window_size,
channel,
data_range,
size_average=True):
mu1 = F.convolution_2d(img1, window, pad=window_size // 2, groups=channel)
mu2 = F.convolution_2d(img2, window, pad=window_size // 2, groups=channel)
mu1_sq = F.square(mu1)
mu2_sq = F.square(mu2)
mu1_mu2 = mu1 * mu2
sigma1_sq = F.convolution_2d(
img1 * img1, window, pad=window_size // 2, groups=channel) - mu1_sq
sigma2_sq = F.convolution_2d(
img2 * img2, window, pad=window_size // 2, groups=channel) - mu2_sq
sigma12 = F.convolution_2d(
img1 * img2, window, pad=window_size // 2, groups=channel) - mu1_mu2
C1 = (0.01 * data_range)**2
C2 = (0.03 * data_range)**2
ssim_map = ((2 * mu1_mu2 + C1) *
(2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) *
(sigma1_sq + sigma2_sq + C2))
if size_average:
return F.mean(ssim_map)
return NotImplementedError()
def dropout_convolution_2d(self, x):
train = configuration.config.train
W, b = self.W, self.b
log_alpha = VDF.calculate_log_alpha(self.W,
self.log_sigma2,
eps=1e-8,
thresholds=(-8., 8.))
clip_mask = (log_alpha.data > self.loga_threshold)
if train:
W = (1. - clip_mask) * W
mu = F.convolution_2d(x, (1. - clip_mask) * W,
b=None,
stride=self.stride,
pad=self.pad,
deterministic=self.deterministic)
si = F.sqrt(
F.convolution_2d(x * x,
F.exp(log_alpha) * W * W,
b=None,
stride=self.stride,
pad=self.pad,
deterministic=self.deterministic) + 1e-8)
normal_noise = self.xp.random.normal(0., 1., mu.shape).astype('f')
activation = mu + si * normal_noise
return F.bias(activation, b)
else:
return F.convolution_2d(x, (1. - clip_mask) * W,
b,
stride=self.stride,
pad=self.pad,
deterministic=self.deterministic)
def __call__(self, x0, x1, cs_map=False):
xp = backend.get_array_module(x0.data)
assert x0.shape[1] == 1, 'x0.shape[1] must be 1'
self.window = xp.asarray(self.window)
mu0 = F.convolution_2d(x0, self.window)
mu1 = F.convolution_2d(x1, self.window)
sigma00 = F.convolution_2d(x0 * x0, self.window)
sigma11 = F.convolution_2d(x1 * x1, self.window)
sigma01 = F.convolution_2d(x0 * x1, self.window)
mu00 = mu0 * mu0
mu11 = mu1 * mu1
mu01 = mu0 * mu1
sigma00 = sigma00 - mu00
sigma11 = sigma11 - mu11
sigma01 = sigma01 - mu01
v1 = 2 * sigma01 + self.c2
v2 = sigma00 + sigma11 + self.c2
if cs_map:
cs = v1 / v2
cs = F.mean(cs, axis=(1, 2, 3))
return cs
w1 = 2 * mu01 + self.c1
w2 = mu00 + mu11 + self.c1
ssim = (w1 * v1) / (w2 * v2)
ssim = F.mean(ssim, axis=(1, 2, 3))
return ssim
def test_forward(self):
""" Forward should work correctly """
fixup_conv2d = FixupConv2D(32, 32, ksize=3, pad=1)
x = chainer.Variable(np.random.random((1, 32, 4, 4)).astype('float32'))
y = fixup_conv2d(x)
z = F.relu(F.convolution_2d(x, fixup_conv2d.conv.W, pad=1))
self.assertTrue(np.allclose(y.array, z.array))
# setup bias_in
fixup_conv2d.bias_in.data = np.array([0.1], dtype=np.float32)
y = fixup_conv2d(x)
z = F.relu(F.convolution_2d(x + 0.1, fixup_conv2d.conv.W, pad=1))
self.assertTrue(np.allclose(y.array, z.array))
# bias_out
fixup_conv2d.bias_out.data = np.array([0.2], dtype=np.float32)
y = fixup_conv2d(x)
z = F.relu(F.convolution_2d(x + 0.1, fixup_conv2d.conv.W, pad=1) + 0.2)
self.assertTrue(np.allclose(y.array, z.array))
# scale
fixup_conv2d.scale.data = np.array([0.1], dtype=np.float32)
y = fixup_conv2d(x)
z = F.relu(
F.convolution_2d(x + 0.1, fixup_conv2d.conv.W, pad=1) * 0.1 + 0.2)
self.assertTrue(np.allclose(y.array, z.array))
def __call__(self, x):
x = x.transpose((0, 1, 3, 2))
real = F.convolution_2d(
x, self.weight_real, stride=(1, self.hop_length))
imag = F.convolution_2d(
x, self.weight_imag, stride=(1, self.hop_length))
return real, imag
def feat(self, h):
h = F.convolution_2d(h, self.W1) * F.convolution_2d(h, self.W2)
h = F.average_pooling_2d(h, 28, 28)
if self.l2normalize:
h = F.reshape(h, (h.data.shape[0], -1))
h = ssq(h)
h = F.normalize(h)
return h
def feat(self, h):
h = F.convolution_2d(h, self.W1) * F.convolution_2d(h, self.W2)
h = F.spatial_pyramid_pooling_2d(h, self.p, F.MaxPooling2D)
if self.l2normalize:
h = F.reshape(h, (h.data.shape[0], -1))
h = ssq(h)
h = F.normalize(h)
return h
def loss_grad_d(diff):
xp = cuda.get_array_module(diff.data)
grad = xp.tile(
xp.asarray([[[[1, 0, -1], [2, 0, -2], [1, 0, -1]]]], dtype=diff.dtype),
(diff.data.shape[1], 1, 1))
dx = F.convolution_2d(diff, grad)
dy = F.convolution_2d(diff, xp.transpose(grad, (0, 1, 3, 2)))
return F.average(dx**2) + F.average(dy**2)
def __call__(self, x):
x = F.average_pooling_2d(x, 2, 2, 0)
depth_smoothness = F.convolution_2d(x, self.diff)
depth_smoothness = F.sum(F.absolute(depth_smoothness), axis=1, keepdims=True)
edge = F.convolution_2d(x, self.laplacian)
loss = F.exp(-F.absolute(edge)) * depth_smoothness
return F.mean(loss)
def total_variation(x,tau=1e-6):
xp = cuda.get_array_module(x.data)
wh = xp.tile(xp.asarray([[[[1,0],[-1,0]]]], dtype=x.dtype),(x.data.shape[1],1,1))
ww = xp.tile(xp.asarray([[[[1, -1],[0, 0]]]], dtype=x.dtype),(x.data.shape[1],1,1))
dx = F.convolution_2d(x, W=wh)
dy = F.convolution_2d(x, W=ww)
d = F.sqrt(dx**2 + dy**2 + xp.full(dx.data.shape, tau**2, dtype=dx.dtype))
return(F.average(d))
def total_variation2(x):
xp = cuda.get_array_module(x.data)
wh = xp.asarray([[[[1], [-1]]]], dtype=x.dtype)
ww = xp.asarray([[[[1, -1]]]], dtype=x.dtype)
dx = F.convolution_2d(x, W=wh)
dy = F.convolution_2d(x, W=ww)
# dx = x[:, 1:, :, :] - x[:, :-1, :, :]
# dy = x[:, :, 1:, :] - x[:, :, :-1, :]
return F.average(F.absolute(dx)) + F.average(F.absolute(dy))
def loss_grad_d(diff):
xp = cuda.get_array_module(diff.data)
grad = xp.tile(
xp.asarray([[[[1, 0, -1], [2, 0, -2], [1, 0, -1]]]], dtype=diff.dtype),
(diff.data.shape[1], 1, 1))
dx = F.convolution_2d(diff, grad)
dy = F.convolution_2d(diff, xp.transpose(grad, (0, 1, 3, 2)))
# target = self.xp.zeros_like(dx.data)
# return 0.5*(F.mean_squared_error(dx,target)+F.mean_squared_error(dy,target))
return F.average(dx**2) + F.average(dy**2)
def transform(self, images):
images = self.process_input(images)
# images.shape == (n_images, n_channels, y, x)
filters_l1 = components_to_filters(
self.pca_l1.components_,
n_channels=images.shape[1],
filter_shape=self.filter_shape_l1,
)
filters_l2 = components_to_filters(self.pca_l2.components_,
n_channels=1,
filter_shape=self.filter_shape_l2)
# if gpu_enabled():
# images = to_gpu(images)
# filters_l1 = to_gpu(filters_l1)
# filters_l2 = to_gpu(filters_l2)
images = convolution_2d(images, filters_l1,
stride=self.step_shape_l1).data
images = xp.swapaxes(images, 0, 1)
# L1.shape == (L1, n_images, y, x)
# iterate over each L1 output
X = []
for maps in images:
n_images, h, w = maps.shape
maps = convolution_2d(
maps.reshape(n_images, 1, h, w), # 1 channel images
filters_l2,
stride=self.step_shape_l2).data
# maps.shape == (n_images, L2, y, x) right here
maps = binarize(maps)
maps = binary_to_decimal(maps)
# maps.shape == (n_images, y, x)
x = self.histogram(maps)
# x is a set of feature vectors.
# The shape of x is (n_images, vector length)
X.append(x)
# concatenate over L1
X = xp.hstack(X)
# if gpu_enabled():
# X = to_cpu(X)
X = X.astype(np.float64)
# The shape of X is (n_images, L1 * vector length)
return X
def local_patch(self, content, style_patch, style_patch_norm):
xp = cuda.get_array_module(content.data)
b,ch,h,w = content.data.shape
correlation = F.convolution_2d(Variable(content.data,volatile=True), W=style_patch_norm.data, stride=1, pad=0)
indices = xp.argmax(correlation.data, axis=1)
nearest_style_patch = style_patch.data.take(indices, axis=0).reshape(b,-1)
content = F.convolution_2d(content, W=Variable(xp.identity(ch*3*3,dtype=xp.float32).reshape((ch*3*3,ch,3,3))),stride=1,pad=0).transpose(0,2,3,1).reshape(b,-1)
style_loss = F.mean_squared_error(content, nearest_style_patch)
return style_loss
def loss_func_tv_l2(x_out):
xp = cuda.get_array_module(x_out.data)
b, ch, h, w = x_out.data.shape
Wx = xp.zeros((ch, ch, 2, 2), dtype="f")
Wy = xp.zeros((ch, ch, 2, 2), dtype="f")
for i in range(ch):
Wx[i, i, 0, 0] = -1
Wx[i, i, 0, 1] = 1
Wy[i, i, 0, 0] = -1
Wy[i, i, 1, 0] = 1
return F.sum(F.convolution_2d(x_out, W=Wx)**2) + F.sum(
F.convolution_2d(x_out, W=Wy)**2)
def tv_norm(self, x, Wh, Ww):
diffh = F.convolution_2d(F.reshape(x,
(3, 1, self.insize, self.insize)),
W=Wh,
pad=(1, 0))
diffw = F.convolution_2d(F.reshape(x,
(3, 1, self.insize, self.insize)),
W=Ww,
pad=(0, 1))
#tv = (F.sum(diffh**2) + F.sum(diffw**2))**(self.beta / 2.)
tv = F.sum((diffh**2 + diffw**2)**(self.beta / 2.))
return tv
def feat(self, h, train=True):
batchsize = h.data.shape[0]
h = F.convolution_2d(h, self.W1) * F.convolution_2d(h, self.W2)
h = F.batch_matmul(
F.reshape(h, (batchsize, pdim, 28 * 28)),
chainer.Variable(chainer.cuda.cupy.tile(self.loadedfilter,
(batchsize, 1, 1)),
volatile='off' if train else 'on'))
if self.l2normalize:
h = F.reshape(h, (h.data.shape[0], -1))
h = ssq(h)
h = F.normalize(h)
return h
def loss_grad(x, y, norm='l1'):
xp = cuda.get_array_module(x.data)
grad = xp.tile(
xp.asarray([[[[1, 0, -1], [2, 0, -2], [1, 0, -1]]]], dtype=x.dtype),
(x.data.shape[1], 1, 1))
dxx = F.convolution_2d(x, grad)
dyx = F.convolution_2d(y, grad)
dxy = F.convolution_2d(x, xp.transpose(grad, (0, 1, 3, 2)))
dyy = F.convolution_2d(y, xp.transpose(grad, (0, 1, 3, 2)))
if norm == 'l1':
return F.mean_absolute_error(dxx, dyx) + F.mean_squared_error(dxy, dyy)
else:
return F.mean_squared_error(dxx, dyx) + F.mean_squared_error(dxy, dyy)
def total_variation(x):
xp = cuda.get_array_module(x.data)
b, ch, h, w = x.data.shape
wh = xp.asarray([[[[1], [-1]], [[0], [0]], [[0], [0]]],
[[[0], [0]], [[1], [-1]], [[0], [0]]],
[[[0], [0]], [[0], [0]], [[1], [-1]]]],
dtype=np.float32)
ww = xp.asarray(
[[[[1, -1]], [[0, 0]], [[0, 0]]], [[[0, 0]], [[1, -1]], [[0, 0]]],
[[[0, 0]], [[0, 0]], [[1, -1]]]],
dtype=np.float32)
return F.sum(F.convolution_2d(x, W=wh)**2) + F.sum(
F.convolution_2d(x, W=ww)**2)
def _run_forward(self, x_data, W_data, b_data):
x = chainer.Variable(x_data)
W = chainer.Variable(W_data)
b = None if self.nobias else chainer.Variable(b_data)
y = F.convolution_2d(x, W, b, stride=self.stride, pad=self.pad,
cover_all=False, groups=self.groups)
return x, W, b, y
def compute_peaks_from_heatmaps(self, heatmaps):
keypoints = []
xp = cuda.get_array_module(heatmaps)
if xp == np:
for i in range(heatmaps.shape[0] - 1):
heatmap = gaussian_filter(heatmaps[i],
sigma=params['gaussian_sigma'])
max_value = heatmap.max()
if max_value > params['hand_heatmap_peak_thresh']:
coords = np.array(
np.where(heatmap == max_value)).flatten().tolist()
keypoints.append([coords[1], coords[0],
max_value]) # x, y, conf
else:
keypoints.append(None)
else:
heatmaps = F.convolution_2d(heatmaps[:, None],
self.gaussian_kernel,
stride=1,
pad=int(params['ksize'] /
2)).data.squeeze().get()
for heatmap in heatmaps[:-1]:
max_value = heatmap.max()
if max_value > params['hand_heatmap_peak_thresh']:
coords = np.array(
np.where(heatmap == max_value)).flatten().tolist()
keypoints.append([coords[1], coords[0],
max_value]) # x, y, conf
else:
keypoints.append(None)
return keypoints
def test_forward_consistency(self, nobias=False):
x_cpu = chainer.Variable(self.x)
W_cpu = chainer.Variable(self.W)
b_cpu = None if nobias else chainer.Variable(self.b)
y_cpu = functions.convolution_2d(
x_cpu, W_cpu, b_cpu, stride=self.stride, pad=self.pad,
use_cudnn=self.use_cudnn, cover_all=self.cover_all)
x_gpu = chainer.Variable(cuda.to_gpu(self.x))
W_gpu = chainer.Variable(cuda.to_gpu(self.W))
b_gpu = None if nobias else chainer.Variable(cuda.to_gpu(self.b))
y_gpu = functions.convolution_2d(
x_gpu, W_gpu, b_gpu, stride=self.stride, pad=self.pad,
use_cudnn=self.use_cudnn, cover_all=self.cover_all)
gradient_check.assert_allclose(y_cpu.data, y_gpu.data.get())
def compute_peaks_from_heatmaps(self, heatmaps):
"""all_peaks: shape = [N, 5], column = (jointtype, x, y, score, index)"""
heatmaps = heatmaps[:-1]
xp = cuda.get_array_module(heatmaps)
if xp == np:
all_peaks = []
peak_counter = 0
for i , heatmap in enumerate(heatmaps):
heatmap = gaussian_filter(heatmap, sigma=params['gaussian_sigma'])
map_left = xp.zeros(heatmap.shape)
map_right = xp.zeros(heatmap.shape)
map_top = xp.zeros(heatmap.shape)
map_bottom = xp.zeros(heatmap.shape)
map_left[1:, :] = heatmap[:-1, :]
map_right[:-1, :] = heatmap[1:, :]
map_top[:, 1:] = heatmap[:, :-1]
map_bottom[:, :-1] = heatmap[:, 1:]
peaks_binary = xp.logical_and.reduce((
heatmap > params['heatmap_peak_thresh'],
heatmap > map_left,
heatmap > map_right,
heatmap > map_top,
heatmap > map_bottom,
))
peaks = zip(xp.nonzero(peaks_binary)[1], xp.nonzero(peaks_binary)[0]) # [(x, y), (x, y)...]のpeak座標配列
peaks_with_score = [(i,) + peak_pos + (heatmap[peak_pos[1], peak_pos[0]],) for peak_pos in peaks]
peaks_id = range(peak_counter, peak_counter + len(peaks_with_score))
peaks_with_score_and_id = [peaks_with_score[i] + (peaks_id[i], ) for i in range(len(peaks_id))]
peak_counter += len(peaks_with_score_and_id)
all_peaks.append(peaks_with_score_and_id)
all_peaks = xp.array([peak for peaks_each_category in all_peaks for peak in peaks_each_category])
else:
heatmaps = F.convolution_2d(heatmaps[:, None], self.gaussian_kernel,
stride=1, pad=int(params['ksize']/2)).data.squeeze()
left_heatmaps = xp.zeros(heatmaps.shape)
right_heatmaps = xp.zeros(heatmaps.shape)
top_heatmaps = xp.zeros(heatmaps.shape)
bottom_heatmaps = xp.zeros(heatmaps.shape)
left_heatmaps[:, 1:, :] = heatmaps[:, :-1, :]
right_heatmaps[:, :-1, :] = heatmaps[:, 1:, :]
top_heatmaps[:, :, 1:] = heatmaps[:, :, :-1]
bottom_heatmaps[:, :, :-1] = heatmaps[:, :, 1:]
peaks_binary = xp.logical_and(heatmaps > params['heatmap_peak_thresh'], heatmaps >= right_heatmaps)
peaks_binary = xp.logical_and(peaks_binary, heatmaps >= top_heatmaps)
peaks_binary = xp.logical_and(peaks_binary, heatmaps >= left_heatmaps)
peaks_binary = xp.logical_and(peaks_binary, heatmaps >= bottom_heatmaps)
peak_c, peak_y, peak_x = xp.nonzero(peaks_binary)
peak_score = heatmaps[peak_c, peak_y, peak_x]
all_peaks = xp.vstack((peak_c, peak_x, peak_y, peak_score)).transpose()
all_peaks = xp.hstack((all_peaks, xp.arange(len(all_peaks)).reshape(-1, 1)))
all_peaks = all_peaks.get()
return all_peaks
def _run_forward(self, x_data, W_data, b_data):
x = chainer.Variable(x_data)
W = chainer.Variable(W_data)
b = None if self.nobias else chainer.Variable(b_data)
y = functions.convolution_2d(
x, W, b, stride=self.stride, pad=self.pad, use_cudnn=True,
cover_all=False, deterministic=True)
return x, W, b, y
def check_forward(self, x, offset, W, b, stride, pad):
with chainer.using_config('use_cudnn', self.use_cudnn):
x = chainer.Variable(x)
offset = chainer.Variable(offset)
out = deformable_convolution_2d_sampler(
x, offset, W, b, stride, pad).data
expeceted = convolution_2d(
x, W, b, stride, pad).data
testing.assert_allclose(out, expeceted)
def ordinal_loss(y, mask):
xp = cuda.get_array_module(y.data)
volatile = y.volatile
b, c, n = y.data.shape
max_y = F.broadcast_to(F.max(y, axis=1, keepdims=True), y.data.shape)
y = y - max_y
sum_y = F.broadcast_to(F.expand_dims(F.sum(y, axis=1), 1), y.data.shape)
down_tri = np.tri(c, dtype=np.float32)
up_tri = down_tri.T
w1 = Variable(xp.asarray(down_tri.reshape(c, c, 1, 1)), volatile=volatile)
w2 = Variable(xp.asarray(up_tri.reshape(c, c, 1, 1)), volatile=volatile)
h = F.exp(F.expand_dims(y, -1))
h1 = F.convolution_2d(h, w1)
h1 = F.convolution_2d(F.log(h1), w1)
h2 = F.convolution_2d(h, w2)
h2 = F.convolution_2d(F.log(h2), w2)
h = F.reshape(h1 + h2, (b, c, n))
return F.sum((h - sum_y - y) * mask) / b
def forward(self, inputs):
x, W, b = inputs
x = chainer.Variable(x)
W = chainer.Variable(W)
b = None if b is None else chainer.Variable(b)
return F.convolution_2d(
x, W, b, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate,
groups=self.groups)
def _hand_estimate_chainer_backend_each(self, hand_bgr, cx, cy, left_hand):
xp = self.hand_net.xp
device_id = self.hand_net._device_id
if left_hand:
hand_bgr = cv2.flip(hand_bgr, 1) # 1 = vertical
resized = cv2.resize(hand_bgr, (368, 368), interpolation=cv2.INTER_CUBIC)
x = np.array(resized[np.newaxis], dtype=np.float32)
x = x.transpose(0, 3, 1, 2)
x = x / 256 - 0.5
if self.gpu >= 0:
with chainer.cuda.get_device_from_id(device_id):
x = chainer.cuda.to_gpu(x)
x = chainer.Variable(x)
heatmaps = self.hand_net(x)
heatmaps = F.resize_images(heatmaps[-1], hand_bgr.shape[:2])[0]
if self.gpu >= 0:
heatmaps.to_cpu()
heatmaps = heatmaps.array
if left_hand:
heatmaps = heatmaps.transpose(1, 2, 0)
heatmaps = cv2.flip(heatmaps, 1)
heatmaps = heatmaps.transpose(2, 0, 1)
# get peak on heatmap
hmaps = []
if xp == np:
# cpu
for i in range(heatmaps.shape[0] - 1):
heatmap = gaussian_filter(heatmaps[i], sigma=self.hand_gaussian_sigma)
hmaps.append(heatmap)
else:
with chainer.cuda.get_device_from_id(device_id):
heatmaps = chainer.cuda.to_gpu(heatmaps)
heatmaps = F.convolution_2d(
heatmaps[:, xp.newaxis], self.hand_gaussian_kernel,
stride=1, pad=int(self.hand_gaussian_ksize / 2))
heatmaps = chainer.cuda.to_cpu(xp.squeeze(heatmaps.array))
for heatmap in heatmaps[:-1]:
hmaps.append(heatmap)
keypoints = []
idx_offset = 0
if left_hand:
idx_offset += len(hmaps)
for i, heatmap in enumerate(hmaps):
conf = heatmap.max()
cds = np.array(np.where(heatmap==conf)).flatten().tolist()
py = cy + cds[0] - hand_bgr.shape[0] / 2
px = cx + cds[1] - hand_bgr.shape[1] / 2
keypoints.append({'x': px, 'y': py, 'score': conf,
'limb': self.index2handname[idx_offset+i]})
return keypoints
def forward_cpu(self, inputs):
x, W, b = inputs
x_cpu = chainer.Variable(x)
W_cpu = chainer.Variable(W)
b_cpu = None if b is None else chainer.Variable(b)
with chainer.using_config('use_ideep', 'never'):
y_cpu = F.convolution_2d(
x_cpu, W_cpu, b_cpu, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate, group=self.group)
return y_cpu,
def test_1(self):
n_batches = 2
in_channels = 3
out_channels = 1 # important
x_shape = (n_batches, in_channels, 10, 10)
w_shape = (out_channels, in_channels, 3, 3)
x = numpy.ones(x_shape, numpy.float32)
w = numpy.ones(w_shape, numpy.float32)
y = F.convolution_2d(x, chainer.Variable(w))
z = F.sum(y)
z.backward()
def test_forward_consistency(self, nobias=False):
x_cpu = chainer.Variable(self.x)
W_cpu = chainer.Variable(self.W)
b_cpu = None if nobias else chainer.Variable(self.b)
with chainer.using_config('cudnn_deterministic',
self.cudnn_deterministic):
y_cpu = functions.convolution_2d(
x_cpu, W_cpu, b_cpu, stride=self.stride, pad=self.pad,
cover_all=self.cover_all)
x_gpu = chainer.Variable(cuda.to_gpu(self.x))
W_gpu = chainer.Variable(cuda.to_gpu(self.W))
b_gpu = None if nobias else chainer.Variable(cuda.to_gpu(self.b))
with chainer.using_config('use_cudnn', self.use_cudnn):
with chainer.using_config('cudnn_deterministic',
self.cudnn_deterministic):
y_gpu = functions.convolution_2d(
x_gpu, W_gpu, b_gpu, stride=self.stride, pad=self.pad,
cover_all=self.cover_all)
testing.assert_allclose(
y_cpu.data, y_gpu.data.get(), **self.check_forward_options)
def test_forward_consistency(self, nobias=False):
x_cpu = chainer.Variable(self.x)
W_cpu = chainer.Variable(self.W)
b_cpu = None if nobias else chainer.Variable(self.b)
with chainer.using_config('cudnn_deterministic',
self.cudnn_deterministic):
y_cpu = F.convolution_2d(
x_cpu, W_cpu, b_cpu, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate)
x_gpu = chainer.Variable(cuda.to_gpu(self.x))
W_gpu = chainer.Variable(cuda.to_gpu(self.W))
b_gpu = None if nobias else chainer.Variable(cuda.to_gpu(self.b))
with chainer.using_config('use_cudnn', self.use_cudnn):
with chainer.using_config('cudnn_deterministic',
self.cudnn_deterministic):
with chainer.using_config('autotune', self.autotune):
y_gpu = F.convolution_2d(
x_gpu, W_gpu, b_gpu, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate)
testing.assert_allclose(
y_cpu.data, y_gpu.data.get(), atol=5e-4, rtol=5e-3)
def check_forward_consistency_regression(self, nobias=False):
x = chainer.Variable(self.x)
W = chainer.Variable(self.W)
b = None if nobias else chainer.Variable(self.b)
y_nd = functions.convolution_nd(
x, W, b, stride=self.stride, pad=self.pad,
use_cudnn=False, cover_all=self.cover_all)
y_2d = functions.convolution_2d(
x, W, b, stride=self.stride, pad=self.pad,
use_cudnn=False, cover_all=self.cover_all)
testing.assert_allclose(
y_nd.data, y_2d.data, **self.check_forward_options)
def compute_peaks_from_heatmaps(self, heatmaps):
peak_counter = 0
all_peaks = []
xp = cuda.get_array_module(heatmaps)
if xp == np:
for i in range(heatmaps.shape[0] - 1):
heatmap = gaussian_filter(heatmaps[i], sigma=params['gaussian_sigma'])
map_left = xp.zeros(heatmap.shape)
map_right = xp.zeros(heatmap.shape)
map_top = xp.zeros(heatmap.shape)
map_bottom = xp.zeros(heatmap.shape)
map_left[1:, :] = heatmap[:-1, :]
map_right[:-1, :] = heatmap[1:, :]
map_top[:, 1:] = heatmap[:, :-1]
map_bottom[:, :-1] = heatmap[:, 1:]
peaks_binary = xp.logical_and.reduce((heatmap >= map_left, heatmap >= map_right, heatmap >= map_top, heatmap >= map_bottom, heatmap > params['heatmap_peak_thresh']))
peaks = zip(xp.nonzero(peaks_binary)[1], xp.nonzero(peaks_binary)[0]) # [(x, y), (x, y)...]のpeak座標配列
peaks_with_score = [peak_pos + (heatmap[peak_pos[1], peak_pos[0]],) for peak_pos in peaks] # [(x, y, score), (x, y, score)...]のpeak配列 scoreはheatmap上のscore
peaks_id = range(peak_counter, peak_counter + len(peaks_with_score))
peaks_with_score_and_id = [peaks_with_score[i] + (peaks_id[i], ) for i in range(len(peaks_id))] # [(x, y, score, id), (x, y, score, id)...]のpeak配列
peak_counter += len(peaks_with_score_and_id)
all_peaks.append(peaks_with_score_and_id)
else:
heatmaps = F.convolution_2d(heatmaps[:, None], self.gaussian_kernel, stride=1, pad=int(params['ksize']/2)).data.squeeze()
left_heatmaps = xp.zeros(heatmaps.shape)
right_heatmaps = xp.zeros(heatmaps.shape)
top_heatmaps = xp.zeros(heatmaps.shape)
bottom_heatmaps = xp.zeros(heatmaps.shape)
left_heatmaps[:, 1:, :] = heatmaps[:, :-1, :]
right_heatmaps[:, :-1, :] = heatmaps[:, 1:, :]
top_heatmaps[:, :, 1:] = heatmaps[:, :, :-1]
bottom_heatmaps[:, :, :-1] = heatmaps[:, :, 1:]
peaks_binary = xp.logical_and(heatmaps >= left_heatmaps, heatmaps >= right_heatmaps)
peaks_binary = xp.logical_and(peaks_binary, heatmaps >= top_heatmaps)
peaks_binary = xp.logical_and(peaks_binary, heatmaps >= bottom_heatmaps)
peaks_binary = xp.logical_and(peaks_binary, heatmaps >= params['heatmap_peak_thresh'])
for ch, peaks_binary_per_ch in enumerate(peaks_binary[:-1]):
heatmap = heatmaps[ch]
peaks = zip(xp.nonzero(peaks_binary_per_ch)[1], xp.nonzero(peaks_binary_per_ch)[0])
peaks_with_score = [peak_pos + (heatmap[peak_pos[1], peak_pos[0]],) for peak_pos in peaks] # [(x, y, score), (x, y, score)...]のpeak配列 scoreはheatmap上のscore
peaks_id = range(peak_counter, peak_counter + len(peaks_with_score))
peaks_with_score_and_id = np.array([peaks_with_score[i] + (peaks_id[i],) for i in range(len(peaks_id))], dtype=np.float32) # [(x, y, score, id), (x, y, score, id)...]のpeak配列
peak_counter += len(peaks_with_score_and_id)
all_peaks.append(peaks_with_score_and_id)
return all_peaks
def check_forward(self, x, offset, W, b, stride, pad):
with chainer.using_config('use_cudnn', self.use_cudnn):
_, _, h, w = x.shape
_, _, kh, kw = W.shape
offset[:, :kh * kw] = -1 * stride[1]
offset[:, kh * kw:] = 1 * stride[0]
x = chainer.Variable(x)
offset = chainer.Variable(offset)
out = deformable_convolution_2d_sampler(
x, offset, W, b, stride, pad).data
pad = (pad[0] + 1 * stride[0], pad[1] + 1 * stride[1])
expeceted = convolution_2d(
x, W, b, stride, pad).data
expeceted = expeceted[:, :, 2:, :-2]
testing.assert_allclose(out, expeceted)
def check_forward(self, x_data, W_data, b_data):
args1 = (x_data, W_data)
args2 = (x_data, W_data)
if b_data is not None:
args1 = args1 + (b_data,)
b_data = sum(numpy.split(b_data, W_data.shape[1]))
args2 = args2 + (b_data,)
y1 = functions.depthwise_convolution_2d(
*args1, stride=self.stride, pad=self.pad)
arys = numpy.split(y1.array, self.W.shape[1], axis=1)
y1 = sum(arys)
y2 = functions.convolution_2d(
*args2, stride=self.stride, pad=self.pad).array
testing.assert_allclose(y1, y2, **self.check_forward_options)
def __call__(self, x):
# Apply a mask to the filters (optional)
if self.filter_mask is not None:
w, m = F.broadcast(self.W, Variable(self.filter_mask))
w = w * m
# w = self.W * Variable(self.filter_mask)
else:
w = self.W
# Transform the filters
# w.shape == (out_channels, in_channels, input_stabilizer_size, ksize, ksize)
# tw.shape == (out_channels, output_stabilizer_size, in_channels, input_stabilizer_size, ksize, ksize)
tw = TransformGFilter(self.inds)(w)
# Fold the transformed filters
tw_shape = (self.out_channels * self.output_stabilizer_size,
self.in_channels * self.input_stabilizer_size,
self.ksize, self.ksize)
tw = F.Reshape(tw_shape)(tw)
# If flat_channels is False, we need to flatten the input feature maps to have a single 1d feature dimension.
if not self.flat_channels:
batch_size = x.data.shape[0]
in_ny, in_nx = x.data.shape[-2:]
x = F.reshape(x, (batch_size, self.in_channels * self.input_stabilizer_size, in_ny, in_nx))
# Perform the 2D convolution
y = F.convolution_2d(x, tw, b=None, stride=self.stride, pad=self.pad, use_cudnn=self.use_cudnn)
# Unfold the output feature maps
# We do this even if flat_channels is True, because we need to add the same bias to each G-feature map
batch_size, _, ny_out, nx_out = y.data.shape
y = F.reshape(y, (batch_size, self.out_channels, self.output_stabilizer_size, ny_out, nx_out))
# Add a bias to each G-feature map
if self.usebias:
bb = F.Reshape((1, self.out_channels, 1, 1, 1))(self.b)
y, b = F.broadcast(y, bb)
y = y + b
# Flatten feature channels if needed
if self.flat_channels:
n, nc, ng, nx, ny = y.data.shape
y = F.reshape(y, (n, nc * ng, nx, ny))
return y
def check_forward_consistency_regression(self, nobias=False):
x = chainer.Variable(self.x)
W = chainer.Variable(self.W)
b = None if nobias else chainer.Variable(self.b)
with chainer.using_config('use_cudnn', 'never'):
y_nd = F.convolution_nd(
x, W, b, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate,
groups=self.groups)
y_2d = F.convolution_2d(
x, W, b, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate,
groups=self.groups)
testing.assert_allclose(
y_nd.data, y_2d.data, **self.check_forward_options)
def check_forward(self, inputs, backend_config):
y_expected, = self.forward_cpu(inputs)
if backend_config.use_cuda:
inputs = cuda.to_gpu(inputs)
x, W, b = inputs
x = chainer.Variable(x)
W = chainer.Variable(W)
b = None if b is None else chainer.Variable(b)
with backend_config:
y_actual = F.convolution_2d(
x, W, b, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate, group=self.group)
testing.assert_allclose(
y_expected.data, y_actual.data, atol=5e-4, rtol=5e-3)
def __call__(self, h, train=True):
"""
in_type:
h: float32
in_shape:
h: (batch_size, hidden_num)
out_type: float32
out_shape: (batch_size, rating_num, predicted_item_num)
"""
xp = cuda.get_array_module(h.data)
h = self.p(h)
if hasattr(self, 'q'):
h = self.q(h)
h = F.reshape(h, (-1, self.rating_num, self.item_num, 1))
w = chainer.Variable(xp.asarray(np.tri(self.rating_num, dtype=np.float32).reshape(self.rating_num, self.rating_num, 1, 1)), volatile=h.volatile)
h = F.convolution_2d(h, w)
return F.reshape(h, (-1, self.rating_num, self.item_num))
def reconstruct(self, v):
"""
:param v: Variable Matrix(batch_size, in_channels, image_height, image_width)
:return: reconstructed_v, Variable Matrix(batch_size, in_channels, image_height, image_width)
"""
batch_size = v.data.shape[0]
xp = cuda.get_array_module(v.data)
if self.real == 0:
h = F.sigmoid(self.conv(v))
else:
std_ch = xp.reshape(self.std, (1, self.in_channels, 1, 1))
h = F.sigmoid(self.conv(v / std_ch))
# F.sigmoid(F.matmul(v, self.l.W, transb=True) + F.broadcast_to(self.l.b, (batch_size, self.n_hidden)))
W_flipped = F.swapaxes(CF.flip(self.conv.W, axes=(2, 3)), axis1=0, axis2=1)
reconstructed_v = F.sigmoid(F.convolution_2d(h, W_flipped, self.conv.a, pad=self.ksize-1))
# = F.sigmoid(F.matmul(h, self.l.W) + F.broadcast_to(self.l.a, (batch_size, self.n_visible)))
return reconstructed_v
def check_backward(self, x_data, W_data, b_data, y_grad):
x = chainer.Variable(x_data)
W = chainer.Variable(W_data)
b = chainer.Variable(b_data)
y = functions.convolution_2d(x, W, b, stride=self.stride, pad=self.pad,
use_cudnn=self.use_cudnn)
y.grad = y_grad
y.backward()
func = y.creator
f = lambda: func.forward((x.data, W.data, b.data))
gx, gW, gb = gradient_check.numerical_grad(
f, (x.data, W.data, b.data), (y.grad,), eps=1e-2)
gradient_check.assert_allclose(gx, x.grad)
gradient_check.assert_allclose(gW, W.grad)
gradient_check.assert_allclose(gb, b.grad)
def compute_peaks_from_heatmaps(self, heatmaps):
keypoints = []
xp = cuda.get_array_module(heatmaps)
if xp == np:
for i in range(heatmaps.shape[0] - 1):
heatmap = gaussian_filter(heatmaps[i], sigma=params['gaussian_sigma'])
max_value = heatmap.max()
if max_value > params['face_heatmap_peak_thresh']:
coords = np.array(np.where(heatmap==max_value)).flatten().tolist()
keypoints.append([coords[1], coords[0], max_value]) # x, y, conf
else:
keypoints.append(None)
else:
heatmaps = F.convolution_2d(heatmaps[:, None], self.gaussian_kernel, stride=1, pad=int(params['ksize']/2)).data.squeeze().get()
for heatmap in heatmaps[:-1]:
max_value = heatmap.max()
if max_value > params['face_heatmap_peak_thresh']:
coords = np.array(np.where(heatmap==max_value)).flatten().tolist()
keypoints.append([coords[1], coords[0], max_value]) # x, y, conf
else:
keypoints.append(None)
return keypoints
def check_forward_consistency_regression(self, backend_config):
inputs = self.generate_inputs()
if self.nobias:
x, W = inputs
b = None
else:
x, W, b = inputs
x = chainer.Variable(backend_config.get_array(x))
W = chainer.Variable(backend_config.get_array(W))
if b is not None:
b = chainer.Variable(backend_config.get_array(b))
with chainer.using_config('use_cudnn', 'never'):
y_nd = F.convolution_nd(
x, W, b, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate,
groups=self.groups)
y_2d = F.convolution_2d(
x, W, b, stride=self.stride, pad=self.pad,
cover_all=self.cover_all, dilate=self.dilate,
groups=self.groups)
testing.assert_allclose(
y_nd.array, y_2d.array, **self.check_forward_options)
def forward(self):
x = chainer.Variable(self.x)
W = chainer.Variable(self.W)
return functions.convolution_2d(
x, W, None, stride=self.stride, pad=self.pad)
import sys
import chainer
import chainer.functions as F
if len(sys.argv) > 1:
chainer.config.use_cudnn = sys.argv[1]
xp = cupy
bs = 32
ch = 1024
x = xp.random.rand(bs, ch, 7, 7)
w = xp.random.rand(ch, 1, 3, 3)
assert (bs, ch, 7, 7) == F.convolution_2d(x, w, pad=1, groups=ch).shape
start = time.time()
count = 0
while True:
F.convolution_2d(x, w, pad=1, groups=ch)
cupy.cuda.device.Device().synchronize()
count += 1
now = time.time()
if now > start + 1:
break
print('%.3f msec (%d times)' % ((now - start) / count * 1000, count))
def forward(self):
x = chainer.Variable(self.x)
W = chainer.Variable(self.W)
return F.convolution_2d(x, W, None, stride=self.stride, pad=self.pad,
dilate=self.dilate, groups=self.groups)
def forward(self):
x = chainer.Variable(self.x)
W = chainer.Variable(self.W)
return functions.convolution_2d(
x, W, None, stride=self.stride, pad=self.pad,
use_cudnn=self.use_cudnn, cover_all=self.cover_all)
def tvw(self, x):
return F.convolution_2d(x, W=self.Ww)
