def __call__(self, x):
h = x
h = F.space2depth(h, 2)
h = self.conv(h)
h = self.b0(h)
h = F.space2depth(h, 2)
h = self.b1(h)
z1, h = F.split_axis(h, axis=1, indices_or_sections=[self.c1])
h = F.space2depth(h, 2)
h = self.b2(h)
z2, h = F.split_axis(h, axis=1, indices_or_sections=[self.c2])
h = F.space2depth(h, 2)
h = self.b3(h)
z3, h = F.split_axis(h, axis=1, indices_or_sections=[self.c3])
h = F.space2depth(h, 2)
z4 = self.b4(h)
z1 = self.bn1(z1)
z1 = F.space2depth(z1, 8)
z2 = self.bn2(z2)
z2 = F.space2depth(z2, 4)
z3 = self.bn3(z3)
z3 = F.space2depth(z3, 2)
z4 = self.bn4(z4)
z = F.concat([z1, z2, z3, z4], axis=1)
return z
def __call__(self, x):
h = F.leaky_relu(self.bias1(self.bn1(self.conv1(x))), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias2(self.bn2(self.conv2(h))), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias3(self.bn3(self.conv3(h))), slope=0.1)
h = F.leaky_relu(self.bias4(self.bn4(self.conv4(h))), slope=0.1)
h = F.leaky_relu(self.bias5(self.bn5(self.conv5(h))), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias6(self.bn6(self.conv6(h))), slope=0.1)
h = F.leaky_relu(self.bias7(self.bn7(self.conv7(h))), slope=0.1)
h = F.leaky_relu(self.bias8(self.bn8(self.conv8(h))), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias9(self.bn9(self.conv9(h))), slope=0.1)
h = F.leaky_relu(self.bias10(self.bn10(self.conv10(h))), slope=0.1)
h = F.leaky_relu(self.bias11(self.bn11(self.conv11(h))), slope=0.1)
h = F.leaky_relu(self.bias12(self.bn12(self.conv12(h))), slope=0.1)
h = F.leaky_relu(self.bias13(self.bn13(self.conv13(h))), slope=0.1)
high_resolution_feature = F.space2depth(h, 2)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias14(self.bn14(self.conv14(h))), slope=0.1)
h = F.leaky_relu(self.bias15(self.bn15(self.conv15(h))), slope=0.1)
h = F.leaky_relu(self.bias16(self.bn16(self.conv16(h))), slope=0.1)
h = F.leaky_relu(self.bias17(self.bn17(self.conv17(h))), slope=0.1)
h = F.leaky_relu(self.bias18(self.bn18(self.conv18(h))), slope=0.1)
h = F.leaky_relu(self.bias19(self.bn19(self.conv19(h))), slope=0.1)
h = F.leaky_relu(self.bias20(self.bn20(self.conv20(h))), slope=0.1)
h = F.concat((high_resolution_feature, h),
axis=1) # output concatenation
h = F.leaky_relu(self.bias21(self.bn21(self.conv21(h))), slope=0.1)
h = self.bias22(self.conv22(h))
return h
def setup(self):
xp = self.xp
x = xp.random.randn(128, 20, 60, 40).astype(xp.float32)
gy = xp.random.randn(128, 80, 30, 20).astype(xp.float32)
r = 2
func = lambda x: F.space2depth(x, r)
self.setup_benchmark(func, (x, ), gy)
def __call__(self, color, spec):
color_res1 = self.color_head(color)
color = self.color_body_pre(color_res1)
spec_res1 = self.spec_head(spec)
spec = self.spec_body_pre(spec_res1)
if self.fix_params_before_bridges:
spec.unchain_backward()
color.unchain_backward()
if self.connected:
color_tmp = self.spec_to_color_bridge(F.depth2space(spec, 2))
spec_tmp = self.color_to_spec_bridge(F.space2depth(color, 2))
color += color_tmp
spec += spec_tmp
color = self.color_body_post(color)
spec = self.spec_body_post(spec)
color += color_res1
spec += spec_res1
color = self.color_tail(color)
spec = self.spec_tail(spec)
return color, spec
def check_forward(self, space_data, depth_data):
space = chainer.Variable(space_data)
s2d = functions.space2depth(space, self.r)
s2d_value = cuda.to_cpu(s2d.data)
self.assertEqual(s2d_value.dtype, self.dtype)
self.assertEqual(s2d_value.shape, (2, 8, 3, 2))
s2d_expect = depth_data
testing.assert_allclose(s2d_value, s2d_expect)
def check_forward(self, space_data, depth_data):
space = chainer.Variable(space_data)
s2d = functions.space2depth(space, self.r)
s2d_value = cuda.to_cpu(s2d.data)
self.assertEqual(s2d_value.dtype, self.dtype)
self.assertEqual(s2d_value.shape, (2, 8, 3, 2))
s2d_expect = depth_data
testing.assert_allclose(s2d_value, s2d_expect)
def check_backward(self, random_array, random_grad_array):
x = chainer.Variable(random_array)
y = functions.space2depth(x, 2)
y.grad = random_grad_array
y.backward()
def func():
return (functions.space2depth(x, 2).data, )
gx, = gradient_check.numerical_grad(func, (x.data, ), (y.grad, ))
testing.assert_allclose(x.grad, gx, rtol=0.0001)
def generate_train_data(in_img):
# random cropping
trim_size = INPUT_SIZE if SR_METHOD == "pe" else INPUT_SIZE * 2
if COLOR_SHIFT:
trim_size += 2
v = random.randint(0, in_img.shape[0] - trim_size)
h = random.randint(0, in_img.shape[1] - trim_size)
sv = in_img[v:v+trim_size, h:h+trim_size].copy()
# random flip/rotation augmentation
if random.randint(0, 1) == 1:
sv = sv[:, ::-1]
if random.randint(0, 1) == 1:
sv = sv[::-1, :]
if random.randint(0, 1) == 1:
sv = np.rot90(sv)
# HLS color shift augmentation
if COLOR_SHIFT:
sv = sv.astype(np.float32)
sv[:, :, 0] = (sv[:, :, 0] + np.random.randn() * 32.) % 180.
sv[:, :, 1] = sv[:, :, 1] + np.random.randn() * 32.
sv[:, :, 2] = sv[:, :, 2] + sv[:, :, 2] * np.random.randn() * 0.25
sv = np.clip(sv, 0., 255.).astype(np.uint8)
sv = cv2.cvtColor(sv, cv2.COLOR_HLS2RGB)[1:-1, 1:-1]
# creates input and supervisory images (JPEG compression augmentation)
tr = uint8_to_float(prepare_input_img(sv, JPEG_COMP))
sv = uint8_to_float(sv.transpose(2, 0, 1))[np.newaxis, :, :, :]
if SR_METHOD == "sp":
sv = F.space2depth(sv, 2).data
elif SR_METHOD == "fp":
sv = sv[:, :, ::2, ::2]
if DEBUG:
show_img(float_to_uint8(tr[0, 0:3].transpose(1, 2, 0)), "train_batch")
show_img(float_to_uint8(sv[0, 0:3].transpose(1, 2, 0)),
"sv_batch", 1.5)
# reshaping and type conversion
return tr, sv
def f(x):
y = functions.space2depth(x, self.r)
return y * y
def f(x):
return functions.space2depth(x, self.r)
def f(x):
return functions.space2depth(x, self.r)
def fixed_decode(self, loop=3, filename='test.bin'):
xp = self.xp
decoder = RC.Decoder(filename)
im_h = decoder.height
im_w = decoder.width
self.im_h = im_h
self.im_w = im_w
h_h = math.ceil(im_h / (2**(loop + self.lossy_shrink)))
h_w = math.ceil(im_w / (2**(loop + self.lossy_shrink)))
decoder = RC.Decoder(filename)
p_init = xp.tile(F.softmax(self.d_pred, axis=1).data, (h_h * h_w, 1))
t_init = xp.array(decoder.call(cuda.to_cpu(p_init)), dtype=np.float32)
h = t_init.reshape((1, h_h, h_w, self.ct4)).transpose((0, 3, 1, 2))
# print('df:\n{}'.format(h[0, 0, :5, :5]))
for i in range(loop):
h_h = math.ceil(im_h / (2**(loop + self.lossy_shrink - i - 1)))
h_w = math.ceil(im_w / (2**(loop + self.lossy_shrink - i - 1)))
h = self.single_scale_decode(
[self.b32_0[loop - i - 1], self.b32_1[loop - i - 1]],
decoder,
h,
out_shape=(1, self.ct4, h_h, h_w))
# print('d32_{}:\n{}'.format(loop-i-1, h[0, 0, :5, :5]))
if self.ct3 != 0:
h_h = math.ceil(im_h / (2**(self.lossy_shrink - 1)))
h_w = math.ceil(im_w / (2**(self.lossy_shrink - 1)))
t = self.single_scale_decode([self.b16_0, self.b16_1],
decoder,
h,
out_shape=(1, self.ct3, h_h, h_w))
ts = xp.pad(t, ((0, 0), (0, 0), (0, t.shape[2] % 2),
(0, t.shape[3] % 2)),
mode='constant')
# print('d16:\n{}'.format(ts[0, 0, :5, :5]))
if self.ct2 == 0:
h = xp.concatenate(
(F.space2depth(ts, 2).data, h[:, self.ct3:]), axis=1)
else:
h = xp.concatenate(
(ts, M.unpooling_2d(h[:, self.ct3:], 2).data), axis=1)
if self.ct2 != 0:
h_h = math.ceil(im_h / (2**(self.lossy_shrink - 2)))
h_w = math.ceil(im_w / (2**(self.lossy_shrink - 2)))
t = self.single_scale_decode([self.b8_0, self.b8_1],
decoder,
h,
out_shape=(1, self.ct2, h_h, h_w))
ts = xp.pad(t, ((0, 0), (0, 0), (0, t.shape[2] % 2),
(0, t.shape[3] % 2)),
mode='constant')
# print('d8:\n{}'.format(ts[0, 0, :5, :5]))
if self.ct1 == 0:
h = xp.concatenate(
(F.space2depth(ts, 4).data,
F.space2depth(h[:, self.ct2:self.ct3],
2).data, h[:, self.ct3:, ::2, ::2]),
axis=1)
else:
h = xp.concatenate(
(ts, M.unpooling_2d(h[:, self.ct2:], 2).data), axis=1)
if self.ct1 != 0:
h_h = math.ceil(im_h / (2**(self.lossy_shrink - 3)))
h_w = math.ceil(im_w / (2**(self.lossy_shrink - 3)))
t = self.single_scale_decode([self.b4_0, self.b4_1],
decoder,
h,
out_shape=(1, self.ct1, h_h, h_w))
ts = xp.pad(t, ((0, 0), (0, 0), (0, t.shape[2] % 2),
(0, t.shape[3] % 2)),
mode='constant')
# print('d4:\n{}'.format(ts[0, 0, :5, :5]))
h = xp.concatenate(
(F.space2depth(ts, 8).data,
F.space2depth(h[:, self.ct1:self.ct2], 4).data,
F.space2depth(h[:, self.ct2:self.ct3, ::2, ::2],
2).data, h[:, self.ct3:, ::4, ::4]),
axis=1)
# print('init:\n{}'.format(h[0, 0, :5, :5]))
decoder.finish()
del decoder
return h
def f(x):
y = functions.space2depth(x, self.r)
return y * y
def downsample(self, x):
return cf.space2depth(x, r=self.scale)
def func(x):
return F.space2depth(x, r)
def func():
return (functions.space2depth(x, 2).data, )
