def network(self, LR):
#180413 - residual connection
#bicubic = tf.image.resize_images(LR[:,:,:,:,int((self.num_input_frames-1)/2)], size=[LR.shape[1]*self.scale, LR.shape[2]*self.scale], method=tf.image.ResizeMethod.BICUBIC)
#print(LR.shape)
LR = tf.reshape(LR, [LR.shape[0], LR.shape[1], LR.shape[2], LR.shape[3]* LR.shape[4]])
#print(LR.shape)
feature_tmp = tf.layers.conv2d(LR, 24, 3, strides = 1, padding = 'SAME', name = 'CONV_1',
kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
feature_tmp = tf.nn.relu(feature_tmp)
feature_tmp = tf.layers.conv2d(feature_tmp, 24, 3, strides = 1, padding = 'SAME', name = 'CONV_2',
kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
feature_tmp = tf.nn.relu(feature_tmp)
feature_tmp = tf.layers.conv2d(feature_tmp, 24, 3, strides = 1, padding = 'SAME', name = 'CONV_3',
kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
feature_tmp = tf.nn.relu(feature_tmp)
feature_tmp = tf.layers.conv2d(feature_tmp, 24, 3, strides = 1, padding = 'SAME', name = 'CONV_4',
kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
feature_tmp = tf.nn.relu(feature_tmp)
feature_out = tf.layers.conv2d(feature_tmp, self.channels*self.scale*self.scale, 3, strides = 1, padding = 'SAME',
name = 'CONV_5', kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
#180415-maybe?
feature_out = tf.nn.relu(feature_out)
if self.mode == "RGB":
feature_out = PS(feature_out, self.scale, color=True)
feature_out = tf.layers.conv2d(feature_out, 3, 1, strides = 1, padding = 'SAME',
name = 'CONV_OUT', kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
return feature_out
#return tf.add(feature_out, bicubic)
else:
feature_out = PS(feature_out, self.scale, color=False)
feature_out = tf.layers.conv2d(feature_out, 1, 1, strides = 1, padding = 'SAME',
name = 'CONV_OUT', kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
return feature_out
def inference_resnet(self):
input_img = self.input_placeholder
conv1 = conv2d(input_img, 3, 6, 5, 1)
x = tf.nn.elu(conv1)
conv2 = conv2d(x, 6, 12, 5, 1)
x = tf.nn.elu(conv2)
conv3 = conv2d(x, 12, 24, 3, 1)
x = tf.nn.elu(conv3)
conv4 = conv2d(x, 24, 48, 3, 1)
x = tf.nn.elu(conv4)
conv5 = conv2d(x, 48, 64, 3, 1)
x = tf.nn.elu(conv5)
for i in range(32):
with tf.name_scope('enhanced_residual_net_%d' % i):
x = self.enhanced_residual_cell(x, 0.1, 64, 64, 3, 1)
x += conv5
conv6 = conv2d(x, 64, 48, 3, 1)
x = tf.nn.elu(conv6 + conv4)
conv7 = conv2d(x, 48, 24, 3, 1)
x = tf.nn.elu(conv7 + conv3)
conv8 = conv2d(x, 24, 12, 3, 1)
x = tf.nn.elu(conv8 + conv2)
conv9 = conv2d(x, 12, 12, 3, 1)
x = tf.nn.elu(conv9)
conv10 = conv2d(x, 12, 12, 3, 1)
middle_feature_maps = conv10
# middle_feature_maps = conv2d(x, 6, 12, 3, 1)
# pred = tf.nn.tanh(PS(elu_middle, 2, color=True))
pred = tf.nn.sigmoid(PS(middle_feature_maps, 2, color=True))
return pred
def generator(self, z):
# project `z` and reshape
self.h0, self.h0_w, self.h0_b = deconv2d(
z, [self.batch_size, 32, 32, self.gf_dim],
k_h=1,
k_w=1,
d_h=1,
d_w=1,
name='g_h0',
with_w=True)
h0 = lrelu(self.h0)
self.h1, self.h1_w, self.h1_b = deconv2d(
h0, [self.batch_size, 32, 32, self.gf_dim],
name='g_h1',
d_h=1,
d_w=1,
with_w=True)
h1 = lrelu(self.h1)
h2, self.h2_w, self.h2_b = deconv2d(h1,
[self.batch_size, 32, 32, 3 * 16],
d_h=1,
d_w=1,
name='g_h2',
with_w=True)
h2 = PS(h2, 4, color=True)
return tf.nn.tanh(h2)
def inference_edsr(self):
input_img = self.input_placeholder
conv1 = conv2d(input_img, 3, 6, 5, 1)
x = tf.nn.elu(conv1)
conv2 = conv2d(x, 6, 12, 5, 1)
x = tf.nn.elu(conv2)
conv3 = conv2d(x, 12, 36, 3, 1)
x = tf.nn.elu(conv3)
conv4 = conv2d(x, 36, 64, 3, 1)
x = tf.nn.elu(conv4)
for i in range(32):
with tf.name_scope('enhanced_residual_net_%d' % i):
x = self.enhanced_residual_cell(x, 0.1, 64, 64, 3, 1)
x += conv4
conv5 = conv2d(x, 64, 36, 3, 1)
x = tf.nn.elu(conv5 + conv3)
conv6 = conv2d(x, 36, 12, 3, 1)
x = tf.nn.elu(conv6 + conv2)
conv9 = conv2d(x, 12, 12, 3, 1)
middle_feature_maps = conv9
# middle_feature_maps = conv2d(x, 6, 12, 3, 1)
# pred = tf.nn.tanh(PS(elu_middle, 2, color=True))
shuffled_img = PS(middle_feature_maps, 2, color=True)
conv10 = conv2d(shuffled_img, 3, 3, 3, 1)
pred = tf.nn.sigmoid(conv10)
return pred
def _write_no_attention(self, h):
scope = "write"
_h,_w,_c = self.h, self.w, self.c
if self.is_3d:
IS_SIMPLE_WRITE = True
if IS_SIMPLE_WRITE :
print('IS_SIMPLE_WRITE:', IS_SIMPLE_WRITE)
return tf.reshape( self.ls.dense(scope, h, _h*_w*_c, tf.nn.elu), [-1, _h, _w, _c])
else:
IS_CONV_LSTM = True
if IS_CONV_LSTM :
raise NotImplementedError
else:
activation = tf.nn.elu
print('h in write:', h) # h.shape is (_b, RNN_SIZES[0])
L = 1
h = tf.reshape( h, (-1, 2,2,64*3)) # should match to RNN_SIZES[0]
h = self.ls.deconv2d(scope+'_1', h, 64*2) # 4
h = activation(h)
L = 2
h = self.ls.deconv2d(scope+'_2', h, 16*3) # 8
h = activation(h)
h = PS(h, 4, color=True)
print('h in write:', h)
return tf.reshape( h, [-1, _h, _w, _c])
else:
return self.ls.dense( scope,h, _h*_w*_c )
def network(self, LR):
feature_tmp = tf.layers.conv2d(LR, 64, 5, strides = 1, padding = 'SAME', name = 'CONV_1',
kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
feature_tmp = tf.nn.relu(feature_tmp)
feature_tmp = tf.layers.conv2d(feature_tmp, 32, 3, strides = 1, padding = 'SAME', name = 'CONV_2',
kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
feature_tmp = tf.nn.relu(feature_tmp)
feature_out = tf.layers.conv2d(feature_tmp, self.channels*self.scale*self.scale, 3, strides = 1, padding = 'SAME',
name = 'CONV_3', kernel_initializer = tf.contrib.layers.xavier_initializer(), reuse=tf.AUTO_REUSE)
#feature_out = PS(feature_out, self.scale, color=True)
if self.mode == "RGB":
feature_out = PS(feature_out, self.scale, color=True)
return tf.multiply(tf.add(tf.nn.tanh(feature_out),1),255/2)
else:
feature_out = PS(feature_out, self.scale, color=False)
return tf.add(tf.multiply(tf.add(tf.nn.tanh(feature_out),1),(235-16)/2), 16) #Y range is in 16-235
def test(self, name = "Set5", load = True):
result_dir = os.path.join("./samples/",str(name))
if not os.path.exists(result_dir):
os.makedirs(result_dir)
img_list = sorted(glob(os.path.join("/home/johnyi/deeplearning/research/SISR_Datasets/test/",str(name),"*.png")))
if load == True:
if self.load(self.checkpoint_dir):
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
avg_PSNR = 0
avg_PSNR_bicubic = 0
for i in range(0,len(img_list)):
image = load_image(img_list[i], self.mode)
image_w = self.scale * int(image.shape[0]/self.scale)
image_h = self.scale * int(image.shape[1]/self.scale)
#image = imresize(image,[image_w, image_h], interp='bicubic')
#LR = imresize(image,[int(image_w/self.scale), int(image_h/self.scale)], interp='bicubic')
#out_bicubic = imresize(LR,[image_w, image_h], interp='bicubic')
image = imresize(image,[image_w, image_h])
LR = imresize(image,[int(image_w/self.scale), int(image_h/self.scale)])
out_bicubic = imresize(LR,[image_w, image_h])
if self.mode == "RGB":
imageio.imwrite(result_dir+"/original_"+str(i)+".png", image)
imageio.imwrite(result_dir+"/bicubic_"+str(i)+".png", out_bicubic)
else:
img_rgb = load_image(img_list[i], "RGB")
img_rgb = imresize(img_rgb,[image_w, image_h])
LR_rgb = imresize(img_rgb,[int(image_w/self.scale), int(image_h/self.scale)], interp='bicubic')
out_bicubic_rgb = imresize(LR_rgb,[image_w, image_h], interp='bicubic')
imageio.imwrite(result_dir+"/original_"+str(i)+".png", img_rgb)
imageio.imwrite(result_dir+"/bicubic_"+str(i)+".png", out_bicubic_rgb)
PSNR = 0
PSNR_bicubic = 0
if self.mode == "RGB":
[out] = self.sess.run([self.output2], feed_dict={ self.input2: [LR]})
#print("out shape:",out.shape)
out = PS(out, self.scale, color=True)
#out = (1+np.tanh(out.eval()[0]))*255/2
out = np.round((1+np.tanh(out.eval()[0]))*255/2)
out = out.astype(np.uint8) #Q) is this needed?
#out_bicubic = imresize(LR,[image_w, image_h])
PSNR = calc_PSNR(out,image)
PSNR_bicubic = calc_PSNR(out_bicubic,image)
avg_PSNR += PSNR
avg_PSNR_bicubic += PSNR_bicubic
#imageio.imwrite(result_dir+"/original_"+str(i)+".png", image)
imageio.imwrite(result_dir+"/HR_RGB_"+str(i)+".png", out)
#imageio.imwrite(result_dir+"/bicubic_"+str(i)+".png", out_bicubic)
else:
Y_HR = np.split(image,3, axis=2)[0]
YCbCr_LR = np.copy(LR)
Y_LR = np.split(YCbCr_LR,3, axis=2)[0]
#print("Y_LR shape:",Y_LR.shape)
Cb_LR = np.split(YCbCr_LR,3, axis=2)[1]
Cr_LR = np.split(YCbCr_LR,3, axis=2)[2]
[out] = self.sess.run([self.output2], feed_dict={ self.input2: [Y_LR]})
#print("out shape:",out.shape)
out = PS_1dim(out[0], self.scale)
#print("out:",Y_LR.shape, out.shape)
#out = ((1+np.tanh(out))*(235-16)/2) + 16
out = np.round(((1+np.tanh(out))*(235-16)/2) + 16)
out = out.astype(np.uint8) #Q) is this needed?
out_bicubic = np.split(imresize(LR,[image_w, image_h], interp='bicubic'),3,axis=2)[0]
#out_bicubic = np.split(imresize(LR,[image_w, image_h]),3,axis=2)[0]
PSNR = calc_PSNR(out,Y_HR)
PSNR_bicubic = calc_PSNR(out_bicubic,Y_HR)
path = result_dir+"/HR_Y_"+str(i)+".png"
save_ycbcr_img(out, Cb_LR, Cr_LR, self.scale, path)
avg_PSNR += PSNR
avg_PSNR_bicubic += PSNR_bicubic
print("["+str(name)+"] image",i, "shape:",image.shape, "downsampled:", LR.shape, "PSNR:", PSNR, "bicubic:",PSNR_bicubic )
return avg_PSNR/len(img_list), avg_PSNR_bicubic/len(img_list)