def __init__(self,
layer_id,
variables,
batch_size,
output_shape,
input,
phase,
params,
shared_variables=True):
with tf.variable_scope(layer_id):
self.input_shape = input.shape.as_list()
self.bn = batch_norm(input, phase, params, 'batchnorm_input')
if shared_variables:
# main decoder pipe, variables are shared between horizontal and vertical
if variables.stride[2] != 1 or variables.stride[3] != 1:
self.bn = tf.stack([
resize_bicubic(self.bn[:, i, :, :, :], output_shape)
for i in range(0, self.input_shape[1])
],
axis=1)
self.bn = tf.pad(self.bn,
[[0, 0], [1, 1], [0, 0], [0, 0], [0, 0]],
'SYMMETRIC')
self.bn = tf.pad(self.bn,
[[0, 0], [0, 0], [1, 1], [1, 1], [0, 0]],
'CONSTANT')
self.conv = tf.nn.conv3d(self.bn,
variables.decoder_variables_W,
strides=[1, 1, 1, 1, 1],
padding='VALID')
self.features = resnet_elu(self.conv +
variables.decoder_variables_b)
if variables.resample:
# bug (?) workaround: conv3d_transpose with strides does not seem to work, no idea why.
# instead, we use upsampling + conv3d_transpose without stride.
self.input_upsampled = input
if variables.stride[2] != 1 or variables.stride[3] != 1:
self.input_upsampled = tf.stack([
resize_bicubic(input[:, i, :, :, :], output_shape)
for i in range(0, self.input_shape[1])
],
axis=1)
identity = tf.nn.conv3d(self.input_upsampled,
variables.decoder_variables_embedding,
strides=[1, 1, 1, 1, 1],
padding='VALID')
else:
identity = input
self.out = self.features + identity
def create_2D_decoder(self, decoder_config):
decoder = dict()
decoder_id = decoder_config['id']
ids = []
for i in range(0, len(self.config.layer_config)):
ids.append(self.config.layer_config[i]['id'])
pos_layout = ids.index(decoder_id)
print('creating decoder pipeline ' + decoder_id)
# create a decoder pathway (center view only)
with tf.variable_scope(decoder_id):
decoder['id'] = decoder_id
decoder['channels'] = decoder_config['channels']
decoder['loss_fn'] = decoder_config['loss_fn']
decoder['weight'] = decoder_config['weight']
decoder['train'] = decoder_config['train']
decoder['preferred_gpu'] = decoder_config['preferred_gpu']
decoder['2D_variables'] = []
decoder['concat_variables'] = []
decoder['upscale_variables'] = []
decoder['start'] = self.config.layer_config[pos_layout]['start']
decoder['end'] = self.config.layer_config[pos_layout]['end']
decoder['bicubic'] = resize_bicubic(
self.stack_v[:, self.cv_pos, :, :,
decoder['start']:decoder['end']],
[self.H_HR, self.W_HR])
sh = self.encoders_3D[decoder_id]['feature_shape']
decoder['nodes'] = sh[2] * sh[3] * sh[4] * self.config.patch_weight
decoder['upconv_in'] = layers.bn_dense(
self.encoders_3D[decoder_id]['features_transposed'],
self.encoders_3D[decoder_id]['encoder_nodes'],
decoder['nodes'], self.phase, self.config.training,
'bn_decoder_' + decoder_id + '_in')
sh = self.encoders_3D[decoder_id]['feature_shape']
decoder['upconv'] = tf.reshape(
decoder['upconv_in'],
[-1, sh[2], sh[3], sh[4] * self.config.patch_weight])
decoder['layers'] = []
########################################################################################################
# decoder variables
layout = []
layout.insert(0, self.config.layer_config[pos_layout]['layout'][0])
for i in range(0, len(self.config.layers['encoder_3D'])):
layout.append(self.config.layers['encoder_3D'][i])
last_layer = min(len(layout), self.max_layer)
patches = self.config.nviews
for i in range(0, last_layer):
layer_id_cat = "skip_patch_%s_%i" % (decoder_id, i)
print(' generating decoder ' + layer_id_cat)
patches = math.ceil(patches / layout[i]['stride'][1])
decoder['concat_variables'].append(
layers.concat_variables(layer_id_cat,
layout[i],
patches,
input_features=[],
output_features=[]))
# maybe change later
layer_id_cat = "skip_patch_%s_%i" % (decoder_id, i + 1)
decoder['concat_variables'].insert(
0,
layers.concat_variables(
layer_id_cat,
layout[i],
self.config.nviews,
input_features=self.config.layer_config[pos_layout]
['layout'][0]['conv'][-2],
output_features=self.config.layer_config[pos_layout]
['upscale'][0]['conv'][-2]))
layout[0] = self.config.layer_config[pos_layout]['upscale'][0]
for i in range(0, last_layer):
layer_id = "decoder_%s_%i" % (decoder_id, i)
print(' generating upconvolution variables ' + layer_id)
decoder['2D_variables'].append(
layers.decoder_variables_2D(layer_id, layout[i], i,
last_layer,
self.config.patch_weight,
self.config.D))
for i in range(0, last_layer):
layer_id = "decoder_%s_layer%i" % (decoder_id,
last_layer - i - 1)
layer_id_cat = "skip_patch_%s_layer%i" % (decoder_id,
last_layer - i - 1)
print(' generating upconvolution layer structure ' +
layer_id)
out_channels = -1
# evil hack
no_relu = False
decoder['layers'].insert(
-1 - i,
layers.layer_upconv2d_v2(layer_id,
decoder['2D_variables'][-1 - i],
self.batch_size,
decoder['upconv'],
self.phase,
self.config.training,
out_channels=out_channels,
no_relu=no_relu))
if self.pinhole:
if i != last_layer - 1:
concat_cv = layers.layer_conv_one(
layer_id_cat, decoder['concat_variables'][-2 - i],
layers.layer_concat(
self.encoders_3D[decoder_id]['conv_layers_v'][
-2 - i].out, self.encoders_3D[decoder_id]
['conv_layers_h'][-2 - i].out).out).out
else:
start = self.config.layer_config[pos_layout]['start']
end = self.config.layer_config[pos_layout]['end']
concat_cv = layers.layer_conv_one(
layer_id_cat, decoder['concat_variables'][-2 - i],
layers.layer_concat(
self.stack_v[:, :, :, :, start:end],
self.stack_h[:, :, :, :, start:end]).out).out
decoder['upconv'] = tf.concat(
[decoder['layers'][-1 - i].out, concat_cv], axis=3)
else:
decoder['upconv'] = decoder['layers'][-1 - i].out
# decoder['layers'] = tf.reverse(decoder['layers'],0)
decoder['upconv_reduce'] = decoder[
'upconv'][:, loss_min_coord_2D:loss_max_coord_2D,
loss_min_coord_2D:loss_max_coord_2D, :]
decoder['input'] = tf.placeholder(
tf.float32, [None, self.H_HR, self.W_HR, decoder['channels']])
decoder['input_reduce'] = decoder[
'input'][:, loss_min_coord_2D:loss_max_coord_2D_1,
loss_min_coord_2D:loss_max_coord_2D_1, :]
layout = []
for i in range(0, len(self.config.layers['upscale'])):
layout.append(self.config.layers['upscale'][i])
last_layer_up = min(len(layout), self.max_layer)
for i in range(0, last_layer_up):
layer_id_upscale = "upscale_%s_%i" % (decoder_id, i)
print(' creating variables for ' + layer_id_upscale)
decoder['upscale_variables'].append(
layers.upscale_variables(layer_id_upscale, layout[i]))
layout_final = self.config.layer_config[pos_layout]['final'][0]
no_relu = False
for upscale in range(0, last_layer_up):
out_channels = -1
layer_id = "upscale_%s_%i" % (decoder_id, upscale)
print(' creating layers for ' + layer_id)
if upscale == 1:
decoder['upconv'] = tf.concat(
[decoder['upconv'], decoder['bicubic']], axis=3)
decoder['upconv'] = layers.layer_upconv2d_v2(
layer_id,
decoder['upscale_variables'][upscale],
self.batch_size,
decoder['upconv'],
self.phase,
self.config.training,
out_channels=out_channels,
no_relu=no_relu).out
if self.config.config['ColorSpace'] == 'YUV' or self.config.config[
'ColorSpace'] == 'LAB':
no_relu = True
decoder['SR'] = layers.layer_pure_conv2D("upscale_final",
layout_final,
decoder['upconv'],
self.phase,
self.config.training,
no_relu=no_relu).out
self.decoders_2D[decoder_id] = decoder
def __init__(self,
layer_id,
variables,
batch_size,
output_shape,
input,
phase,
params,
shared_variables=True):
with tf.variable_scope(layer_id):
self.input_shape = input.shape.as_list()
self.output_shape = [
batch_size, output_shape[0] + 2, output_shape[1] + 2,
variables.C_out
]
self.bn = batch_norm(input, phase, params, 'batchnorm_input')
if shared_variables:
# main decoder pipe, variables are shared between horizontal and vertical
if variables.stride[1] != 1 or variables.stride[2] != 1:
self.bn = resize_bicubic(self.bn, output_shape)
self.conv = tf.nn.conv2d_transpose(
self.bn,
variables.decoder_variables_W,
output_shape=self.output_shape,
strides=[1, 1, 1, 1],
padding='VALID')
self.conv = self.conv[:, 1:-1, 1:-1, :]
self.features = resnet_elu(self.conv +
variables.decoder_variables_b)
if variables.resample:
# bug (?) workaround: conv3d_transpose with strides does not seem to work, no idea why.
# instead, we use upsampling + conv3d_transpose without stride.
self.input_upsampled = input
if variables.stride[1] != 1 or variables.stride[2] != 1:
self.input_upsampled = resize_bicubic(input, output_shape)
self.input_upsampled = tf.pad(self.input_upsampled,
[[0, 0], [1, 1], [1, 1], [0, 0]],
'CONSTANT')
identity = tf.nn.conv2d_transpose(
self.input_upsampled,
variables.decoder_variables_embedding,
output_shape=self.output_shape,
strides=[1, 1, 1, 1],
padding='VALID')
# if variables.stride[1] > 1:
identity = identity[:, 1:-1, 1:-1, :]
else:
identity = input
self.out = self.features + identity
self.out = self.features
def __init__(self,
layer_id,
variables,
batch_size,
input,
phase,
params,
out_channels=-1,
no_relu=False):
with tf.variable_scope(layer_id):
# define in/out shapes
self.shape = variables.shape
self.stride = variables.stride
# self.input_shape = input.shape.as_list()
self.input_shape = input.shape
self.C_in = variables.C_in
self.C_out = variables.C_out
self.output_shape = input.shape.as_list()
self.output_shape[0] = batch_size
self.output_shape[1] = self.output_shape[1] * self.stride[1]
self.output_shape[2] = self.output_shape[2] * self.stride[2]
self.output_shape[3] = self.C_out
self.W = variables.decoder_variables_W
self.b = variables.decoder_variables_b
sh = input.shape.as_list()
self.resample = variables.resample
if self.resample:
self.embedding = variables.decoder_variables_embedding
if out_channels != -1:
# output channel override
self.output_shape[3] = out_channels
self.shape[2] = out_channels
self.C_in = out_channels
self.resample = True
# generate layers
self.bn = batch_norm(input, phase, params, 'batchnorm_input')
if self.stride[1] != 1 or self.stride[2] != 1:
self.bn = resize_bicubic(
self.bn,
[np.int(sh[1] * 2), np.int(sh[2] * 2)])
self.conv = tf.nn.conv2d(self.bn,
self.W,
strides=[1, 1, 1, 1],
padding='SAME')
if no_relu:
self.features = self.conv + self.b
else:
self.features = resnet_elu(self.conv + self.b)
if variables.resample:
if self.stride[1] != 1 or self.stride[2] != 1:
self.input_upsampled = resize_bicubic(
input, [np.int(sh[1] * 2),
np.int(sh[2] * 2)])
else:
self.input_upsampled = input
identity = tf.nn.conv2d(self.input_upsampled,
self.embedding,
strides=[1, 1, 1, 1],
padding='SAME')
else:
identity = input
self.out = self.features + identity