def create_dense(self):
if len(self.config.layers['2D_encoder_nodes']) > 0:
num_nodes = self.config.layers['2D_encoder_nodes'][0]
if self.config.layers['merge_encoders']:
with tf.device('/device:GPU:%i' %
(self.config.layers['preferred_gpu'])):
self.merged['features_transposed'] = layers.bn_dense(
self.merged['features_transposed'],
self.merged['encoder_nodes'], num_nodes, self.phase,
self.config.training, 'bn_encoder_merged_in')
self.merged['encoder_nodes'] = self.merged[
'features_transposed'].shape.as_list()[1]
else:
for encoder_config in self.config.encoders_3D:
with tf.device('/device:GPU:%i' %
(encoder_config['preferred_gpu'])):
encoder_id = encoder_config['id']
self.encoders_3D[encoder_id][
'features_transposed'] = layers.bn_dense(
self.encoders_3D[encoder_id]
['features_transposed'],
self.encoders_3D[encoder_id]['encoder_nodes'],
num_nodes, self.phase, self.config.training,
'bn_encoder_' + encoder_id + '_in')
self.encoders_3D[encoder_id][
'encoder_nodes'] = self.encoders_3D[encoder_id][
'features_transposed'].shape.as_list()[1]
def create_vae_features(self):
if self.config.layers['vae']:
with tf.device('/device:GPU:%i' %
(self.config.layers['preferred_gpu'])):
with tf.variable_scope('vae_encoder'):
print(' creating vae encoder nodes.')
self.merged['mn_features_transposed'] = None
self.merged['sd_features_transposed'] = None
n = self.merged['encoder_nodes']
self.merged['mn_features_transposed'] = layers.bn_dense(
self.merged['features_transposed'], n, n, self.phase,
self.config.training,
'mn_feature_transposed_encoder_%i' % n)
self.merged['sd_features_transposed'] = layers.bn_dense(
self.merged['features_transposed'], n, n, self.phase,
self.config.training,
'sd_feature_transposed_encoder_%i' % n)
tf.summary.histogram('hist_mn',
self.merged['mn_features_transposed'])
tf.summary.histogram('hist_sd',
self.merged['sd_features_transposed'])
self.dim_mn_features = self.merged[
'mn_features_transposed'].shape.as_list()[1]
# Draw samples from the distribution P(z|X),
# to generate samples for the decoder input from the encoder distribution
# For this, we need a normal distribution for "reparameterization trick"
eps = tf.random_normal(shape=(self.batch_size,
self.dim_mn_features),
mean=0.0,
stddev=1.0)
# we get z_mean, z_log_sigma_sq from encoder, and draw z from N(z_mean,z_sigma^2)
# self.features = tf.add(self.mn_features, tf.multiply(tf.multiply(self.config.sigma_vae,tf.sqrt(tf.exp(self.sd_features))), eps)) # using reparameterization trick
# self.merged['features_transposed'] = tf.add(self.merged['mn_features_transposed'],
# tf.multiply(tf.multiply(self.config.sigma_vae,tf.sqrt(tf.exp(self.merged['sd_features_transposed']))), eps)) # using reparameterization trick
self.merged['features_transposed'] = tf.add(
self.merged['mn_features_transposed'],
tf.multiply(
tf.sqrt(
tf.exp(self.merged['sd_features_transposed'])),
eps))
self.merged['encoder_nodes'] = self.merged[
'features_transposed'].shape.as_list()[1]
def create_dense_feature_layers(self):
print(' creating dense layers')
with tf.device('/device:GPU:%i' %
(self.config.layers['preferred_gpu'])):
self.feature_shape = self.features_v.shape.as_list()
sh = self.feature_shape
self.encoder_input_size = sh[1] * sh[2] * sh[3] * sh[4]
# setup shared feature space between horizontal/vertical encoder
self.features = tf.concat([
tf.reshape(self.features_h, [-1, self.encoder_input_size]),
tf.reshape(self.features_v, [-1, self.encoder_input_size])
], 1)
self.features_transposed = tf.concat([
tf.reshape(tf.transpose(self.features_h, [0, 1, 3, 2, 4]),
[-1, self.encoder_input_size]),
tf.reshape(self.features_v, [-1, self.encoder_input_size])
], 1)
self.encoder_nodes = self.features.shape.as_list()[1]
# encode both h/v features into autoencoder bottleneck
with tf.variable_scope('feature_encoder'):
for n in self.config.layers['autoencoder_nodes']:
print(' creating %i encoder nodes for dense layer.' % n)
size_in = self.features.shape.as_list()[1]
self.features = layers.bn_dense(
self.features, size_in, n, self.phase,
self.config.training, 'bn_feature_encoder_%i' % n)
self.features_transposed = layers.bn_dense(
self.features_transposed, size_in, n, self.phase,
self.config.training, 'bn_feature_encoder_%i' % n)
self.encoder_nodes = self.features.shape.as_list()[1]
with tf.variable_scope('input'):
self.input_features = tf.placeholder(
tf.float32, self.features.shape.as_list())
def create_2D_decoder(self, decoder_config):
decoder = dict()
decoder_id = decoder_config['id']
print('creating decoder pipeline ' + decoder_id)
# create a decoder pathway (center view only)
with tf.variable_scope(decoder_id):
sh = self.feature_shape
decoder['id'] = decoder_id
decoder['channels'] = decoder_config['channels']
decoder['loss_fn'] = decoder_config['loss_fn']
decoder['weight'] = decoder_config['weight']
decoder['train'] = decoder_config['train']
decoder['preferred_gpu'] = decoder_config['preferred_gpu']
# decode features to upwards input using configured dense layers
num_layers = len(self.config.layers['2D_decoder_nodes'])
if num_layers > 0 and decoder['loss_fn'] != 'L2':
decoder['features'] = self.features_transposed
size_list = self.config.layers['2D_decoder_nodes']
for n in size_list:
print(' creating %i decoder nodes for dense layer.' % n)
size_in = decoder['features'].shape.as_list()[1]
decoder['features'] = layers.bn_dense(
decoder['features'], size_in, n, self.phase,
self.config.training, 'bn_feature_decoder_%i' % n)
self.encoder_nodes = self.features.shape.as_list()[1]
size_in = decoder['features'].shape.as_list()[1]
H = loss_max_coord_2D - loss_min_coord_2D
W = loss_max_coord_2D - loss_min_coord_2D
nodes = H * W * decoder['channels']
decoder['upconv_reduce'] = layers.bn_dense(
decoder['features'], size_in, nodes, self.phase,
self.config.training, 'bn_upconv')
decoder['upconv_reduce'] = tf.reshape(
decoder['upconv_reduce'], [-1, H, W, decoder['channels']])
decoder['input'] = tf.placeholder(
tf.float32, [None, self.H, self.W, decoder['channels']])
decoder['input_reduce'] = decoder[
'input'][:, loss_min_coord_2D:loss_max_coord_2D,
loss_min_coord_2D:loss_max_coord_2D, :]
else:
decoder['nodes'] = sh[2] * sh[3] * sh[4]
decoder['upconv_in'] = layers.bn_dense(
self.features_transposed, self.encoder_nodes,
decoder['nodes'], self.phase, self.config.training,
'bn_decoder_' + decoder_id + '_in')
layout = self.config.layers['autoencoder']
decoder['layers'] = []
# decoding layer 0 - all variables already created
sh = self.feature_shape
decoder['upconv'] = tf.reshape(decoder['upconv_in'],
[-1, sh[2], sh[3], sh[4]])
last_layer = min(len(layout), self.max_layer)
for i in range(0, last_layer):
layer_id = "decoder_%s_%i" % (decoder_id,
last_layer - i - 1)
print(' generating upconvolution layer structure ' +
layer_id)
out_channels = -1
if i == last_layer - 1:
out_channels = decoder['channels']
# evil hack
no_relu = False
if i == last_layer - 1 and decoder_id == 'depth':
print(' skipping last ReLU for depth regression.')
no_relu = True
decoder['layers'].insert(
i,
layers.layer_upconv2d(layer_id,
self.encoder_variables[-1 - i],
self.batch_size,
decoder['upconv'],
self.phase,
self.config.training,
out_channels=out_channels,
no_relu=no_relu))
if self.pinhole and i != last_layer - 1:
cv_pos = np.int32(
(self.conv_layers_h[-2 - i].output_shape[1] - 1) /
2)
features_add = tf.concat([
tf.transpose(self.conv_layers_h[-2 - i].out,
[0, 1, 3, 2, 4])[:, cv_pos, :, :, :],
self.conv_layers_v[-2 - i].out[:, cv_pos, :, :, :]
], 3)
decoder['upconv'] = tf.concat(
[decoder['layers'][-1].out, features_add], axis=3)
decoder['layers'][
i].pinhole_weight = layers.pinhole_weight_2d(
decoder['layers'][i], decoder['upconv'])
decoder['upconv'] = layers.pinhole_conv2d(
decoder['layers'][i], decoder['upconv'])
else:
decoder['upconv'] = decoder['layers'][-1].out
decoder['upconv_reduce'] = decoder[
'upconv'][:, loss_min_coord_2D:loss_max_coord_2D,
loss_min_coord_2D:loss_max_coord_2D, :]
decoder['input'] = tf.placeholder(
tf.float32, [None, self.H, self.W, decoder['channels']])
decoder['input_reduce'] = decoder[
'input'][:, loss_min_coord_2D:loss_max_coord_2D,
loss_min_coord_2D:loss_max_coord_2D, :]
self.decoders_2D[decoder_id] = decoder
def create_linked_3D_decoder(self, linked_decoders):
decoder_ids = [
linked_decoders[i]['id'] for i in range(0, len(linked_decoders))
]
assert (loss_min_coord_3D == 0)
assert (loss_max_coord_3D == 48)
for decoder_config in linked_decoders:
with tf.device('/device:GPU:%i' %
(decoder_config['preferred_gpu'])):
decoder = dict()
decoder_id = decoder_config['id']
print('creating decoder pipeline ' + decoder_id)
# create a decoder pathway (center view only)
with tf.variable_scope(decoder_id):
sh = self.feature_shape
decoder['id'] = decoder_id
decoder['channels'] = decoder_config['channels']
decoder['loss_fn'] = decoder_config['loss_fn']
decoder['weight'] = decoder_config['weight']
decoder['train'] = decoder_config['train']
decoder['preferred_gpu'] = decoder_config['preferred_gpu']
decoder['nodes'] = sh[2] * sh[3] * sh[4]
decoder['upconv_in'] = layers.bn_dense(
self.features, self.encoder_nodes, decoder['nodes'],
self.phase, self.config.training,
'bn_decoder_' + decoder_id + '_in')
decoder['layers_v'] = []
decoder['layers_h'] = []
decoder['variables'] = []
# decode features to upwards input using configured dense layers
with tf.variable_scope('feature_decoder'):
num_layers = len(
self.config.layers['autoencoder_nodes'])
if num_layers > 0:
decoder['features'] = self.features
size_list = reversed(
self.config.layers['autoencoder_nodes'][0:-1])
for n in size_list:
print(
' creating %i decoder nodes for dense layer.'
% n)
size_in = decoder['features'].shape.as_list(
)[1]
decoder['features'] = layers.bn_dense(
decoder['features'], size_in, n,
self.phase, self.config.training,
'bn_feature_decoder_%i' % n)
self.encoder_nodes = self.features.shape.as_list(
)[1]
size_in = decoder['features'].shape.as_list()[1]
decoder['upconv_v'] = layers.bn_dense(
decoder['features'], size_in,
self.encoder_input_size, self.phase,
self.config.training, 'bn_upconv_v_in')
decoder['upconv_v'] = tf.reshape(
decoder['upconv_v'],
[-1, sh[1], sh[2], sh[3], sh[4]])
decoder['upconv_h'] = layers.bn_dense(
decoder['features'], size_in,
self.encoder_input_size, self.phase,
self.config.training, 'bn_upconv_h_in')
decoder['upconv_h'] = tf.reshape(
decoder['upconv_h'],
[-1, sh[1], sh[2], sh[3], sh[4]])
else:
# no dense encoder layers - wire directly
decoder['upconv_v'] = self.features_v
decoder['upconv_h'] = self.features_h
self.decoders_3D[decoder_id] = decoder
del decoder
# now the convolutional decoder layers
layout = self.config.layers['autoencoder']
last_layer = min(len(layout), self.max_layer)
start_layer = 0
for c_layer in self.config.connect_layers:
for decoder_config in linked_decoders:
decoder_id = decoder_config['id']
with tf.device('/device:GPU:%i' %
(decoder_config['preferred_gpu'])):
for i in range(start_layer, c_layer):
layer_id = "decoder_%s_%i" % (decoder_id,
last_layer - i - 1)
print(' generating upconvolution layer structure ' +
layer_id)
self.decoders_3D[decoder_id]['variables'].insert(
i,
layers.decoder_variables(
layer_id, self.encoder_variables[-1 - i]))
self.decoders_3D[decoder_id]['layers_v'].insert(
i,
layers.layer_upconv3d(
layer_id + "_v",
self.decoders_3D[decoder_id]['variables'][i],
self.batch_size,
self.decoders_3D[decoder_id]['upconv_v'],
self.phase,
self.config.training,
shared_variables=True))
if self.pinhole and i != c_layer - 1:
self.decoders_3D[decoder_id][
'upconv_v'] = tf.concat([
self.decoders_3D[decoder_id]['layers_v']
[-1].out, self.conv_layers_v[-2 - i].out
],
axis=4)
self.decoders_3D[decoder_id]['variables'][
i].pinhole_weight = layers.pinhole_weight(
self.decoders_3D[decoder_id]['variables']
[i],
self.decoders_3D[decoder_id]['upconv_v'])
self.decoders_3D[decoder_id][
'upconv_v'] = layers.pinhole_conv3d(
self.decoders_3D[decoder_id]['variables']
[i],
self.decoders_3D[decoder_id]['upconv_v'])
else:
self.decoders_3D[decoder_id][
'upconv_v'] = self.decoders_3D[decoder_id][
'layers_v'][-1].out
self.decoders_3D[decoder_id]['layers_h'].insert(
i,
layers.layer_upconv3d(
layer_id + "_h",
self.decoders_3D[decoder_id]['variables'][i],
self.batch_size,
self.decoders_3D[decoder_id]['upconv_h'],
self.phase,
self.config.training,
shared_variables=True))
if self.pinhole and i != c_layer - 1:
self.decoders_3D[decoder_id][
'upconv_h'] = tf.concat([
self.decoders_3D[decoder_id]['layers_h']
[-1].out, self.conv_layers_h[-2 - i].out
],
axis=4)
self.decoders_3D[decoder_id][
'upconv_h'] = layers.pinhole_conv3d(
self.decoders_3D[decoder_id]['variables']
[i],
self.decoders_3D[decoder_id]['upconv_h'])
else:
self.decoders_3D[decoder_id][
'upconv_h'] = self.decoders_3D[decoder_id][
'layers_h'][-1].out
# concatenate components
start_layer = c_layer
if start_layer != last_layer:
joined_v = tf.concat(
[self.decoders_3D[id]['upconv_v'] for id in decoder_ids],
axis=4)
joined_h = tf.concat(
[self.decoders_3D[id]['upconv_h'] for id in decoder_ids],
axis=4)
if self.pinhole:
joined_v = tf.concat(
[joined_v, self.conv_layers_v[-1 - c_layer].out],
axis=4)
joined_h = tf.concat(
[joined_h, self.conv_layers_h[-1 - c_layer].out],
axis=4)
for decoder_config in linked_decoders:
decoder_id = decoder_config['id']
with tf.device('/device:GPU:%i' %
(decoder_config['preferred_gpu'])):
with tf.variable_scope(decoder_id):
self.decoders_3D[decoder_id]['variables'][
c_layer -
1].pinhole_weight = layers.pinhole_weight(
self.decoders_3D[decoder_id]['variables'][
c_layer - 1], joined_v)
for decoder_config in linked_decoders:
decoder_id = decoder_config['id']
with tf.device('/device:GPU:%i' %
(decoder_config['preferred_gpu'])):
with tf.variable_scope(decoder_id):
self.decoders_3D[decoder_id][
'upconv_v'] = layers.pinhole_conv3d(
self.decoders_3D[decoder_id]['variables'][
c_layer - 1], joined_v)
self.decoders_3D[decoder_id][
'upconv_h'] = layers.pinhole_conv3d(
self.decoders_3D[decoder_id]['variables'][
c_layer - 1], joined_h)
for decoder_config in linked_decoders:
decoder_id = decoder_config['id']
with tf.device('/device:GPU:%i' %
(decoder_config['preferred_gpu'])):
self.decoders_3D[decoder_id][
'upconv_v_reduce'] = self.decoders_3D[decoder_id][
'upconv_v'][:, :, loss_min_coord_3D:loss_max_coord_3D,
loss_min_coord_3D:loss_max_coord_3D, :]
self.decoders_3D[decoder_id][
'upconv_h_reduce'] = self.decoders_3D[decoder_id][
'upconv_h'][:, :, loss_min_coord_3D:loss_max_coord_3D,
loss_min_coord_3D:loss_max_coord_3D, :]
self.decoders_3D[decoder_id]['input_v'] = tf.placeholder(
tf.float32,
self.decoders_3D[decoder_id]['upconv_v'].shape.as_list())
self.decoders_3D[decoder_id]['input_h'] = tf.placeholder(
tf.float32,
self.decoders_3D[decoder_id]['upconv_h'].shape.as_list())
self.decoders_3D[decoder_id][
'input_v_reduce'] = self.decoders_3D[decoder_id][
'input_v'][:, :, loss_min_coord_3D:loss_max_coord_3D,
loss_min_coord_3D:loss_max_coord_3D, :]
self.decoders_3D[decoder_id][
'input_h_reduce'] = self.decoders_3D[decoder_id][
'input_h'][:, :, loss_min_coord_3D:loss_max_coord_3D,
loss_min_coord_3D:loss_max_coord_3D, :]
def create_3D_decoder(self, decoder_config):
decoder = dict()
decoder_id = decoder_config['id']
print('creating decoder pipeline ' + decoder_id)
# create a decoder pathway (center view only)
with tf.variable_scope(decoder_id):
sh = self.feature_shape
decoder['id'] = decoder_id
decoder['channels'] = decoder_config['channels']
decoder['loss_fn'] = decoder_config['loss_fn']
decoder['weight'] = decoder_config['weight']
decoder['train'] = decoder_config['train']
decoder['preferred_gpu'] = decoder_config['preferred_gpu']
decoder['nodes'] = sh[2] * sh[3] * sh[4]
decoder['upconv_in'] = layers.bn_dense(
self.features, self.encoder_nodes, decoder['nodes'],
self.phase, self.config.training,
'bn_decoder_' + decoder_id + '_in')
layout = self.config.layers['autoencoder']
decoder['layers_v'] = []
decoder['layers_h'] = []
decoder['variables'] = []
# decode features to upwards input using configured dense layers
with tf.variable_scope('feature_decoder'):
num_layers = len(self.config.layers['autoencoder_nodes'])
if num_layers > 0:
decoder['features'] = self.features
size_list = reversed(
self.config.layers['autoencoder_nodes'][0:-1])
for n in size_list:
print(' creating %i decoder nodes for dense layer.' %
n)
size_in = decoder['features'].shape.as_list()[1]
decoder['features'] = layers.bn_dense(
decoder['features'], size_in, n, self.phase,
self.config.training, 'bn_feature_decoder_%i' % n)
self.encoder_nodes = self.features.shape.as_list()[1]
size_in = decoder['features'].shape.as_list()[1]
decoder['upconv_v'] = layers.bn_dense(
decoder['features'], size_in, self.encoder_input_size,
self.phase, self.config.training, 'bn_upconv_v_in')
decoder['upconv_v'] = tf.reshape(
decoder['upconv_v'], [-1, sh[1], sh[2], sh[3], sh[4]])
decoder['upconv_h'] = layers.bn_dense(
decoder['features'], size_in, self.encoder_input_size,
self.phase, self.config.training, 'bn_upconv_h_in')
decoder['upconv_h'] = tf.reshape(
decoder['upconv_h'], [-1, sh[1], sh[2], sh[3], sh[4]])
else:
# no dense encoder layers - wire directly
decoder['upconv_v'] = self.features_v
decoder['upconv_h'] = self.features_h
# now the convolutional decoder layers
last_layer = min(len(layout), self.max_layer)
for i in range(0, last_layer):
layer_id = "decoder_%s_%i" % (decoder_id, last_layer - i - 1)
print(' generating upconvolution layer structure ' +
layer_id)
decoder['variables'].insert(
i,
layers.decoder_variables(layer_id,
self.encoder_variables[-1 - i]))
decoder['layers_v'].insert(
i,
layers.layer_upconv3d(layer_id + "_v",
decoder['variables'][i],
self.batch_size, decoder['upconv_v'],
self.phase, self.config.training))
if self.pinhole and i != last_layer - 1:
decoder['upconv_v'] = tf.concat([
decoder['layers_v'][-1].out,
self.conv_layers_v[-2 - i].out
],
axis=4)
decoder['variables'][
i].pinhole_weight = layers.pinhole_weight(
decoder['variables'][i], decoder['upconv_v'])
decoder['upconv_v'] = layers.pinhole_conv3d(
decoder['variables'][i], decoder['upconv_v'])
print('pinhole')
else:
print('no pinhole')
decoder['upconv_v'] = decoder['layers_v'][-1].out
decoder['layers_h'].insert(
i,
layers.layer_upconv3d(layer_id + "_h",
decoder['variables'][i],
self.batch_size, decoder['upconv_h'],
self.phase, self.config.training))
if self.pinhole and i != last_layer - 1:
decoder['upconv_h'] = tf.concat([
decoder['layers_h'][-1].out,
self.conv_layers_h[-2 - i].out
],
axis=4)
decoder['upconv_h'] = layers.pinhole_conv3d(
decoder['variables'][i], decoder['upconv_h'])
else:
decoder['upconv_h'] = decoder['layers_h'][-1].out
# to save some memory, as we do not use it atm
assert (loss_min_coord_3D == 0)
assert (loss_max_coord_3D == 48)
decoder['upconv_v_reduce'] = decoder[
'upconv_v'][:, :, loss_min_coord_3D:loss_max_coord_3D,
loss_min_coord_3D:loss_max_coord_3D, :]
decoder['upconv_h_reduce'] = decoder[
'upconv_h'][:, :, loss_min_coord_3D:loss_max_coord_3D,
loss_min_coord_3D:loss_max_coord_3D, :]
decoder['input_v'] = tf.placeholder(
tf.float32, decoder['upconv_v'].shape.as_list())
decoder['input_h'] = tf.placeholder(
tf.float32, decoder['upconv_h'].shape.as_list())
decoder['input_v_reduce'] = decoder[
'input_v'][:, :, loss_min_coord_3D:loss_max_coord_3D,
loss_min_coord_3D:loss_max_coord_3D, :]
decoder['input_h_reduce'] = decoder[
'input_h'][:, :, loss_min_coord_3D:loss_max_coord_3D,
loss_min_coord_3D:loss_max_coord_3D, :]
self.decoders_3D[decoder_id] = decoder
def create_2D_decoder(self, decoder_config):
decoder = dict()
decoder_id = decoder_config['id']
ids = []
for i in range(0, len(self.config.layer_config)):
ids.append(self.config.layer_config[i]['id'])
pos_layout = ids.index(decoder_id)
print('creating decoder pipeline ' + decoder_id)
# create a decoder pathway (center view only)
with tf.variable_scope(decoder_id):
decoder['id'] = decoder_id
decoder['channels'] = decoder_config['channels']
decoder['loss_fn'] = decoder_config['loss_fn']
decoder['weight'] = decoder_config['weight']
decoder['train'] = decoder_config['train']
decoder['preferred_gpu'] = decoder_config['preferred_gpu']
decoder['2D_variables'] = []
decoder['concat_variables'] = []
decoder['upscale_variables'] = []
decoder['start'] = self.config.layer_config[pos_layout]['start']
decoder['end'] = self.config.layer_config[pos_layout]['end']
decoder['no_relu'] = decoder_config['no_relu']
sh = self.encoders_3D[decoder_id]['feature_shape']
decoder['nodes'] = sh[2] * sh[3] * sh[4] * self.config.patch_weight
if self.config.layers['merge_encoders']:
decoder['upconv_in'] = layers.bn_dense(
self.merged['features_transposed'],
self.merged['encoder_nodes'], decoder['nodes'], self.phase,
self.config.training, 'bn_decoder_' + decoder_id + '_in')
else:
decoder['upconv_in'] = layers.bn_dense(
self.encoders_3D[decoder_id]['features_transposed'],
self.encoders_3D[decoder_id]['encoder_nodes'],
decoder['nodes'], self.phase, self.config.training,
'bn_decoder_' + decoder_id + '_in')
sh = self.encoders_3D[decoder_id]['feature_shape']
decoder['upconv'] = tf.reshape(
decoder['upconv_in'],
[-1, sh[2], sh[3], sh[4] * self.config.patch_weight])
decoder['layers'] = []
########################################################################################################
# decoder variables
layout = []
layout.insert(0, self.config.layer_config[pos_layout]['layout'][0])
for i in range(0, len(self.config.layers['encoder_3D'])):
layout.append(self.config.layers['encoder_3D'][i])
last_layer = min(len(layout), self.max_layer)
patches = self.config.nviews
for i in range(0, last_layer):
layer_id_cat = "skip_patch_%s_%i" % (decoder_id, i)
print(' generating decoder ' + layer_id_cat)
patches = math.ceil(patches / layout[i]['stride'][1])
decoder['concat_variables'].append(
layers.concat_variables(layer_id_cat,
layout[i],
patches,
input_features=[],
output_features=[]))
# maybe change later
layer_id_cat = "skip_patch_%s_%i" % (decoder_id, i + 1)
decoder['concat_variables'].insert(
0,
layers.concat_variables(
layer_id_cat,
layout[i],
self.config.nviews,
input_features=self.config.layer_config[pos_layout]
['layout'][0]['conv'][-2],
output_features=self.config.layer_config[pos_layout]
['upscale'][0]['conv'][-2]))
layout[0] = self.config.layer_config[pos_layout]['upscale'][0]
for i in range(0, last_layer):
layer_id = "decoder_%s_%i" % (decoder_id, i)
print(' generating upconvolution variables ' + layer_id)
decoder['2D_variables'].append(
layers.decoder_variables_2D(layer_id, layout[i], i,
last_layer,
self.config.patch_weight,
self.config.D))
for i in range(0, last_layer):
layer_id = "decoder_%s_layer%i" % (decoder_id,
last_layer - i - 1)
layer_id_cat = "skip_patch_%s_layer%i" % (decoder_id,
last_layer - i - 1)
print(' generating upconvolution layer structure ' +
layer_id)
out_channels = -1
# evil hack
no_relu = False
decoder['layers'].insert(
-1 - i,
layers.layer_upconv2d_v2(layer_id,
decoder['2D_variables'][-1 - i],
self.batch_size,
decoder['upconv'],
self.phase,
self.config.training,
out_channels=out_channels,
no_relu=no_relu))
if self.pinhole:
if i != last_layer - 1:
concat_cv = layers.layer_conv_one(
layer_id_cat, decoder['concat_variables'][-2 - i],
layers.layer_concat(
self.encoders_3D[decoder_id]['conv_layers_v'][
-2 - i].out, self.encoders_3D[decoder_id]
['conv_layers_h'][-2 - i].out).out).out
else:
start = self.config.layer_config[pos_layout]['start']
end = self.config.layer_config[pos_layout]['end']
concat_cv = layers.layer_conv_one(
layer_id_cat, decoder['concat_variables'][-2 - i],
layers.layer_concat(
self.stack_v[:, :, :, :, start:end],
self.stack_h[:, :, :, :, start:end]).out).out
decoder['upconv'] = tf.concat(
[decoder['layers'][-1 - i].out, concat_cv], axis=3)
else:
decoder['upconv'] = decoder['layers'][-1 - i].out
# decoder['layers'] = tf.reverse(decoder['layers'],0)
decoder['upconv_reduce'] = decoder[
'upconv'][:, loss_min_coord_2D:loss_max_coord_2D,
loss_min_coord_2D:loss_max_coord_2D, :]
decoder['input'] = tf.placeholder(
tf.float32, [None, self.H_HR, self.W_HR, decoder['channels']])
decoder['input_reduce'] = decoder[
'input'][:, loss_min_coord_2D:loss_max_coord_2D_1,
loss_min_coord_2D:loss_max_coord_2D_1, :]
layout = []
for i in range(0, len(self.config.layers['upscale'])):
layout.append(self.config.layers['upscale'][i])
last_layer_up = min(len(layout), self.max_layer)
for i in range(0, last_layer_up):
layer_id_upscale = "upscale_%s_%i" % (decoder_id, i)
print(' creating variables for ' + layer_id_upscale)
decoder['upscale_variables'].append(
layers.upscale_variables(layer_id_upscale, layout[i]))
layout_final = self.config.layer_config[pos_layout]['final'][0]
no_relu = False
for upscale in range(0, last_layer_up):
out_channels = -1
layer_id = "upscale_%s_%i" % (decoder_id, upscale)
print(' creating layers for ' + layer_id)
decoder['upconv'] = layers.layer_upconv2d_v2(
layer_id,
decoder['upscale_variables'][upscale],
self.batch_size,
decoder['upconv'],
self.phase,
self.config.training,
out_channels=out_channels,
no_relu=no_relu).out
no_relu = decoder['no_relu']
decoder['SR'] = layers.layer_pure_conv2D("upscale_final",
layout_final,
decoder['upconv'],
self.phase,
self.config.training,
no_relu=no_relu).out
self.decoders_2D[decoder_id] = decoder
