def encode(self, reflectance_map, background):
rm_encodings = self.singlet(reflectance_map)
bg_encodings = self.singlet(background)
fully_encoded = tf.concat([rm_encodings[-1], bg_encodings[-1]],
axis=-1)
#fully_encoded = tf.nn.dropout(fully_encoded, 0.5)
fully_encoded = layers.encode_layer(fully_encoded,
1024, (3, 3), (1, 1),
1,
maxpool=False)
decode_1 = layers.decode_layer(fully_encoded, 512, (3, 3), (2, 2), 3)
decode_2 = layers.decode_layer(
tf.concat([decode_1, rm_encodings[3]], axis=-1), 512, (3, 3),
(2, 2), 3)
decode_3 = layers.decode_layer(
tf.concat([decode_2, rm_encodings[2]], axis=-1), 256, (3, 3),
(2, 2), 3)
decode_4 = layers.decode_layer(
tf.concat([decode_3, rm_encodings[1]], axis=-1), 128, (3, 3),
(2, 2), 1)
return layers.encode_layer(decode_4,
3, (1, 1), (1, 1),
1,
activation=None,
norm=False,
maxpool=False)
def encode(self, left, right):
#l_encoding, l_encodings_downsampled, _ = layers.siamese_encode_2(left, reuse=False, depth=FLAGS.depth)
r_encoding, r_encodings_downsampled, r_multiscale = layers.siamese_encode_2(right, reuse=False,
depth=FLAGS.depth)
#if FLAGS.dotprod:
# l_end = tf.nn.l2_normalize(l_encoding, dim=3)
# r_end = tf.nn.l2_normalize(r_encoding, dim=3)
# joint = tf.matmul(l_end, r_end, transpose_b=True)
# joint = tf.concat([l_encoding, r_encoding, joint], axis=-1)
#elif FLAGS.dotprod_pyramid:
# joint = tf.concat([l_encoding, r_encoding], axis=-1)
# for l, r in zip(l_encodings_downsampled, r_encodings_downsampled):
# l_norm = tf.nn.l2_normalize(l, dim=3)
# r_norm = tf.nn.l2_normalize(r, dim=3)
# similarity = tf.matmul(l_norm, r_norm, transpose_b=True)
# joint = tf.concat([joint, similarity], axis=-1)
#else:
# joint = tf.concat([l_encoding, r_encoding], axis=-1)
#if FLAGS.sep_bg:
# bg_encoding = layers.basic_encode(self.bg_image, depth=FLAGS.depth)
# joint = tf.concat([joint, bg_encoding], axis=-1)
joint = r_encoding
joint = layers.encode_layer(joint, 512, (3, 3), (2, 2), 2, maxpool=False)
joint_b = layers.encode_layer(joint, 512, (3, 3), (2, 2), 2, maxpool=True)
joint_c = layers.encode_layer(joint_b, 1024, (3, 3), (2, 2), 1, maxpool=True)
norm_pred = self.decode(joint_c)
return norm_pred, norm_pred
def encode(self, image, out_size):
encode_1 = ed.encode_layer(image, 3, (11, 11), (2, 2))
encode_2 = ed.encode_layer(encode_1, 64, (7, 7), (2, 2))
encode_3 = ed.encode_layer(encode_2, 128, (3, 3), (2, 2))
encode_4 = ed.encode_layer(encode_3, 512, (8, 8), (16, 16))
full_1 = tf.contrib.layers.fully_connected(encode_4, out_size)
return full_1, [encode_4, encode_3, encode_2]
def encode(self, left, right):
l_encodings = layers.siamese_encode(left, reuse=False)
r_encodings = layers.siamese_encode(right, reuse=True)
joint = tf.concat([l_encodings[-1], r_encodings[-1]], axis=-1)
# fully_encoded = tf.nn.dropout(fully_encoded, 0.5)
joint_b = layers.encode_layer(joint, 512, (3, 3), (2, 2), 1, maxpool=True)
joint_c = layers.encode_layer(joint_b, 512, (3, 3), (2, 2), 1, maxpool=True)
joint_d = layers.encode_layer(joint_c, 1024, (3, 3), (2, 2), 1, maxpool=True)
env_pred = self.decode(joint_d)
norm_pred = self.decode(joint_d)
return env_pred, norm_pred
def decode(self, encoded, multiscale):
if FLAGS.multiscale:
encoded = tf.concat([encoded, multiscale[-1]], axis=-1)
decode_1 = layers.decode_layer(encoded, 512, (3, 3), (2, 2),
FLAGS.depth)
decode_1 = tf.concat([decode_1, multiscale[-2]], axis=-1)
decode_2 = layers.decode_layer(decode_1, 512, (3, 3), (2, 2),
FLAGS.depth)
decode_2 = tf.concat([decode_2, multiscale[-3]], axis=-1)
decode_3 = layers.decode_layer(decode_2, 256, (3, 3), (2, 2),
FLAGS.depth)
else:
decode_1 = layers.decode_layer(encoded, 512, (3, 3), (2, 2),
FLAGS.depth)
decode_2 = layers.decode_layer(decode_1, 512, (3, 3), (2, 2),
FLAGS.depth)
decode_3 = layers.decode_layer(decode_2, 256, (3, 3), (2, 2),
FLAGS.depth)
decode_4 = layers.decode_layer(decode_3, 128, (3, 3), (2, 2),
FLAGS.depth)
return layers.encode_layer(decode_4,
3, (1, 1), (1, 1),
1,
activation=None,
norm=False,
maxpool=False)
def decode(self, encoded):
decode_1 = layers.decode_layer(encoded, 512, (3, 3), (2, 2), FLAGS.depth)
decode_2 = layers.decode_layer(decode_1, 512, (3, 3), (2, 2), FLAGS.depth)
decode_3 = layers.decode_layer(decode_2, 256, (3, 3), (2, 2), FLAGS.depth)
decode_4 = layers.decode_layer(decode_3, 128, (3, 3), (2, 2), FLAGS.depth)
decode_5 = layers.decode_layer(decode_4, 64, (3, 3), (2, 2), FLAGS.depth)
return layers.encode_layer(decode_5, 3, (1, 1), (1, 1), 1, activation=None, norm=False, maxpool=False)
def encode(self, left, right):
l_encoding, l_encodings_downsampled, _ = layers.siamese_encode_2(
left, reuse=False, depth=FLAGS.depth)
r_encoding, r_encodings_downsampled, r_multiscale = layers.siamese_encode_2(
right, reuse=True, depth=FLAGS.depth)
bg_encoding = layers.basic_encode(self.bg_lab,
depth=FLAGS.depth,
name="bg")
if FLAGS.dotprod:
l_end = tf.nn.l2_normalize(l_encoding, dim=3)
r_end = tf.nn.l2_normalize(r_encoding, dim=3)
joint = tf.matmul(l_end, r_end, transpose_b=True)
joint = tf.concat([l_encoding, r_encoding, joint], axis=-1)
elif FLAGS.dotprod_pyramid:
joint = tf.concat([l_encoding, r_encoding], axis=-1)
for l, r in zip(l_encodings_downsampled, r_encodings_downsampled):
l_norm = tf.nn.l2_normalize(l, dim=3)
r_norm = tf.nn.l2_normalize(r, dim=3)
similarity = tf.matmul(l_norm, r_norm, transpose_b=True)
joint = tf.concat([joint, similarity], axis=-1)
else:
joint = tf.concat([l_encoding, r_encoding], axis=-1)
fused_a = layers.encode_layer(joint,
512, (3, 3), (2, 2),
2,
maxpool=False)
norms = self.decode_norms(fused_a)
ref_map = self.encode_rm(norms, right)
fused_b = tf.concat([fused_a, bg_encoding, ref_map], axis=-1)
fused_b = layers.encode_layer(fused_b,
512, (3, 3), (2, 2),
2,
maxpool=True)
fused_b = layers.encode_layer(fused_b,
1024, (3, 3), (2, 2),
1,
maxpool=True)
env_pred = self.decode(fused_b)
return env_pred, norms
def singlet(input):
encode_1 = layers.encode_layer(input,
64, (3, 3), (2, 2),
2,
regularizer=regularizer)
encode_2 = layers.encode_layer(encode_1,
128, (3, 3), (2, 2),
2,
regularizer=regularizer)
encode_3 = layers.encode_layer(encode_2,
256, (3, 3), (2, 2),
3,
regularizer=regularizer)
encode_4 = layers.encode_layer(encode_3,
512, (3, 3), (2, 2),
3,
regularizer=regularizer)
encode_5 = layers.encode_layer(encode_4,
512, (3, 3), (2, 2),
3,
regularizer=regularizer)
return [encode_1, encode_2, encode_3, encode_4, encode_5]