def fusion_model(self, img):
with tf.compat.v1.variable_scope('fusion_model'):
with tf.compat.v1.variable_scope('layer1'):
weights = tf.compat.v1.get_variable("w1", [5, 5, 2, 256], initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable("b1", [256], initializer=tf.constant_initializer(0.0))
conv1_ir = tf.contrib.layers.batch_norm(tf.nn.conv2d(img, weights, strides=[1, 1, 1, 1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv1_ir = lrelu(conv1_ir)
with tf.compat.v1.variable_scope('layer2'):
weights = tf.compat.v1.get_variable("w2", [5, 5, 256, 128], initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable("b2", [128], initializer=tf.constant_initializer(0.0))
conv2_ir = tf.contrib.layers.batch_norm(tf.nn.conv2d(conv1_ir, weights, strides=[1, 1, 1, 1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv2_ir = lrelu(conv2_ir)
with tf.compat.v1.variable_scope('layer3'):
weights = tf.compat.v1.get_variable("w3", [3, 3, 128, 64], initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable("b3", [64], initializer=tf.constant_initializer(0.0))
conv3_ir = tf.contrib.layers.batch_norm(tf.nn.conv2d(conv2_ir, weights, strides=[1, 1, 1, 1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv3_ir = lrelu(conv3_ir)
with tf.compat.v1.variable_scope('layer4'):
weights = tf.compat.v1.get_variable("w4", [3, 3, 64, 32], initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable("b4", [32], initializer=tf.constant_initializer(0.0))
conv4_ir = tf.contrib.layers.batch_norm(tf.nn.conv2d(conv3_ir, weights, strides=[1, 1, 1, 1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv4_ir = lrelu(conv4_ir)
with tf.compat.v1.variable_scope('layer5'):
weights = tf.compat.v1.get_variable("w5", [1, 1, 32, 1], initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable("b5", [1], initializer=tf.constant_initializer(0.0))
conv5_ir = tf.nn.conv2d(conv4_ir, weights, strides=[1, 1, 1, 1], padding='VALID') + bias
conv5_ir = tf.nn.tanh(conv5_ir)
return conv5_ir
def discriminator(self,img,reuse,update_collection=None):
with tf.variable_scope('discriminator',reuse=reuse):
print(img.shape)
with tf.variable_scope('layer_1'):
weights=tf.get_variable("w_1",[3,3,1,32],initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights=weights_spectral_norm(weights,update_collection=update_collection)
bias=tf.get_variable("b_1",[32],initializer=tf.constant_initializer(0.0))
conv1_vi=tf.nn.conv2d(img, weights, strides=[1,2,2,1], padding='VALID') + bias
conv1_vi = lrelu(conv1_vi)
with tf.variable_scope('layer_2'):
weights=tf.get_variable("w_2",[3,3,32,64],initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights=weights_spectral_norm(weights,update_collection=update_collection)
bias=tf.get_variable("b_2",[64],initializer=tf.constant_initializer(0.0))
conv2_vi= tf.contrib.layers.batch_norm(tf.nn.conv2d(conv1_vi, weights, strides=[1,2,2,1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv2_vi = lrelu(conv2_vi)
with tf.variable_scope('layer_3'):
weights=tf.get_variable("w_3",[3,3,64,128],initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights=weights_spectral_norm(weights,update_collection=update_collection)
bias=tf.get_variable("b_3",[128],initializer=tf.constant_initializer(0.0))
conv3_vi= tf.contrib.layers.batch_norm(tf.nn.conv2d(conv2_vi, weights, strides=[1,2,2,1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv3_vi=lrelu(conv3_vi)
with tf.variable_scope('layer_4'):
weights=tf.get_variable("w_4",[3,3,128,256],initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights=weights_spectral_norm(weights,update_collection=update_collection)
bias=tf.get_variable("b_4",[256],initializer=tf.constant_initializer(0.0))
conv4_vi= tf.contrib.layers.batch_norm(tf.nn.conv2d(conv3_vi, weights, strides=[1,2,2,1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv4_vi=lrelu(conv4_vi)
conv4_vi = tf.reshape(conv4_vi,[self.batch_size,6*6*256])
with tf.variable_scope('line_5'):
weights=tf.get_variable("w_5",[6*6*256,2],initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights=weights_spectral_norm(weights,update_collection=update_collection)
bias=tf.get_variable("b_5",[2],initializer=tf.constant_initializer(0.0))
line_5=tf.matmul(conv4_vi, weights) + bias
return line_5
def ir_feature_extraction_network(self, ir_image):
with tf.compat.v1.variable_scope('ir_extraction_network'):
with tf.compat.v1.variable_scope('conv1'):
weights = tf.compat.v1.get_variable(
"w", [5, 5, 1, 16],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [16], initializer=tf.constant_initializer(0.0))
conv1 = tf.nn.conv2d(
ir_image, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv1 = tf.contrib.layers.batch_norm(conv1, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv1 = tf.nn.leaky_relu(conv1)
# state size: 32
with tf.compat.v1.variable_scope('conv2'):
weights = tf.compat.v1.get_variable(
"w", [3, 3, 16, 16],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [16], initializer=tf.constant_initializer(0.0))
conv2 = tf.nn.conv2d(
conv1, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv2 = tf.contrib.layers.batch_norm(conv2, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv2 = tf.nn.leaky_relu(conv2)
# concat_conv2 = tf.concat([conv2, conv1], axis=-1)
# state size: 32
with tf.compat.v1.variable_scope('conv3'):
weights = tf.compat.v1.get_variable(
"w", [3, 3, 16, 32],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [32], initializer=tf.constant_initializer(0.0))
conv3 = tf.nn.conv2d(
conv2, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv3 = tf.contrib.layers.batch_norm(conv3, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv3 = tf.nn.leaky_relu(conv3)
concat_conv3 = tf.concat([conv3, conv2], axis=-1)
# state size: 64
with tf.compat.v1.variable_scope('conv4'):
weights = tf.compat.v1.get_variable(
"w", [3, 3, 32, 64],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [64], initializer=tf.constant_initializer(0.0))
conv4 = tf.nn.conv2d(
conv3, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv4 = tf.contrib.layers.batch_norm(conv4, decay=0.9, updates_collections=None, epsilon=1e-5,
# scale=True)
conv4 = tf.nn.sigmoid(conv4)
concat_conv4 = tf.concat([conv4, concat_conv3], axis=-1)
encoding_feature = concat_conv4
return encoding_feature
def feature_reconstruction_network(self, feature):
with tf.compat.v1.variable_scope('reconstruction_network'):
with tf.compat.v1.variable_scope('conv1'):
weights = tf.compat.v1.get_variable(
"w", [3, 3, 224, 128],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [128], initializer=tf.constant_initializer(0.0))
conv1 = tf.nn.conv2d(
feature, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv1 = tf.contrib.layers.batch_norm(conv1, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv1 = tf.nn.leaky_relu(conv1)
# state size: 128
with tf.compat.v1.variable_scope('conv2'):
weights = tf.compat.v1.get_variable(
"w", [3, 3, 128, 64],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [64], initializer=tf.constant_initializer(0.0))
conv2 = tf.nn.conv2d(
conv1, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv2 = tf.contrib.layers.batch_norm(conv2, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv2 = tf.nn.leaky_relu(conv2)
# state size: 64
with tf.compat.v1.variable_scope('conv3'):
weights = tf.compat.v1.get_variable(
"w", [3, 3, 64, 32],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [32], initializer=tf.constant_initializer(0.0))
conv3 = tf.nn.conv2d(
conv2, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv3 = tf.contrib.layers.batch_norm(conv3, decay=0.9, updates_collections=None, epsilon=1e-5,
# scale=True)
conv3 = tf.nn.leaky_relu(conv3)
# state size: 32
with tf.compat.v1.variable_scope('conv4'):
weights = tf.compat.v1.get_variable(
"w", [3, 3, 32, 16],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [16], initializer=tf.constant_initializer(0.0))
conv4 = tf.nn.conv2d(
conv3, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv4 = tf.contrib.layers.batch_norm(conv4, decay=0.9, updates_collections=None, epsilon=1e-5,
# scale=True)
conv4 = tf.nn.leaky_relu(conv4)
with tf.compat.v1.variable_scope('conv5'):
weights = tf.compat.v1.get_variable(
"w", [3, 3, 16, 1],
initializer=tf.truncated_normal_initializer(stddev=1e-3))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b", [1], initializer=tf.constant_initializer(0.0))
conv5 = tf.nn.conv2d(
conv4, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv5 = tf.contrib.layers.batch_norm(conv5, decay=0.9, updates_collections=None, epsilon=1e-5,
# scale=True)
conv5 = tf.nn.tanh(conv5)
fusion_image = conv5
return fusion_image
def ir_feature_extraction_network(self, ir_image, reader):
with tf.compat.v1.variable_scope('ir_extraction_network'):
with tf.compat.v1.variable_scope('conv1'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/ir_extraction_network/conv1/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/ir_extraction_network/conv1/b')))
conv1 = tf.nn.conv2d(
ir_image, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv1 = tf.contrib.layers.batch_norm(conv1, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv1 = tf.nn.leaky_relu(conv1)
# state size: 32
with tf.compat.v1.variable_scope('conv2'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/ir_extraction_network/conv2/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/ir_extraction_network/conv2/b')))
conv2 = tf.nn.conv2d(
conv1, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv2 = tf.contrib.layers.batch_norm(conv2, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv2 = tf.nn.leaky_relu(conv2)
# concat_conv2 = tf.concat([conv2, conv1], axis=-1)
# state size: 32
with tf.compat.v1.variable_scope('conv3'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/ir_extraction_network/conv3/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/ir_extraction_network/conv3/b')))
conv3 = tf.nn.conv2d(
conv2, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv3 = tf.contrib.layers.batch_norm(conv3, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv3 = tf.nn.leaky_relu(conv3)
concat_conv3 = tf.concat([conv3, conv2], axis=-1)
# state size: 64
with tf.compat.v1.variable_scope('conv4'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/ir_extraction_network/conv4/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/ir_extraction_network/conv4/b')))
conv4 = tf.nn.conv2d(
conv3, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv4 = tf.contrib.layers.batch_norm(conv4, decay=0.9, updates_collections=None, epsilon=1e-5,
# scale=True)
conv4 = tf.nn.sigmoid(conv4)
concat_conv4 = tf.concat([conv4, concat_conv3], axis=-1)
encoding_feature = concat_conv4
return encoding_feature
def feature_reconstruction_network(self, feature, reader):
with tf.compat.v1.variable_scope('reconstruction_network'):
with tf.compat.v1.variable_scope('conv1'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv1/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv1/b')))
conv1 = tf.nn.conv2d(
feature, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv1 = tf.contrib.layers.batch_norm(conv1, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv1 = tf.nn.leaky_relu(conv1)
# state size: 128
with tf.compat.v1.variable_scope('conv2'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv2/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv2/b')))
conv2 = tf.nn.conv2d(
conv1, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv2 = tf.contrib.layers.batch_norm(conv2, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv2 = tf.nn.leaky_relu(conv2)
# state size: 64
with tf.compat.v1.variable_scope('conv3'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv3/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv3/b')))
conv3 = tf.nn.conv2d(
conv2, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv3 = tf.contrib.layers.batch_norm(conv3, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True)
conv3 = tf.nn.leaky_relu(conv3)
# state size: 32
with tf.compat.v1.variable_scope('conv4'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv4/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv4/b')))
conv4 = tf.nn.conv2d(
conv3, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv4 = tf.contrib.layers.batch_norm(conv4, decay=0.9, updates_collections=None, epsilon=1e-5,
# scale=True)
conv4 = tf.nn.leaky_relu(conv4)
with tf.compat.v1.variable_scope('conv5'):
weights = tf.compat.v1.get_variable(
"w",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv5/w')))
weights = weights_spectral_norm(weights)
bias = tf.compat.v1.get_variable(
"b",
initializer=tf.constant(
reader.get_tensor(
'STMFusion_model/reconstruction_network/conv5/b')))
conv5 = tf.nn.conv2d(
conv4, weights, strides=[1, 1, 1, 1
], padding='SAME') + bias
# conv5 = tf.contrib.layers.batch_norm(conv5, decay=0.9, updates_collections=None, epsilon=1e-5,
# scale=True)
conv5 = tf.nn.tanh(conv5)
fusion_image = conv5
return fusion_image
