def fusion_model(self, img): with tf.compat.v1.variable_scope('fusion_model'): with tf.compat.v1.variable_scope('layer1'): weights = tf.compat.v1.get_variable("w1", [5, 5, 2, 256], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable("b1", [256], initializer=tf.constant_initializer(0.0)) conv1_ir = tf.contrib.layers.batch_norm(tf.nn.conv2d(img, weights, strides=[1, 1, 1, 1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv1_ir = lrelu(conv1_ir) with tf.compat.v1.variable_scope('layer2'): weights = tf.compat.v1.get_variable("w2", [5, 5, 256, 128], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable("b2", [128], initializer=tf.constant_initializer(0.0)) conv2_ir = tf.contrib.layers.batch_norm(tf.nn.conv2d(conv1_ir, weights, strides=[1, 1, 1, 1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv2_ir = lrelu(conv2_ir) with tf.compat.v1.variable_scope('layer3'): weights = tf.compat.v1.get_variable("w3", [3, 3, 128, 64], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable("b3", [64], initializer=tf.constant_initializer(0.0)) conv3_ir = tf.contrib.layers.batch_norm(tf.nn.conv2d(conv2_ir, weights, strides=[1, 1, 1, 1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv3_ir = lrelu(conv3_ir) with tf.compat.v1.variable_scope('layer4'): weights = tf.compat.v1.get_variable("w4", [3, 3, 64, 32], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable("b4", [32], initializer=tf.constant_initializer(0.0)) conv4_ir = tf.contrib.layers.batch_norm(tf.nn.conv2d(conv3_ir, weights, strides=[1, 1, 1, 1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv4_ir = lrelu(conv4_ir) with tf.compat.v1.variable_scope('layer5'): weights = tf.compat.v1.get_variable("w5", [1, 1, 32, 1], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable("b5", [1], initializer=tf.constant_initializer(0.0)) conv5_ir = tf.nn.conv2d(conv4_ir, weights, strides=[1, 1, 1, 1], padding='VALID') + bias conv5_ir = tf.nn.tanh(conv5_ir) return conv5_ir
def discriminator(self,img,reuse,update_collection=None): with tf.variable_scope('discriminator',reuse=reuse): print(img.shape) with tf.variable_scope('layer_1'): weights=tf.get_variable("w_1",[3,3,1,32],initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights=weights_spectral_norm(weights,update_collection=update_collection) bias=tf.get_variable("b_1",[32],initializer=tf.constant_initializer(0.0)) conv1_vi=tf.nn.conv2d(img, weights, strides=[1,2,2,1], padding='VALID') + bias conv1_vi = lrelu(conv1_vi) with tf.variable_scope('layer_2'): weights=tf.get_variable("w_2",[3,3,32,64],initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights=weights_spectral_norm(weights,update_collection=update_collection) bias=tf.get_variable("b_2",[64],initializer=tf.constant_initializer(0.0)) conv2_vi= tf.contrib.layers.batch_norm(tf.nn.conv2d(conv1_vi, weights, strides=[1,2,2,1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv2_vi = lrelu(conv2_vi) with tf.variable_scope('layer_3'): weights=tf.get_variable("w_3",[3,3,64,128],initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights=weights_spectral_norm(weights,update_collection=update_collection) bias=tf.get_variable("b_3",[128],initializer=tf.constant_initializer(0.0)) conv3_vi= tf.contrib.layers.batch_norm(tf.nn.conv2d(conv2_vi, weights, strides=[1,2,2,1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv3_vi=lrelu(conv3_vi) with tf.variable_scope('layer_4'): weights=tf.get_variable("w_4",[3,3,128,256],initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights=weights_spectral_norm(weights,update_collection=update_collection) bias=tf.get_variable("b_4",[256],initializer=tf.constant_initializer(0.0)) conv4_vi= tf.contrib.layers.batch_norm(tf.nn.conv2d(conv3_vi, weights, strides=[1,2,2,1], padding='VALID') + bias, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv4_vi=lrelu(conv4_vi) conv4_vi = tf.reshape(conv4_vi,[self.batch_size,6*6*256]) with tf.variable_scope('line_5'): weights=tf.get_variable("w_5",[6*6*256,2],initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights=weights_spectral_norm(weights,update_collection=update_collection) bias=tf.get_variable("b_5",[2],initializer=tf.constant_initializer(0.0)) line_5=tf.matmul(conv4_vi, weights) + bias return line_5
def ir_feature_extraction_network(self, ir_image): with tf.compat.v1.variable_scope('ir_extraction_network'): with tf.compat.v1.variable_scope('conv1'): weights = tf.compat.v1.get_variable( "w", [5, 5, 1, 16], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [16], initializer=tf.constant_initializer(0.0)) conv1 = tf.nn.conv2d( ir_image, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv1 = tf.contrib.layers.batch_norm(conv1, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv1 = tf.nn.leaky_relu(conv1) # state size: 32 with tf.compat.v1.variable_scope('conv2'): weights = tf.compat.v1.get_variable( "w", [3, 3, 16, 16], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [16], initializer=tf.constant_initializer(0.0)) conv2 = tf.nn.conv2d( conv1, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv2 = tf.contrib.layers.batch_norm(conv2, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv2 = tf.nn.leaky_relu(conv2) # concat_conv2 = tf.concat([conv2, conv1], axis=-1) # state size: 32 with tf.compat.v1.variable_scope('conv3'): weights = tf.compat.v1.get_variable( "w", [3, 3, 16, 32], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [32], initializer=tf.constant_initializer(0.0)) conv3 = tf.nn.conv2d( conv2, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv3 = tf.contrib.layers.batch_norm(conv3, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv3 = tf.nn.leaky_relu(conv3) concat_conv3 = tf.concat([conv3, conv2], axis=-1) # state size: 64 with tf.compat.v1.variable_scope('conv4'): weights = tf.compat.v1.get_variable( "w", [3, 3, 32, 64], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [64], initializer=tf.constant_initializer(0.0)) conv4 = tf.nn.conv2d( conv3, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv4 = tf.contrib.layers.batch_norm(conv4, decay=0.9, updates_collections=None, epsilon=1e-5, # scale=True) conv4 = tf.nn.sigmoid(conv4) concat_conv4 = tf.concat([conv4, concat_conv3], axis=-1) encoding_feature = concat_conv4 return encoding_feature
def feature_reconstruction_network(self, feature): with tf.compat.v1.variable_scope('reconstruction_network'): with tf.compat.v1.variable_scope('conv1'): weights = tf.compat.v1.get_variable( "w", [3, 3, 224, 128], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [128], initializer=tf.constant_initializer(0.0)) conv1 = tf.nn.conv2d( feature, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv1 = tf.contrib.layers.batch_norm(conv1, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv1 = tf.nn.leaky_relu(conv1) # state size: 128 with tf.compat.v1.variable_scope('conv2'): weights = tf.compat.v1.get_variable( "w", [3, 3, 128, 64], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [64], initializer=tf.constant_initializer(0.0)) conv2 = tf.nn.conv2d( conv1, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv2 = tf.contrib.layers.batch_norm(conv2, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv2 = tf.nn.leaky_relu(conv2) # state size: 64 with tf.compat.v1.variable_scope('conv3'): weights = tf.compat.v1.get_variable( "w", [3, 3, 64, 32], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [32], initializer=tf.constant_initializer(0.0)) conv3 = tf.nn.conv2d( conv2, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv3 = tf.contrib.layers.batch_norm(conv3, decay=0.9, updates_collections=None, epsilon=1e-5, # scale=True) conv3 = tf.nn.leaky_relu(conv3) # state size: 32 with tf.compat.v1.variable_scope('conv4'): weights = tf.compat.v1.get_variable( "w", [3, 3, 32, 16], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [16], initializer=tf.constant_initializer(0.0)) conv4 = tf.nn.conv2d( conv3, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv4 = tf.contrib.layers.batch_norm(conv4, decay=0.9, updates_collections=None, epsilon=1e-5, # scale=True) conv4 = tf.nn.leaky_relu(conv4) with tf.compat.v1.variable_scope('conv5'): weights = tf.compat.v1.get_variable( "w", [3, 3, 16, 1], initializer=tf.truncated_normal_initializer(stddev=1e-3)) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", [1], initializer=tf.constant_initializer(0.0)) conv5 = tf.nn.conv2d( conv4, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv5 = tf.contrib.layers.batch_norm(conv5, decay=0.9, updates_collections=None, epsilon=1e-5, # scale=True) conv5 = tf.nn.tanh(conv5) fusion_image = conv5 return fusion_image
def ir_feature_extraction_network(self, ir_image, reader): with tf.compat.v1.variable_scope('ir_extraction_network'): with tf.compat.v1.variable_scope('conv1'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/ir_extraction_network/conv1/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/ir_extraction_network/conv1/b'))) conv1 = tf.nn.conv2d( ir_image, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv1 = tf.contrib.layers.batch_norm(conv1, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv1 = tf.nn.leaky_relu(conv1) # state size: 32 with tf.compat.v1.variable_scope('conv2'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/ir_extraction_network/conv2/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/ir_extraction_network/conv2/b'))) conv2 = tf.nn.conv2d( conv1, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv2 = tf.contrib.layers.batch_norm(conv2, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv2 = tf.nn.leaky_relu(conv2) # concat_conv2 = tf.concat([conv2, conv1], axis=-1) # state size: 32 with tf.compat.v1.variable_scope('conv3'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/ir_extraction_network/conv3/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/ir_extraction_network/conv3/b'))) conv3 = tf.nn.conv2d( conv2, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv3 = tf.contrib.layers.batch_norm(conv3, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv3 = tf.nn.leaky_relu(conv3) concat_conv3 = tf.concat([conv3, conv2], axis=-1) # state size: 64 with tf.compat.v1.variable_scope('conv4'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/ir_extraction_network/conv4/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/ir_extraction_network/conv4/b'))) conv4 = tf.nn.conv2d( conv3, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv4 = tf.contrib.layers.batch_norm(conv4, decay=0.9, updates_collections=None, epsilon=1e-5, # scale=True) conv4 = tf.nn.sigmoid(conv4) concat_conv4 = tf.concat([conv4, concat_conv3], axis=-1) encoding_feature = concat_conv4 return encoding_feature
def feature_reconstruction_network(self, feature, reader): with tf.compat.v1.variable_scope('reconstruction_network'): with tf.compat.v1.variable_scope('conv1'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv1/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv1/b'))) conv1 = tf.nn.conv2d( feature, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv1 = tf.contrib.layers.batch_norm(conv1, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv1 = tf.nn.leaky_relu(conv1) # state size: 128 with tf.compat.v1.variable_scope('conv2'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv2/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv2/b'))) conv2 = tf.nn.conv2d( conv1, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv2 = tf.contrib.layers.batch_norm(conv2, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv2 = tf.nn.leaky_relu(conv2) # state size: 64 with tf.compat.v1.variable_scope('conv3'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv3/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv3/b'))) conv3 = tf.nn.conv2d( conv2, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv3 = tf.contrib.layers.batch_norm(conv3, decay=0.9, updates_collections=None, epsilon=1e-5, scale=True) conv3 = tf.nn.leaky_relu(conv3) # state size: 32 with tf.compat.v1.variable_scope('conv4'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv4/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv4/b'))) conv4 = tf.nn.conv2d( conv3, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv4 = tf.contrib.layers.batch_norm(conv4, decay=0.9, updates_collections=None, epsilon=1e-5, # scale=True) conv4 = tf.nn.leaky_relu(conv4) with tf.compat.v1.variable_scope('conv5'): weights = tf.compat.v1.get_variable( "w", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv5/w'))) weights = weights_spectral_norm(weights) bias = tf.compat.v1.get_variable( "b", initializer=tf.constant( reader.get_tensor( 'STMFusion_model/reconstruction_network/conv5/b'))) conv5 = tf.nn.conv2d( conv4, weights, strides=[1, 1, 1, 1 ], padding='SAME') + bias # conv5 = tf.contrib.layers.batch_norm(conv5, decay=0.9, updates_collections=None, epsilon=1e-5, # scale=True) conv5 = tf.nn.tanh(conv5) fusion_image = conv5 return fusion_image