def pan_feature_extraction(self, name, pan): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): with tf.variable_scope('low_level_extraction'): feature = pan feature = conv('conv1', feature, 32) feature = conv('conv2', feature, 32) with tf.variable_scope('down_sample1'): feature = self.downsample(feature, 32) feature = dense_block('dense_block1', feature, 12, conv_num=6, input_include=True) feature = bottle_neck('bottle_neck1', feature, 64) with tf.variable_scope('down_sample2'): feature = self.downsample(feature, 64) feature = dense_block('dense_block2', feature, 12, conv_num=6, input_include=True) feature = bottle_neck('bottle_neck2', feature, 64) feature = dense_block('dense_block3', feature, 12, conv_num=8, input_include=True) feature = bottle_neck('bottle_neck3', feature, 96) with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE): pan_feature_1 = self.upsample(feature, out_channel=96) # pan_feature_1 = conv('conv', tf.concat([feature2, feature3], -1), filter_num=64) with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE): pan_feature_2 = self.upsample(pan_feature_1, out_channel=96) # pan_feature_2 = conv('conv', tf.concat([up_pan_feature_1, feature1], -1), filter_num=64) return pan_feature_1, pan_feature_2
def ms_feature_extraction(self, name, ms): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): with tf.variable_scope('low_level_extraction', reuse=tf.AUTO_REUSE): feature1 = ms feature1 = conv('conv1', feature1, 64) feature1 = conv('conv2', feature1, 64) feature2 = dense_block('dense_block1', feature1, 12, conv_num=6, input_include=True) feature2 = bottle_neck('bottle_neck1', feature2, 64) feature3 = dense_block('dense_block2', feature2, 12, conv_num=6, input_include=True) feature3 = bottle_neck('bottle_neck2', feature3, 64) with tf.variable_scope('combanation'): feature = tf.concat([feature1, feature2, feature3], -1) feature = bottle_neck('bottle_neck', feature, 64) with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE): ms_feature_1 = self.upsample(feature, out_channel=64) with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE): ms_feature_2 = self.upsample(ms_feature_1, out_channel=64) return ms_feature_1, ms_feature_2
def feature_fusion(self, name, ms_feature, pan_feature): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): fusion_feature = tf.concat([ms_feature, pan_feature], axis=-1) channel_in = fusion_feature.get_shape().as_list()[-1] fusion_feature = conv('conv1', fusion_feature, filter_num=96) fusion_feature = conv('conv2', fusion_feature, filter_num=64) detail = conv('conv3', fusion_feature, 4, activation=None) return detail
def forward(self, ms, pan): with tf.variable_scope('PNN', reuse=tf.AUTO_REUSE): up_ms = tf.image.resize_bicubic(ms, size=(self.ms_size * 4, self.ms_size * 4)) x = tf.concat([up_ms, pan], -1) with tf.variable_scope('opt1', reuse=tf.AUTO_REUSE): feature = conv('conv1', x, 64, 9, activation='lrelu') feature = conv('conv2', feature, 32, 5, activation='lrelu') with tf.variable_scope('opt2', reuse=tf.AUTO_REUSE): fusion=conv('conv3',feature,4,5,activation=None)\ +up_ms return fusion
def pan_feature_extraction(self, name, pan): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): feature = pan for i in range(8): feature = conv('conv%d' % (i + 1), feature, 64, 5) with tf.variable_scope('downsample', reuse=tf.AUTO_REUSE): feature1 = self.downsample(feature, 64) return feature1, feature
def ms_feature_extraction(self, name, ms): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): feature = ms for i in range(6): feature = conv('conv%d' % (i + 1), feature, 64) with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE): ms_feature_1 = self.upsample(feature, out_channel=64) with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE): ms_feature_2 = self.upsample(ms_feature_1, out_channel=64) return ms_feature_1, ms_feature_2
def forward(self, ms, pan): with tf.variable_scope('RSIFNN', reuse=tf.AUTO_REUSE): with tf.variable_scope('ms', reuse=tf.AUTO_REUSE): up_ms = tf.image.resize_bicubic( ms, (self.ms_size * 4, self.ms_size * 4)) feature_ms = conv('conv1', up_ms, activation='lrelu') feature_ms = conv('conv2', feature_ms, filter_num=32, activation='lrelu') with tf.variable_scope('pan', reuse=tf.AUTO_REUSE): feature_pan = pan for i in range(1, 8): feature_pan = conv('conv%d' % (i), feature_pan, activation='lrelu') feature_pan = conv('conv%d' % (i + 1), feature_pan, filter_num=32, activation='lrelu') with tf.variable_scope('fusion', reuse=tf.AUTO_REUSE): features = tf.concat([feature_ms, feature_pan], -1) return up_ms + conv('conv', features, 4, activation=None)
def pan_feature_extraction(self, name, pan): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): l1, l2, l3 = LapFusionConfig.pan_depth l1, l2 = l1 - 1, l2 - 1 feature_channel = LapFusionConfig.pan_featurec_channel with tf.variable_scope('block1', reuse=tf.AUTO_REUSE): feature = pan for i in range(1, l1 + 1): feature = conv('conv%d' % (i), feature, feature_channel[0]) with tf.variable_scope('down_sample1', reuse=tf.AUTO_REUSE): feature = self.downsample(feature, out_channel=feature_channel[0]) with tf.variable_scope('block2', reuse=tf.AUTO_REUSE): for i in range(1, l2 + 1): feature = conv('conv%d' % (i), feature, feature_channel[1]) # res=feature # for i in range(1,l2): # res=conv('conv%d'%(i),res,feature_channel[1]) # res=conv('conv%d'%(i+1),res,64,activation=None) # feature=tf.nn.leaky_relu(res+feature) with tf.variable_scope('down_sample2', reuse=tf.AUTO_REUSE): feature = self.downsample(feature, out_channel=feature_channel[1]) with tf.variable_scope('block3', reuse=tf.AUTO_REUSE): for i in range(1, l3 + 1): feature = conv('conv%d' % (i), feature, feature_channel[1]) # res=feature # for i in range(1,l3): # res=conv('conv%d'%(i),res,feature_channel[2]) # res=conv('conv%d'%(i+1),res,feature_channel[2],activation=None) # feature=tf.nn.leaky_relu(res+feature) with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE): pan_feature_1 = self.upsample(feature, out_channel=64) with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE): pan_feature_2 = self.upsample(pan_feature_1, out_channel=64) return pan_feature_1, pan_feature_2
def ms_feature_extraction(self, name, ms): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): with tf.variable_scope('low_level_extraction', reuse=tf.AUTO_REUSE): feature = ms feature = conv('conv1', feature, 32) feature = conv('conv2', feature, 32) feature = dense_block('dense_block1', feature, 12, conv_num=4, input_include=True) feature = bottle_neck('bottle_neck1', feature, 64) feature = dense_block('dense_block2', feature, 12, conv_num=6, input_include=True) feature = bottle_neck('bottle_neck2', feature, 96) with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE): ms_feature_1 = self.upsample(feature, out_channel=96) with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE): ms_feature_2 = self.upsample(ms_feature_1, out_channel=96) return ms_feature_1, ms_feature_2
def ms_feature_extraction(self, name, ms): with tf.variable_scope(name, reuse=tf.AUTO_REUSE): # feature=conv('conv1',ms,LapFusionConfig.ms_feature_channel) # feature=conv('conv2',feature,LapFusionConfig.ms_feature_channel) # res=feature # for i in range(3,8): # res=conv('conv%d'%(i),res,LapFusionConfig.ms_feature_channel) # res=conv('conv%d'%(i+1),res,LapFusionConfig.ms_feature_channel,activation=None) # feature=tf.nn.leaky_relu(feature+res) feature = ms for i in range(1, LapFusionConfig.ms_depth + 1): feature = conv('conv%d' % (i), feature, LapFusionConfig.ms_feature_channel) with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE): ms_feature_1 = self.upsample(feature, out_channel=64) with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE): ms_feature_2 = self.upsample(ms_feature_1, out_channel=64) return ms_feature_1, ms_feature_2
def downsample(self, feature, out_channel, scale=2): # with tf.variable_scope(name,reuse=tf.AUTO_REUSE): return conv('down_conv', feature, out_channel, strides=scale)
def yolov4(input_img): net = module.conv(input_img, 32, 3, 1) net = module.conv(net, 64, 3, 2) route = module.conv(net, 64, 1, 1) net = module.conv(net, 64, 1, 1) tmp = net net = module.conv(net, 32, 1, 1) net = module.conv(net, 64, 3, 1) net = tmp + net net = module.conv(net, 64, 1, 1) net = tf.concat([net, route], -1) net = module.conv(net, 64, 1, 1) net = module.conv(net, 128, 3, 2) route = module.conv(net, 64, 1, 1) net = module.conv(net, 64, 1, 1) for _ in range(2): tmp = net net = module.conv(net, 64, 1, 1) net = module.conv(net, 64, 3, 1) net = tmp + net net = module.conv(net, 64, 1, 1) net = tf.concat([net, route], -1) net = module.conv(net, 128, 1, 1) net = module.conv(net, 256, 3, 2) route = module.conv(net, 128, 1, 1) net = module.conv(net, 128, 1, 1) for _ in range(8): tmp = net net = module.conv(net, 128, 1, 1) net = module.conv(net, 128, 3, 1) net = tmp + net net = module.conv(net, 128, 1, 1) net = tf.concat([net, route], -1) net = module.conv(net, 256, 1, 1) for_output1 = net net = module.conv(net, 512, 3, 2) route = module.conv(net, 256, 1, 1) net = module.conv(net, 256, 1, 1) for _ in range(8): tmp = net net = module.conv(net, 256, 1, 1) net = module.conv(net, 256, 3, 1) net = tmp + net net = module.conv(net, 256, 1, 1) net = tf.concat([net, route], -1) net = module.conv(net, 512, 1, 1) for_output2 = net net = module.conv(net, 1024, 3, 2) route = module.conv(net, 512, 1, 1) net = module.conv(net, 512, 1, 1) for _ in range(4): tmp = net net = module.conv(net, 512, 1, 1) net = module.conv(net, 512, 3, 1) net = tmp + net net = module.conv(net, 512, 1, 1) net = tf.concat([net, route], -1) net = module.conv(net, 1024, 1, 1) # SPP pool_5 = tf.nn.max_pool(net, [1, 5, 5, 1], [1, 1, 1, 1], 'SAME') pool_9 = tf.nn.max_pool(net, [1, 9, 9, 1], [1, 1, 1, 1], 'SAME') pool_13 = tf.nn.max_pool(net, [1, 13, 13, 1], [1, 1, 1, 1], 'SAME') net = tf.concat([pool_13, pool_9, pool_5, net], -1) net = module.conv(net, 512, 1, 1) net = module.conv(net, 1024, 3, 1) net = module.conv(net, 512, 1, 1) output3 = net net = module.conv(net, 256, 1, 1) shape = tf.shape(net) out_height, out_width = shape[1]*2, shape[2]*2 net = tf.compat.v1.image.resize_nearest_neighbor(net, (out_height, out_width)) for_output2 = module.conv(for_output2, 256, 1, 1) net = tf.concat([for_output2, net], -1) net = module.conv(net, 256, 1, 1) net = module.conv(net, 512, 3, 1) net = module.conv(net, 256, 1, 1) net = module.conv(net, 512, 3, 1) net = module.conv(net, 256, 1, 1) output2 = net net = module.conv(net, 128, 1, 1) shape = tf.shape(net) out_height, out_width = shape[1]*2, shape[2]*2 net = tf.compat.v1.image.resize_nearest_neighbor(net, (out_height, out_width)) for_output1 = module.conv(for_output1, 128, 1, 1) net = tf.concat([for_output1, net], -1) net = module.conv(net, 128, 1, 1) net = module.conv(net, 256, 3, 1) net = module.conv(net, 128, 1, 1) net = module.conv(net, 256, 3, 1) net = module.conv(net, 128, 1, 1) output1 = net net = module.conv(output1, 256, 3, 1) net = module.conv(net, 3*(4+1+config.class_num), 1, 1, normalizer_fn=None, activation_fn=None, biases_initializer=tf.zeros_initializer()) label3 = net net = module.conv(output1, 256, 3, 2) net = tf.concat([net, output2], -1) net = module.conv(net, 256, 1, 1) net = module.conv(net, 512, 3, 1) net = module.conv(net, 256, 1, 1) net = module.conv(net, 512, 3, 1) net = module.conv(net, 256, 1, 1) tmp_route = net net = module.conv(net, 512, 3, 1) net = module.conv(net, 3*(4+1+config.class_num), 1, 1, normalizer_fn=None, activation_fn=None, biases_initializer=tf.zeros_initializer()) label2 = net net = module.conv(tmp_route, 512, 3, 2) net = tf.concat([net, output3], -1) net = module.conv(net, 512, 1, 1) net = module.conv(net, 1024, 3, 1) net = module.conv(net, 512, 1, 1) net = module.conv(net, 1024, 3, 1) net = module.conv(net, 512, 1, 1) net = module.conv(net, 1024, 3, 1) net = module.conv(net, 3*(4+1+config.class_num), 1, 1, normalizer_fn=None, activation_fn=None, biases_initializer=tf.zeros_initializer()) label1 = net return label1, label2, label3