def pan_feature_extraction(self, name, pan):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
with tf.variable_scope('low_level_extraction'):
feature = pan
feature = conv('conv1', feature, 32)
feature = conv('conv2', feature, 32)
with tf.variable_scope('down_sample1'):
feature = self.downsample(feature, 32)
feature = dense_block('dense_block1',
feature,
12,
conv_num=6,
input_include=True)
feature = bottle_neck('bottle_neck1', feature, 64)
with tf.variable_scope('down_sample2'):
feature = self.downsample(feature, 64)
feature = dense_block('dense_block2',
feature,
12,
conv_num=6,
input_include=True)
feature = bottle_neck('bottle_neck2', feature, 64)
feature = dense_block('dense_block3',
feature,
12,
conv_num=8,
input_include=True)
feature = bottle_neck('bottle_neck3', feature, 96)
with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE):
pan_feature_1 = self.upsample(feature, out_channel=96)
# pan_feature_1 = conv('conv', tf.concat([feature2, feature3], -1), filter_num=64)
with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE):
pan_feature_2 = self.upsample(pan_feature_1, out_channel=96)
# pan_feature_2 = conv('conv', tf.concat([up_pan_feature_1, feature1], -1), filter_num=64)
return pan_feature_1, pan_feature_2
def ms_feature_extraction(self, name, ms):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
with tf.variable_scope('low_level_extraction',
reuse=tf.AUTO_REUSE):
feature1 = ms
feature1 = conv('conv1', feature1, 64)
feature1 = conv('conv2', feature1, 64)
feature2 = dense_block('dense_block1',
feature1,
12,
conv_num=6,
input_include=True)
feature2 = bottle_neck('bottle_neck1', feature2, 64)
feature3 = dense_block('dense_block2',
feature2,
12,
conv_num=6,
input_include=True)
feature3 = bottle_neck('bottle_neck2', feature3, 64)
with tf.variable_scope('combanation'):
feature = tf.concat([feature1, feature2, feature3], -1)
feature = bottle_neck('bottle_neck', feature, 64)
with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE):
ms_feature_1 = self.upsample(feature, out_channel=64)
with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE):
ms_feature_2 = self.upsample(ms_feature_1, out_channel=64)
return ms_feature_1, ms_feature_2
def feature_fusion(self, name, ms_feature, pan_feature):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
fusion_feature = tf.concat([ms_feature, pan_feature], axis=-1)
channel_in = fusion_feature.get_shape().as_list()[-1]
fusion_feature = conv('conv1', fusion_feature, filter_num=96)
fusion_feature = conv('conv2', fusion_feature, filter_num=64)
detail = conv('conv3', fusion_feature, 4, activation=None)
return detail
def forward(self, ms, pan):
with tf.variable_scope('PNN', reuse=tf.AUTO_REUSE):
up_ms = tf.image.resize_bicubic(ms,
size=(self.ms_size * 4,
self.ms_size * 4))
x = tf.concat([up_ms, pan], -1)
with tf.variable_scope('opt1', reuse=tf.AUTO_REUSE):
feature = conv('conv1', x, 64, 9, activation='lrelu')
feature = conv('conv2', feature, 32, 5, activation='lrelu')
with tf.variable_scope('opt2', reuse=tf.AUTO_REUSE):
fusion=conv('conv3',feature,4,5,activation=None)\
+up_ms
return fusion
def pan_feature_extraction(self, name, pan):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
feature = pan
for i in range(8):
feature = conv('conv%d' % (i + 1), feature, 64, 5)
with tf.variable_scope('downsample', reuse=tf.AUTO_REUSE):
feature1 = self.downsample(feature, 64)
return feature1, feature
def ms_feature_extraction(self, name, ms):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
feature = ms
for i in range(6):
feature = conv('conv%d' % (i + 1), feature, 64)
with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE):
ms_feature_1 = self.upsample(feature, out_channel=64)
with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE):
ms_feature_2 = self.upsample(ms_feature_1, out_channel=64)
return ms_feature_1, ms_feature_2
def forward(self, ms, pan):
with tf.variable_scope('RSIFNN', reuse=tf.AUTO_REUSE):
with tf.variable_scope('ms', reuse=tf.AUTO_REUSE):
up_ms = tf.image.resize_bicubic(
ms, (self.ms_size * 4, self.ms_size * 4))
feature_ms = conv('conv1', up_ms, activation='lrelu')
feature_ms = conv('conv2',
feature_ms,
filter_num=32,
activation='lrelu')
with tf.variable_scope('pan', reuse=tf.AUTO_REUSE):
feature_pan = pan
for i in range(1, 8):
feature_pan = conv('conv%d' % (i),
feature_pan,
activation='lrelu')
feature_pan = conv('conv%d' % (i + 1),
feature_pan,
filter_num=32,
activation='lrelu')
with tf.variable_scope('fusion', reuse=tf.AUTO_REUSE):
features = tf.concat([feature_ms, feature_pan], -1)
return up_ms + conv('conv', features, 4, activation=None)
def pan_feature_extraction(self, name, pan):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
l1, l2, l3 = LapFusionConfig.pan_depth
l1, l2 = l1 - 1, l2 - 1
feature_channel = LapFusionConfig.pan_featurec_channel
with tf.variable_scope('block1', reuse=tf.AUTO_REUSE):
feature = pan
for i in range(1, l1 + 1):
feature = conv('conv%d' % (i), feature, feature_channel[0])
with tf.variable_scope('down_sample1', reuse=tf.AUTO_REUSE):
feature = self.downsample(feature,
out_channel=feature_channel[0])
with tf.variable_scope('block2', reuse=tf.AUTO_REUSE):
for i in range(1, l2 + 1):
feature = conv('conv%d' % (i), feature, feature_channel[1])
# res=feature
# for i in range(1,l2):
# res=conv('conv%d'%(i),res,feature_channel[1])
# res=conv('conv%d'%(i+1),res,64,activation=None)
# feature=tf.nn.leaky_relu(res+feature)
with tf.variable_scope('down_sample2', reuse=tf.AUTO_REUSE):
feature = self.downsample(feature,
out_channel=feature_channel[1])
with tf.variable_scope('block3', reuse=tf.AUTO_REUSE):
for i in range(1, l3 + 1):
feature = conv('conv%d' % (i), feature, feature_channel[1])
# res=feature
# for i in range(1,l3):
# res=conv('conv%d'%(i),res,feature_channel[2])
# res=conv('conv%d'%(i+1),res,feature_channel[2],activation=None)
# feature=tf.nn.leaky_relu(res+feature)
with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE):
pan_feature_1 = self.upsample(feature, out_channel=64)
with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE):
pan_feature_2 = self.upsample(pan_feature_1, out_channel=64)
return pan_feature_1, pan_feature_2
def ms_feature_extraction(self, name, ms):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
with tf.variable_scope('low_level_extraction',
reuse=tf.AUTO_REUSE):
feature = ms
feature = conv('conv1', feature, 32)
feature = conv('conv2', feature, 32)
feature = dense_block('dense_block1',
feature,
12,
conv_num=4,
input_include=True)
feature = bottle_neck('bottle_neck1', feature, 64)
feature = dense_block('dense_block2',
feature,
12,
conv_num=6,
input_include=True)
feature = bottle_neck('bottle_neck2', feature, 96)
with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE):
ms_feature_1 = self.upsample(feature, out_channel=96)
with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE):
ms_feature_2 = self.upsample(ms_feature_1, out_channel=96)
return ms_feature_1, ms_feature_2
def ms_feature_extraction(self, name, ms):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
# feature=conv('conv1',ms,LapFusionConfig.ms_feature_channel)
# feature=conv('conv2',feature,LapFusionConfig.ms_feature_channel)
# res=feature
# for i in range(3,8):
# res=conv('conv%d'%(i),res,LapFusionConfig.ms_feature_channel)
# res=conv('conv%d'%(i+1),res,LapFusionConfig.ms_feature_channel,activation=None)
# feature=tf.nn.leaky_relu(feature+res)
feature = ms
for i in range(1, LapFusionConfig.ms_depth + 1):
feature = conv('conv%d' % (i), feature,
LapFusionConfig.ms_feature_channel)
with tf.variable_scope('upsample1', reuse=tf.AUTO_REUSE):
ms_feature_1 = self.upsample(feature, out_channel=64)
with tf.variable_scope('upsample2', reuse=tf.AUTO_REUSE):
ms_feature_2 = self.upsample(ms_feature_1, out_channel=64)
return ms_feature_1, ms_feature_2
def downsample(self, feature, out_channel, scale=2):
# with tf.variable_scope(name,reuse=tf.AUTO_REUSE):
return conv('down_conv', feature, out_channel, strides=scale)
def yolov4(input_img):
net = module.conv(input_img, 32, 3, 1)
net = module.conv(net, 64, 3, 2)
route = module.conv(net, 64, 1, 1)
net = module.conv(net, 64, 1, 1)
tmp = net
net = module.conv(net, 32, 1, 1)
net = module.conv(net, 64, 3, 1)
net = tmp + net
net = module.conv(net, 64, 1, 1)
net = tf.concat([net, route], -1)
net = module.conv(net, 64, 1, 1)
net = module.conv(net, 128, 3, 2)
route = module.conv(net, 64, 1, 1)
net = module.conv(net, 64, 1, 1)
for _ in range(2):
tmp = net
net = module.conv(net, 64, 1, 1)
net = module.conv(net, 64, 3, 1)
net = tmp + net
net = module.conv(net, 64, 1, 1)
net = tf.concat([net, route], -1)
net = module.conv(net, 128, 1, 1)
net = module.conv(net, 256, 3, 2)
route = module.conv(net, 128, 1, 1)
net = module.conv(net, 128, 1, 1)
for _ in range(8):
tmp = net
net = module.conv(net, 128, 1, 1)
net = module.conv(net, 128, 3, 1)
net = tmp + net
net = module.conv(net, 128, 1, 1)
net = tf.concat([net, route], -1)
net = module.conv(net, 256, 1, 1)
for_output1 = net
net = module.conv(net, 512, 3, 2)
route = module.conv(net, 256, 1, 1)
net = module.conv(net, 256, 1, 1)
for _ in range(8):
tmp = net
net = module.conv(net, 256, 1, 1)
net = module.conv(net, 256, 3, 1)
net = tmp + net
net = module.conv(net, 256, 1, 1)
net = tf.concat([net, route], -1)
net = module.conv(net, 512, 1, 1)
for_output2 = net
net = module.conv(net, 1024, 3, 2)
route = module.conv(net, 512, 1, 1)
net = module.conv(net, 512, 1, 1)
for _ in range(4):
tmp = net
net = module.conv(net, 512, 1, 1)
net = module.conv(net, 512, 3, 1)
net = tmp + net
net = module.conv(net, 512, 1, 1)
net = tf.concat([net, route], -1)
net = module.conv(net, 1024, 1, 1)
# SPP
pool_5 = tf.nn.max_pool(net, [1, 5, 5, 1], [1, 1, 1, 1], 'SAME')
pool_9 = tf.nn.max_pool(net, [1, 9, 9, 1], [1, 1, 1, 1], 'SAME')
pool_13 = tf.nn.max_pool(net, [1, 13, 13, 1], [1, 1, 1, 1], 'SAME')
net = tf.concat([pool_13, pool_9, pool_5, net], -1)
net = module.conv(net, 512, 1, 1)
net = module.conv(net, 1024, 3, 1)
net = module.conv(net, 512, 1, 1)
output3 = net
net = module.conv(net, 256, 1, 1)
shape = tf.shape(net)
out_height, out_width = shape[1]*2, shape[2]*2
net = tf.compat.v1.image.resize_nearest_neighbor(net, (out_height, out_width))
for_output2 = module.conv(for_output2, 256, 1, 1)
net = tf.concat([for_output2, net], -1)
net = module.conv(net, 256, 1, 1)
net = module.conv(net, 512, 3, 1)
net = module.conv(net, 256, 1, 1)
net = module.conv(net, 512, 3, 1)
net = module.conv(net, 256, 1, 1)
output2 = net
net = module.conv(net, 128, 1, 1)
shape = tf.shape(net)
out_height, out_width = shape[1]*2, shape[2]*2
net = tf.compat.v1.image.resize_nearest_neighbor(net, (out_height, out_width))
for_output1 = module.conv(for_output1, 128, 1, 1)
net = tf.concat([for_output1, net], -1)
net = module.conv(net, 128, 1, 1)
net = module.conv(net, 256, 3, 1)
net = module.conv(net, 128, 1, 1)
net = module.conv(net, 256, 3, 1)
net = module.conv(net, 128, 1, 1)
output1 = net
net = module.conv(output1, 256, 3, 1)
net = module.conv(net, 3*(4+1+config.class_num), 1, 1,
normalizer_fn=None, activation_fn=None,
biases_initializer=tf.zeros_initializer())
label3 = net
net = module.conv(output1, 256, 3, 2)
net = tf.concat([net, output2], -1)
net = module.conv(net, 256, 1, 1)
net = module.conv(net, 512, 3, 1)
net = module.conv(net, 256, 1, 1)
net = module.conv(net, 512, 3, 1)
net = module.conv(net, 256, 1, 1)
tmp_route = net
net = module.conv(net, 512, 3, 1)
net = module.conv(net, 3*(4+1+config.class_num), 1, 1,
normalizer_fn=None, activation_fn=None,
biases_initializer=tf.zeros_initializer())
label2 = net
net = module.conv(tmp_route, 512, 3, 2)
net = tf.concat([net, output3], -1)
net = module.conv(net, 512, 1, 1)
net = module.conv(net, 1024, 3, 1)
net = module.conv(net, 512, 1, 1)
net = module.conv(net, 1024, 3, 1)
net = module.conv(net, 512, 1, 1)
net = module.conv(net, 1024, 3, 1)
net = module.conv(net, 3*(4+1+config.class_num), 1, 1,
normalizer_fn=None, activation_fn=None,
biases_initializer=tf.zeros_initializer())
label1 = net
return label1, label2, label3