def Bisnet(input_height, input_width, n_classes=20):
input_image = tf.keras.Input(shape=(input_height, input_width, 3),
name="input_image",
dtype=tf.float32)
# x = Lambda(lambda image: preprocess_input(image))(inputs)
x = input_image
xception = Xception(weights='imagenet', input_tensor=x, include_top=False)
tail_prev = xception.get_layer('block13_pool').output
tail = xception.output
layer_13, layer_14 = tail_prev, tail
x = ConvAndBatch(x, 32, strides=2)
x = ConvAndBatch(x, 64, strides=2)
x = ConvAndBatch(x, 156, strides=2)
# context path
cp = ContextPath(layer_13, layer_14)
# fusion = self.FeatureFusionModule(cp, x, 32)
fusion = FeatureFusionModule(cp, x, n_classes)
ans = UpSampling2D(size=(8, 8), interpolation='bilinear')(fusion)
output = tf.keras.layers.Softmax(axis=3, dtype=tf.float32)(ans)
return tf.keras.Model(inputs=input_image, outputs=output, name="bisnet")
def pretrainedmodel(classes,
name_model='VGG16',
use_l2=True,
is_trainable=False):
models = ['VGG16', 'ResNet50', 'InceptionV3', 'Xception']
if name_model not in models:
print('Name model not found. Try {}'.format(models))
return
if name_model == 'VGG16':
expected_dim = (224, 224, 3)
vgg16 = VGG16(weights='imagenet', include_top=False, pooling='avg')
basemodel = Model(inputs=vgg16.inputs,
outputs=vgg16.get_layer(index=-1).output)
if name_model == 'ResNet50':
expected_dim = (224, 224, 3)
resnet = ResNet50(weights='imagenet', include_top=False, pooling='avg')
basemodel = Model(inputs=resnet.inputs,
outputs=resnet.get_layer(index=-1).output)
if name_model == 'InceptionV3':
expected_dim = (299, 299, 3)
iv3 = InceptionV3(weights='imagenet', include_top=False, pooling='avg')
basemodel = Model(inputs=iv3.inputs,
outputs=iv3.get_layer(index=-1).output)
if name_model == 'Xception':
expected_dim = (299, 299, 3)
xcep = Xception(weights='imagenet', include_top=False, pooling='avg')
basemodel = Model(inputs=xcep.inputs,
outputs=xcep.get_layer(index=-1).output)
# freeze weights
basemodel.trainable = is_trainable
x = basemodel.output
# add l2-norm constraint
if use_l2:
x = L2Layer()(x)
# predictions
y = Dense(classes, activation='softmax', name='predictions')(x)
model = Model(inputs=basemodel.input, outputs=y)
return model, expected_dim
def build_Xception_pretrain(img_shape=(128,128,3),num_classes=1, lr=0.001):
base_model = Xception(include_top=False, input_shape = img_shape,weights='imagenet')
base_model.trainable = False
x = base_model.get_layer('avg_pool').output
outputs = Dense(num_classes
,kernel_initializer = "he_normal"
, activation='sigmoid')(x)
final_model = Model(inputs=[base_model.input], outputs=[outputs])
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0)
final_model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
return final_model
class FTCF_Net(tf.keras.Model):
def __init__(self, layer_name='add_3', weight_decay=0.005):
super(FTCF_Net, self).__init__()
self.weight_decay = weight_decay
self.layer_name = layer_name
self.basemodel = Xception(weights='imagenet',
input_shape=(224, 224, 3),
include_top=False)
# Feature Extractor for Every Frames
self.backbone = tf.keras.Model(inputs=self.basemodel.input,
outputs=self.basemodel.get_layer(
self.layer_name).output)
self.backbone.trainable = False
self.cbm1 = CBM(filters=364, kernel=(1, 1, 2))
# Fusion Stage 1
self.ftcf1 = FTCF_Block(32)
self.avgpooling3d_1 = AveragePooling3D((2, 2, 2),
strides=(2, 2, 2),
padding='same')
# Fusion Stage 2
self.ftcf2 = FTCF_Block(32)
self.avgpooling3d_2 = AveragePooling3D((2, 2, 2),
strides=(2, 2, 2),
padding='same')
# Fusion Stage 3
self.ftcf3 = FTCF_Block(64)
self.avgpooling3d_3 = AveragePooling3D((2, 2, 2),
strides=(2, 2, 2),
padding='same')
self.cbm2 = CBM(filters=128, kernel=(2, 2, 2), padding='valid')
self.globalpooling3d = GlobalAveragePooling3D()
self.dense_2 = Dense(512,
activation='relu',
kernel_regularizer=l2(self.weight_decay))
self.dropout_2 = Dropout(0.5)
self.output_tensor = Dense(2,
activation='softmax',
kernel_regularizer=l2(self.weight_decay))
def mish(self, x):
return x * tf.nn.tanh(tf.nn.softplus(x))
def call(self, input_tensor, training=True):
single_frames = []
for i in range(15):
frame = self.backbone(input_tensor[:, :, :, i, :], training=False)
frame = tf.expand_dims(frame, axis=3)
single_frames.append(frame)
x = concatenate(single_frames, axis=3)
x = self.mish(x)
x = self.cbm1(x, training=training)
x = self.ftcf1(x, training=training)
x = self.avgpooling3d_1(x)
x = self.ftcf2(x, training=training)
x = self.avgpooling3d_2(x)
x = self.ftcf3(x, training=training)
x = self.avgpooling3d_3(x)
x = self.cbm2(x, training=training)
x = self.globalpooling3d(x)
x = self.dense_2(x)
x = self.dropout_2(x)
return self.output_tensor(x)
def __init__(self, conf):
"""
Parameters
----------
conf: dictionary
Semantic segmentation model configuration dictionary.
"""
# Check exception.
assert conf['nn_arch']['output_stride'] == 8 or conf['nn_arch'][
'output_stride'] == 16
# Initialize.
self.conf = conf
self.raw_data_path = self.conf['raw_data_path']
self.hps = self.conf['hps']
self.nn_arch = self.conf['nn_arch']
self.model_loading = self.conf['model_loading']
if self.model_loading:
opt = optimizers.Adam(lr=self.hps['lr'],
beta_1=self.hps['beta_1'],
beta_2=self.hps['beta_2'],
decay=self.hps['decay'])
with CustomObjectScope({
'CategoricalCrossentropyWithLabelGT':
CategoricalCrossentropyWithLabelGT,
'MeanIoUExt': MeanIoUExt
}):
if self.conf['multi_gpu']:
self.model = load_model(
os.path.join(self.raw_data_path, self.MODEL_PATH))
self.parallel_model = multi_gpu_model(
self.model, gpus=self.conf['num_gpus'])
self.parallel_model.compile(optimizer=opt,
loss=self.model.losses,
metrics=self.model.metrics)
else:
self.model = load_model(
os.path.join(self.raw_data_path, self.MODEL_PATH))
# self.model.compile(optimizer=opt,
# , loss=CategoricalCrossentropyWithLabelGT(num_classes=self.nn_arch['num_classes'])
# , metrics=[MeanIoUExt(num_classes=NUM_CLASSES)]
else:
# Design the semantic segmentation model.
# Load a base model.
if self.conf['base_model'] == BASE_MODEL_MOBILENETV2:
# Load mobilenetv2 as the base model.
mv2 = MobileNetV2(
include_top=False
) # , depth_multiplier=self.nn_arch['mv2_depth_multiplier'])
if self.nn_arch['output_stride'] == 8:
self.base = Model(
inputs=mv2.inputs,
outputs=mv2.get_layer('block_5_add').output
) # Layer satisfying output stride of 8.
else:
self.base = Model(
inputs=mv2.inputs,
outputs=mv2.get_layer('block_12_add').output
) # Layer satisfying output stride of 16.
self.base.trainable = True
for layer in self.base.layers:
layer.trainable = True # ?
self.base._init_set_name('base')
elif self.conf['base_model'] == BASE_MODEL_XCEPTION:
# Load xception as the base model.
mv2 = Xception(
include_top=False
) # , depth_multiplier=self.nn_arch['mv2_depth_multiplier'])
if self.nn_arch['output_stride'] == 8:
self.base = Model(
inputs=mv2.inputs,
outputs=mv2.get_layer('block4_sepconv2_bn').output
) # Layer satisfying output stride of 8.
else:
self.base = Model(
inputs=mv2.inputs,
outputs=mv2.get_layer('block13_sepconv2_bn').output
) # Layer satisfying output stride of 16.
self.base.trainable = True
for layer in self.base.layers:
layer.trainable = True # ?
self.base._init_set_name('base')
# Make the encoder-decoder model.
self._make_encoder()
self._make_decoder()
inputs = self.encoder.inputs
features = self.encoder(inputs)
outputs = self.decoder([inputs[0], features]) if self.nn_arch['boundary_refinement'] \
else self.decoder(features)
self.model = Model(inputs, outputs)
# Compile.
opt = optimizers.Adam(lr=self.hps['lr'],
beta_1=self.hps['beta_1'],
beta_2=self.hps['beta_2'],
decay=self.hps['decay'])
self.model.compile(optimizer=opt,
loss=CategoricalCrossentropyWithLabelGT(
num_classes=self.nn_arch['num_classes']),
metrics=[MeanIoUExt(num_classes=NUM_CLASSES)])
self.model._init_set_name('deeplabv3plus_mnv2')
if self.conf['multi_gpu']:
self.parallel_model = multi_gpu_model(
self.model, gpus=self.conf['num_gpus'])
self.parallel_model.compile(optimizer=opt,
loss=self.model.losses,
metrics=self.model.metrics)
def model_ctpn(self):
input_layer = Input(shape=self.image_shape,
batch_size=self.bs,
name='input_1')
if self.base_model_name.lower(
) == 'vgg16' or self.base_model_name == None:
self.scale = 16
base_model = VGG16(include_top=False, input_tensor=input_layer)
base_model.trainable = True
# input_layer=base_model.input
base_output_layer = base_model.get_layer('block5_conv3').output
elif self.base_model_name.lower() == 'resnet50':
base_model = ResNet50(include_top=False, input_tensor=input_layer)
self.scale = 32
base_model.trainable = True
# input_layer=base_model.input
base_output_layer = base_model.output
elif self.base_model_name.lower() == 'inception':
base_model = InceptionResNetV2(include_top=False,
input_tensor=input_layer)
base_model.trainable = True
# input_layer=base_model.input
base_output_layer = base_model.output
self.scale = 32
elif self.base_model_name.lower() == 'xception':
base_model = Xception(include_top=False, input_tensor=input_layer)
base_model.trainable = True
# input_layer = base_model.input
base_output_layer = base_model.output
self.scale = 32
else:
base_model = DenseNet121(include_top=False,
input_tensor=input_layer)
base_model.trainable = True
# input_layer = base_model.input
base_output_layer = base_model.get_layer(
'conv5_block16_concat').output
self.scale = 32
layer_base = layers.Conv2D(
512, (3, 3),
strides=(1, 1),
padding='same',
activation='relu',
name='share_layer',
kernel_initializer='he_normal')(base_output_layer)
#这里得到share layer之后,把输出层(bs,h,w,c)->(bs*w,h,c),因为我们是垂直列识别，把每一个width的列提出进行训练
shape = layer_base.get_shape()
layer = layers.Lambda(backend_reshape,
output_shape=(shape[1], shape[-1]),
name='lambda_reshape_1')(layer_base)
layer = layers.Bidirectional(layers.LSTM(
128, return_sequences=True, kernel_initializer='he_normal'),
merge_mode='concat')(layer)
layer = layers.Lambda(backend_reshape_2,
output_shape=(shape[1], shape[2], 256),
name='lambda_reshape_2')([layer, layer_base])
layer = layers.Conv2D(128, (1, 1),
padding='same',
activation='relu',
kernel_initializer='he_normal',
name='lstm_fc')(layer)
cls = layers.Conv2D(self.k * 2, (1, 1),
padding='same',
kernel_initializer='he_normal',
activation='linear',
name='rpn_class')(layer)
regr = layers.Conv2D(self.k * 2, (1, 1),
padding='same',
kernel_initializer='he_normal',
activation='linear',
name='rpn_regr')(layer)
shape2 = cls.get_shape()
# #bs,H*W*k,2
cls = layers.Reshape(target_shape=(shape2[1] * shape2[2] * self.k, 2),
batch_size=shape2[0],
name='rpn_class_finnal')(cls)
cls_prob = layers.Activation('softmax', name='rpn_cls_softmax')(cls)
shape3 = regr.get_shape()
regr = layers.Reshape(target_shape=(shape3[1] * shape3[2] * self.k, 2),
batch_size=shape3[0],
name='rpn_regr_finnal')(regr)
predict_model = Model(input_layer, [cls, regr, cls_prob])
train_model = Model(input_layer, [cls, regr])
train_model.summary()
return train_model, predict_model
