def deep_supervised(backbone_name='',
input_shape=(144, 912, 3),
encoder_weights='imagenet',
activation='sigmoid',
classes=4):
backbone_name = backbone_name[4:]
model = PSPNet(backbone_name=backbone_name,
input_shape=input_shape,
encoder_weights=encoder_weights,
activation=activation,
classes=classes)
def classification_model(feature_layer):
feature_layer = GlobalAveragePooling2D()(feature_layer)
feature_layer = Dropout(0.25)(feature_layer)
classification = Dense(64, activation='sigmoid',
use_bias=False)(feature_layer)
return Dense(5, activation='sigmoid', use_bias=False)(classification)
if backbone_name.startswith('seresnet'):
feature_layer = model.layers[get_feature_layers(backbone_name)
[1]].output
else:
feature_layer = model.get_layer(
get_feature_layers(backbone_name)[1]).output
classification = classification_model(feature_layer)
def resize_to_fit(classification):
classification = ZeroPadding1D(
(0, input_shape[0] - 5))(classification[...,
np.newaxis])[...,
np.newaxis]
classification = K.repeat_elements(classification, 4, axis=-1)
return classification
classification = Lambda(resize_to_fit)(classification)
output = Concatenate(axis=2)([classification, model.output])
return Model(inputs=model.input, outputs=output)
def build(x):
tensor = x
if padding_fix:
tensor = PadToMultiple(32)(tensor)
tensor = layers.Lambda(lambda v: K.concatenate([v, v, v], axis=-1))(tensor)
encoder_backbone = get_backbone(backbone_name,
input_shape=(None, None, 3),
input_tensor=tensor,
weights=weights,
include_top=False)
feature_layer_names = get_feature_layers(backbone_name, n=4)
backbone_model = build_unet(encoder_backbone, 1, feature_layer_names)
tensor = backbone_model.get_layer('decoder_stage4_relu2').output
if padding_fix:
return CropLike()([tensor, x])
return tensor
def get_maskrcnn_feature_layers(config, backbone):
try:
feature_layers = []
# We want our feature layers to be listed in decreasing output size (width, height)
if config.FEATURE_LAYERS == 'default':
layer_names = get_feature_layers(config.BACKBONE)
else:
layer_names = config.FEATURE_LAYERS
for l in reversed(layer_names[:-1]):
feature_layers.append(l)
# and we use the last backbone layer output instead of the last layer in DEFAULT_FEATURE_LAYERS
feature_layers.append(backbone.layers[-1].name)
return feature_layers
except KeyError:
print('No default feature layers for {} provided yet.',
'Please specify the layer names (or indexes) explicitly in config file'.format(config.BACKBONE))
exit(1)
def get_backbone_and_feature_layers(self, num_feature_layers):
super(SemanticModelWrapper, self).build()
image_shape = self.config.IMAGE_SHAPE
classifier, self.preprocess_input = Classifiers.get(
self.config.BACKBONE)
backbone = classifier(input_shape=image_shape,
input_tensor=None,
weights=self.config.BACKBONE_WEIGHTS,
include_top=False)
for layer in backbone.layers:
self.backbone_layer_names.append(layer.name)
if self.config.FEATURE_LAYERS == 'default':
feature_layers = get_feature_layers(self.config.BACKBONE,
n=num_feature_layers)
else:
feature_layers = self.config.FEATURE_LAYERS
return backbone, feature_layers
def save_model_to_json(modeljsonfname):
layers = get_feature_layers(backbone_name, 4)
if backbone_type == 'FPN':
model = FPN(input_shape=(None, None, num_channels),
classes=num_mask_channels,
encoder_weights=encoder_weights,
backbone_name=backbone_name,
activation=act_fcn,
encoder_features=layers)
elif backbone_type == 'Unet':
model = Unet(input_shape=(None, None, num_channels),
classes=num_mask_channels,
encoder_weights=encoder_weights,
backbone_name=backbone_name,
activation=act_fcn,
encoder_features=layers)
#model.summary()
# serialize model to JSON
model_json = model.to_json()
with open(modeljsonfname, "w") as json_file:
json_file.write(model_json)
def FPN(backbone_name='vgg16',
input_shape=(None, None, 3),
input_tensor=None,
classes=21,
activation='softmax',
encoder_weights='imagenet',
encoder_freeze=False,
encoder_features='default',
pyramid_block_filters=256,
pyramid_use_batchnorm=True,
pyramid_dropout=None,
final_interpolation='bilinear',
norm_type='BN',
**kwargs):
"""FPN_ is a fully convolution neural network for image semantic segmentation
Args:
backbone_name: name of classification model (without last dense layers) used as feature
extractor to build segmentation model.
input_shape: shape of input data/image ``(H, W, C)``, in general
case you do not need to set ``H`` and ``W`` shapes, just pass ``(None, None, C)`` to make your model be
able to process images af any size, but ``H`` and ``W`` of input images should be divisible by factor ``32``.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model
(works only if ``encoder_weights`` is ``None``).
classes: a number of classes for output (output shape - ``(h, w, classes)``).
activation: name of one of ``keras.activations`` for last model layer (e.g. ``sigmoid``, ``softmax``, ``linear``).
encoder_weights: one of ``None`` (random initialization), ``imagenet`` (pre-training on ImageNet).
encoder_freeze: if ``True`` set all layers of encoder (backbone model) as non-trainable.
encoder_features: a list of layer numbers or names starting from top of the model.
Each of these layers will be used to build features pyramid. If ``default`` is used
layer names are taken from ``DEFAULT_FEATURE_PYRAMID_LAYERS``.
pyramid_block_filters: a number of filters in Feature Pyramid Block of FPN_.
pyramid_use_batchnorm: if ``True``, ``BatchNormalisation`` layer between ``Conv2D`` and ``Activation`` layers
is used.
pyramid_dropout: spatial dropout rate for feature pyramid in range (0, 1).
final_interpolation: interpolation type for upsampling layers, on of ``nearest``, ``bilinear``.
Returns:
``keras.models.Model``: **FPN**
.. _FPN:
http://presentations.cocodataset.org/COCO17-Stuff-FAIR.pdf
"""
backbone = get_backbone(backbone_name,
input_shape=input_shape,
input_tensor=input_tensor,
weights=encoder_weights,
include_top=False)
if encoder_features == 'default':
encoder_features = get_feature_layers(backbone_name, n=3)
upsample_rates = [2] * len(encoder_features)
last_upsample = 2**(5 - len(encoder_features))
model = build_fpn(backbone,
encoder_features,
classes=classes,
pyramid_filters=pyramid_block_filters,
segmentation_filters=pyramid_block_filters // 2,
upsample_rates=upsample_rates,
use_batchnorm=pyramid_use_batchnorm,
dropout=pyramid_dropout,
last_upsample=last_upsample,
interpolation=final_interpolation,
activation=activation,
norm_type=norm_type)
if encoder_freeze:
freeze_model(backbone)
model.name = 'fpn-{}'.format(backbone.name)
return model
def train_generatorh5(params):
from hitif_losses import dice_coef_loss_bce
from hitif_losses import double_head_loss
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
# Prepare for splitting the training set
imgs_ind = np.arange(number_of_imgs)
np.random.shuffle(imgs_ind)
# Split 80-20
train_last_id = int(number_of_imgs * 0.80)
# Generators
training_generator = DataGeneratorH5(
source_target_list_IDs=imgs_ind[0:train_last_id].copy(), **params)
validation_generator = DataGeneratorH5(
source_target_list_IDs=imgs_ind[train_last_id:number_of_imgs].copy(),
**params)
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
layers = get_feature_layers(backbone_name, 4)
if backbone_type == 'FPN':
model = FPN(input_shape=(None, None, num_channels),
classes=num_mask_channels,
encoder_weights=encoder_weights,
encoder_freeze=freezeFlag,
backbone_name=backbone_name,
activation=act_fcn,
encoder_features=layers)
elif backbone_type == 'Unet':
model = Unet(input_shape=(None, None, num_channels),
classes=num_mask_channels,
encoder_weights=encoder_weights,
encoder_freeze=freezeFlag,
backbone_name=backbone_name,
activation=act_fcn,
encoder_features=layers)
#model.summary()
#model.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy', metrics=['binary_crossentropy','mean_squared_error',dice_coef, dice_coef_batch, dice_coef_loss_bce,focal_loss()])
#model.compile(optimizer=Adam(lr=1e-3), loss=dice_coef_loss_bce, metrics=['binary_crossentropy','mean_squared_error',dice_coef, dice_coef_batch,focal_loss()])
#model.compile(optimizer=Adam(lr=1e-3), loss=loss_fcn, metrics=['binary_crossentropy','mean_squared_error',dice_coef, dice_coef_batch,focal_loss()])
if loss_fcn == 'dice_coef_loss_bce':
model.compile(optimizer=Adam(lr=1e-3), loss=dice_coef_loss_bce)
elif loss_fcn == 'double_head_loss':
model.compile(optimizer=Adam(lr=1e-3), loss=double_head_loss)
else:
model.compile(optimizer=Adam(lr=1e-3), loss=loss_fcn)
# Loading previous weights for restarting
if oldmodelwtsfname is not None:
if os.path.isfile(oldmodelwtsfname) and reloadFlag:
print('-' * 30)
print('Loading previous weights ...')
weights = np.load(oldmodelwtsfname, allow_pickle=True)
model.set_weights(weights)
#model.load_weights(oldmodelwtsfname)
checkpoint_path = get_checkpoint_path(log_dir_name)
print("checkpoint_path:", checkpoint_path)
model_checkpoint = ModelCheckpoint(checkpoint_path,
monitor='val_loss',
save_best_only=True,
save_weights_only=True)
custom_checkpoint = Checkpoints(checkpoint_path,
monitor='val_loss',
verbose=1)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=25,
min_lr=1e-6,
verbose=1,
restore_best_weights=True)
model_es = EarlyStopping(monitor='val_loss',
min_delta=1e-7,
patience=15,
verbose=1,
mode='auto')
csv_logger = CSVLogger(csvfname, append=True)
#my_callbacks = [reduce_lr, model_es, csv_logger]
#my_callbacks = [model_checkpoint, reduce_lr, model_es, csv_logger]
my_callbacks = [custom_checkpoint, reduce_lr, model_es, csv_logger]
print('-' * 30)
print('Fitting model encoder freeze...')
print('-' * 30)
if freezeFlag and num_coldstart_epochs > 0:
model.fit_generator(generator=training_generator,
validation_data=validation_generator,
use_multiprocessing=True,
workers=num_gen_workers,
epochs=num_coldstart_epochs,
callbacks=my_callbacks,
verbose=2)
# release all layers for training
set_trainable(model) # set all layers trainable and recompile model
#model.summary()
print('-' * 30)
print('Fitting full model...')
print('-' * 30)
## Retrain after the cold-start
model.fit_generator(generator=training_generator,
validation_data=validation_generator,
use_multiprocessing=True,
workers=num_gen_workers,
epochs=num_finetuning_epochs,
callbacks=my_callbacks,
verbose=2)
## <<FIXME>>: GZ will work on it.
# Find the last best epoch model weights and then symlink it to the modelwtsfname
# Note that the symlink will have issues on NON-Linux OS so it is better to copy.
'''
def SiameseUnet(backbone_name='vgg16',
input_shape=(None, None, 3),
classes=1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=False,
encoder_features='default',
decoder_block_type='upsampling',
decoder_filters=(256, 128, 64, 32, 16),
decoder_use_batchnorm=True,
**kwargs):
""" Unet_ is a fully convolution neural network for image semantic segmentation
Args:
backbone_name: name of classification model (without last dense layers) used as feature
extractor to build segmentation model.
input_shape: shape of input data/image ``(H, W, C)``, in general
case you do not need to set ``H`` and ``W`` shapes, just pass ``(None, None, C)`` to make your model be
able to process images af any size, but ``H`` and ``W`` of input images should be divisible by factor ``32``.
classes: a number of classes for output (output shape - ``(h, w, classes)``).
activation: name of one of ``keras.activations`` for last model layer
(e.g. ``sigmoid``, ``softmax``, ``linear``).
encoder_weights: one of ``None`` (random initialization), ``imagenet`` (pre-training on ImageNet).
encoder_freeze: if ``True`` set all layers of encoder (backbone model) as non-trainable.
encoder_features: a list of layer numbers or names starting from top of the model.
Each of these layers will be concatenated with corresponding decoder block. If ``default`` is used
layer names are taken from ``DEFAULT_SKIP_CONNECTIONS``.
decoder_block_type: one of blocks with following layers structure:
- `upsampling`: ``Upsampling2D`` -> ``Conv2D`` -> ``Conv2D``
- `transpose`: ``Transpose2D`` -> ``Conv2D``
decoder_filters: list of numbers of ``Conv2D`` layer filters in decoder blocks
decoder_use_batchnorm: if ``True``, ``BatchNormalisation`` layer between ``Conv2D`` and ``Activation`` layers
is used.
Returns:
``keras.models.Model``: **Unet**
.. _Unet:
https://arxiv.org/pdf/1505.04597
"""
load_weights_from = None
if encoder_weights is not "imagenet" and encoder_weights is not None:
load_weights_from = encoder_weights
encoder_weights = None
backbone = get_backbone(backbone_name,
input_shape=input_shape,
input_tensor=None,
weights=encoder_weights,
include_top=False)
if load_weights_from is not None:
model_to_load_weights_from = load_model(load_weights_from)
# now let's assume that this loaded model had its own "top" upsampling section trained on another task
# let's transplant what we can, that is the backbone encoder
output = model_to_load_weights_from.layers[
len(backbone.layers) -
1].output # remove activation and last conv layer
transplant = keras.models.Model(model_to_load_weights_from.input,
output)
#transplant.summary()
transplant.save("transplant.h5") # hacky way
backbone.load_weights("transplant.h5")
# Check if the weights have been loaded
"""
inspect_i = 0
import numpy as np
w1 = np.asarray(transplant.get_weights()[inspect_i])
print(w1)
w2 = np.asarray(backbone.get_weights()[inspect_i])
print(w2)
"""
print("Loaded weights into ", backbone_name, "from", load_weights_from)
if encoder_features == 'default':
encoder_features = get_feature_layers(backbone_name, n=4)
model = build_siamese_unet(backbone,
classes,
encoder_features,
decoder_filters=decoder_filters,
block_type=decoder_block_type,
activation=activation,
n_upsample_blocks=len(decoder_filters),
upsample_rates=(2, 2, 2, 2, 2),
use_batchnorm=decoder_use_batchnorm,
input_shape=input_shape)
# lock encoder weights for fine-tuning
if encoder_freeze:
freeze_model(backbone)
model.name = 'u-{}'.format(backbone_name)
return model
def Unet(backbone_name='vgg16',
input_shape=(None, None, 3),
classes=1,
activation='sigmoid',
encoder_weights='imagenet',
encoder_freeze=False,
encoder_features='default',
decoder_block_type='upsampling',
decoder_filters=(256, 128, 64, 32, 16),
decoder_use_batchnorm=True,
**kwargs):
""" Unet_ is a fully convolution neural network for image semantic segmentation
Args:
backbone_name: name of classification model (without last dense layers) used as feature
extractor to build segmentation model.
input_shape: shape of input data/image ``(H, W, C)``, in general
case you do not need to set ``H`` and ``W`` shapes, just pass ``(None, None, C)`` to make your model be
able to process images af any size, but ``H`` and ``W`` of input images should be divisible by factor ``32``.
classes: a number of classes for output (output shape - ``(h, w, classes)``).
activation: name of one of ``keras.activations`` for last model layer
(e.g. ``sigmoid``, ``softmax``, ``linear``).
encoder_weights: one of ``None`` (random initialization), ``imagenet`` (pre-training on ImageNet).
encoder_freeze: if ``True`` set all layers of encoder (backbone model) as non-trainable.
encoder_features: a list of layer numbers or names starting from top of the model.
Each of these layers will be concatenated with corresponding decoder block. If ``default`` is used
layer names are taken from ``DEFAULT_SKIP_CONNECTIONS``.
decoder_block_type: one of blocks with following layers structure:
- `upsampling`: ``Upsampling2D`` -> ``Conv2D`` -> ``Conv2D``
- `transpose`: ``Transpose2D`` -> ``Conv2D``
decoder_filters: list of numbers of ``Conv2D`` layer filters in decoder blocks
decoder_use_batchnorm: if ``True``, ``BatchNormalisation`` layer between ``Conv2D`` and ``Activation`` layers
is used.
Returns:
``keras.models.Model``: **Unet**
.. _Unet:
https://arxiv.org/pdf/1505.04597
"""
backbone = get_backbone(backbone_name,
input_shape=input_shape,
input_tensor=None,
weights=encoder_weights,
include_top=False)
if encoder_features == 'default':
encoder_features = get_feature_layers(backbone_name, n=4)
model = build_unet(backbone,
classes,
encoder_features,
decoder_filters=decoder_filters,
block_type=decoder_block_type,
activation=activation,
n_upsample_blocks=len(decoder_filters),
upsample_rates=(2, 2, 2, 2, 2),
use_batchnorm=decoder_use_batchnorm)
# lock encoder weights for fine-tuning
if encoder_freeze:
freeze_model(backbone)
model.name = 'u-{}'.format(backbone_name)
return model
def create_backbone_unet(input_shape=(256, 256, 3), pretrained_weights_file=None, backbone='vgg16'):
# build model for segmenting all masks
if pretrained_weights_file == 'imagenet':
encoder_weights = 'imagenet'
else:
encoder_weights = None
if pretrained_weights_file != None:
encoder_freeze = True
else:
encoder_freeze = False
# input
# input = Input(shape=input_shape, name='input_layer_0')
# x = conv2d_bn_leakyrelu(input, filters=16, kernel_size=3, dilation_rate=(2, 2), name='dilation_1')
# x = conv2d_bn_leakyrelu(x, filters=16, kernel_size=3, dilation_rate=(2, 2), name='dilation_2')
# x = ReLU(name='input_1')(x)
# x = Input(shape=input_shape, tensor=x)
# print(x)
# encoder
encoder = get_backbone(name=backbone, input_shape=input_shape, weights=encoder_weights, include_top=False)
# get skip connections
stages = []
feature_layers = get_feature_layers(name=backbone, n=5)
for feature_layer in feature_layers:
stages.append(encoder.get_layer(feature_layer).output)
# decoder
x = encoder.output
# x = block_n(x)
# x = aspp(x, input_shape=input_shape, out_stride=32)
# x = conv2d_bn_leakyrelu(x, filters=512, kernel_size=1)
x = Dropout(0.2)(x)
for i in range(5):
x = UpSampling2D(size=(2, 2), interpolation='bilinear')(x)
if i < len(stages):
x = conv2d_bn_leakyrelu(x, filters=2 ** (8 - i), kernel_size=3)
x = Concatenate()([x, stages[i]])
x = conv2d_bn_leakyrelu(x, filters=2 ** (8 - i), kernel_size=3)
x = conv2d_bn_leakyrelu(x, filters=2 ** (8 - i), kernel_size=3)
# output
x = Conv2D(filters=4, kernel_size=1)(x)
x = UpSampling2D(size=(2, 2), interpolation='bilinear')(x)
x = UpSampling2D(size=(2, 2), interpolation='bilinear')(x)
x = Activation('softmax')(x)
# freeze encoder
if encoder_freeze:
for layer in encoder.layers:
if not isinstance(layer, BatchNormalization):
layer.trainable = False
# create model
model = Model(input=encoder.input, output=x)
print('Create U-Net, input shape = {}, output shape = {}'.format(input_shape, model.output.shape))
if pretrained_weights_file != None and pretrained_weights_file != 'imagenet':
model.load_weights(pretrained_weights_file, skip_mismatch=True, by_name=True)
print('Weights loaded from', pretrained_weights_file)
return model
from keras import layers
from segmentation_models.unet.builder import build_unet
from segmentation_models.backbones import get_preprocessing, get_backbone, get_feature_layers
BACKBONE = 'resnet50'
preprocess_input = get_preprocessing(BACKBONE)
inp = layers.Input((None, None, 3))
encoder_backbone = get_backbone(BACKBONE,
input_shape=(None, None, 3),
input_tensor=inp,
weights='imagenet',
include_top=False)
backbone_model = build_unet(encoder_backbone, 1,
get_feature_layers(BACKBONE, n=4))
backbone_model.summary(line_length=150)
feature_layer = backbone_model.get_layer('decoder_stage4_relu2')
print('Ende gelaende')