def test_get_featurenet3d_backbone(self, data_format):
backbone = 'featurenet3d'
input_shape = (40, 256, 256, 3)
inputs = Input(shape=input_shape)
with self.cached_session():
K.set_image_data_format(data_format)
model, output_dict = backbone_utils.get_backbone(
backbone, inputs, return_dict=True)
assert isinstance(output_dict, dict)
assert all(k.startswith('C') for k in output_dict)
assert isinstance(model, Model)
# No imagenet weights for featurenet backbone
with self.assertRaises(ValueError):
backbone_utils.get_backbone(backbone, inputs, use_imagenet=True)
def LabelDetectionModel(input_shape=(None, None, 1),
inputs=None,
backbone='mobilenetv2',
use_pretrained_weights=True):
"""Classify a microscopy image as Nuclear, Cytoplasm, or Phase.
This can be helpful in determining the type of data (nuclear, cytoplasm,
etc.) so that this data can be forwared to the correct segmenation model.
"""
required_channels = 3 # required for most backbones
if inputs is None:
inputs = keras.layers.Input(shape=input_shape)
if keras.backend.image_data_format() == 'channels_first':
channel_axis = 0
else:
channel_axis = -1
norm = ImageNormalization2D(norm_method='whole_image')(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[channel_axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
backbone_model = get_backbone(backbone,
fixed_inputs,
use_imagenet=False,
return_dict=False,
include_top=False,
weights=None,
input_shape=fixed_input_shape,
pooling=None)
x = keras.layers.AveragePooling2D(4)(backbone_model.outputs[0])
x = TensorProduct(256)(x)
x = TensorProduct(3)(x)
x = keras.layers.Flatten()(x)
outputs = keras.layers.Activation('softmax')(x)
model = keras.Model(inputs=backbone_model.inputs, outputs=outputs)
if use_pretrained_weights:
local_name = 'LabelDetectionModel_{}.h5'.format(backbone)
if backbone.lower() in {'mobilenetv2' or 'mobilenet_v2'}:
weights_path = get_file(
local_name,
MOBILENETV2_WEIGHTS_PATH,
cache_subdir='models',
md5_hash='b8231f32f01c1cd6448d06e276dd5949')
else:
raise ValueError('Backbone %s does not have a weights file.' %
backbone)
model.load_weights(weights_path)
return model
def ScaleDetectionModel(input_shape=(None, None, 1),
inputs=None,
backbone='mobilenetv2',
use_pretrained_weights=True):
"""Create a ScaleDetectionModel for detecting scales of input data.
This enables data to be scaled appropriately for other segmentation models
which may not be resolution tolerant.
"""
required_channels = 3 # required for most backbones
if inputs is None:
inputs = keras.layers.Input(shape=input_shape)
if keras.backend.image_data_format() == 'channels_first':
channel_axis = 0
else:
channel_axis = -1
norm = ImageNormalization2D(norm_method='whole_image')(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[channel_axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
backbone_model = get_backbone(backbone,
fixed_inputs,
use_imagenet=False,
return_dict=False,
include_top=False,
weights=None,
input_shape=fixed_input_shape,
pooling=None)
x = keras.layers.AveragePooling2D(4)(backbone_model.outputs[0])
x = TensorProduct(256, activation='relu')(x)
x = TensorProduct(1)(x)
outputs = keras.layers.Flatten()(x)
model = keras.Model(inputs=backbone_model.inputs, outputs=outputs)
if use_pretrained_weights:
local_name = 'ScaleDetectionModel_{}.h5'.format(backbone)
if backbone.lower() in {'mobilenetv2' or 'mobilenet_v2'}:
weights_path = get_file(
local_name,
MOBILENETV2_WEIGHTS_PATH,
cache_subdir='models',
md5_hash='b9943554a86096fb66608ec66078aa46')
else:
raise ValueError('Backbone %s does not have a weights file.' %
backbone)
model.load_weights(weights_path)
return model
def MaskRCNN(backbone,
num_classes,
input_shape,
norm_method='whole_image',
crop_size=(14, 14),
weights=None,
pooling=None,
mask_dtype=K.floatx(),
required_channels=3,
**kwargs):
"""Constructs a mrcnn model using a backbone from keras-applications.
Args:
backbone: string, name of backbone to use.
num_classes: Number of classes to classify.
input_shape: The shape of the input data.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
pooling: optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
required_channels: integer, the required number of channels of the
backbone. 3 is the default for all current backbones.
Returns:
RetinaNet model with a backbone.
"""
inputs = Input(shape=input_shape)
# force the channel size for backbone input to be `required_channels`
norm = ImageNormalization2D(norm_method=norm_method)(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
model_kwargs = {
'include_top': False,
'input_tensor': fixed_inputs,
'weights': weights,
'pooling': pooling
}
layer_outputs = get_backbone(backbone, inputs, **model_kwargs)
kwargs['backbone_layers'] = layer_outputs
# create the full model
return retinanet_mask(inputs=inputs,
num_classes=num_classes,
crop_size=crop_size,
name='{}_retinanet_mask'.format(backbone),
mask_dtype=mask_dtype,
**kwargs)
def test_get_backbone(self, backbone):
with self.cached_session():
K.set_image_data_format('channels_last')
inputs = Input(shape=(256, 256, 3))
model, output_dict = backbone_utils.get_backbone(
backbone, inputs, return_dict=True)
assert isinstance(output_dict, dict)
assert all(k.startswith('C') for k in output_dict)
assert isinstance(model, Model)
def LabelDetectionModel(input_shape=(None, None, 1),
inputs=None,
backbone='mobilenetv2',
num_classes=3):
"""Classify a microscopy image as Nuclear, Cytoplasm, or Phase.
This can be helpful in determining the type of data (nuclear, cytoplasm,
etc.) so that this data can be forwared to the correct segmenation model.
Based on a standard backbone with an intiial ``ImageNormalization2D`` and
final ``AveragePooling2D``, ``TensorProduct``, and ``Softmax`` layers.
Args:
input_shape (tuple): a 3-length tuple of the input data shape.
inputs (tensorflow.keras.Layer): Optional input layer of the model.
If not provided, creates a ``Layer`` based on ``input_shape``.
backbone (str): name of the backbone to use for the model.
num_classes (int): The number of labels to detect.
"""
required_channels = 3 # required for most backbones
if inputs is None:
inputs = tf.keras.layers.Input(shape=input_shape)
if tf.keras.backend.image_data_format() == 'channels_first':
channel_axis = 0
else:
channel_axis = -1
norm = ImageNormalization2D(norm_method='whole_image')(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[channel_axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
backbone_model = get_backbone(
backbone,
fixed_inputs,
use_imagenet=False,
return_dict=False,
include_top=False,
weights=None,
input_shape=fixed_input_shape,
pooling=None)
x = tf.keras.layers.AveragePooling2D(4)(backbone_model.outputs[0])
x = tf.keras.layers.Flatten()(x)
x = TensorProduct(256)(x)
x = TensorProduct(num_classes)(x)
outputs = tf.keras.layers.Softmax(dtype=tf.keras.backend.floatx())(x)
model = tf.keras.Model(inputs=backbone_model.inputs, outputs=outputs)
return model
def ScaleDetectionModel(input_shape=(None, None, 1),
inputs=None,
backbone='mobilenetv2'):
"""Create a ``ScaleDetectionModel`` for detecting scales of input data.
This enables data to be scaled appropriately for other segmentation models
which may not be resolution tolerant.
Based on a standard backbone with an intiial ``ImageNormalization2D`` and
final ``AveragePooling2D`` and ``TensorProduct`` layers.
Args:
input_shape (tuple): a 3-length tuple of the input data shape.
inputs (tensorflow.keras.Layer): Optional input layer of the model.
If not provided, creates a ``Layer`` based on ``input_shape``.
backbone (str): name of the backbone to use for the model.
"""
required_channels = 3 # required for most backbones
if inputs is None:
inputs = tf.keras.layers.Input(shape=input_shape)
if tf.keras.backend.image_data_format() == 'channels_first':
channel_axis = 0
else:
channel_axis = -1
norm = ImageNormalization2D(norm_method='whole_image')(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[channel_axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
backbone_model = get_backbone(
backbone,
fixed_inputs,
use_imagenet=False,
return_dict=False,
include_top=False,
weights=None,
input_shape=fixed_input_shape,
pooling=None)
x = tf.keras.layers.AveragePooling2D(4)(backbone_model.outputs[0])
x = TensorProduct(256, activation='relu')(x)
x = TensorProduct(1)(x)
outputs = tf.keras.layers.Flatten()(x)
model = tf.keras.Model(inputs=backbone_model.inputs, outputs=outputs)
return model
def RetinaMask(backbone,
num_classes,
input_shape,
inputs=None,
backbone_levels=['C3', 'C4', 'C5'],
pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'],
norm_method='whole_image',
location=False,
use_imagenet=False,
crop_size=(14, 14),
pooling=None,
mask_dtype=K.floatx(),
required_channels=3,
frames_per_batch=1,
**kwargs):
"""Constructs a mrcnn model using a backbone from keras-applications.
Args:
backbone (str): Name of backbone to use.
num_classes (int): Number of classes to classify.
input_shape (tuple): The shape of the input data.
inputs (tensor): Optional input tensor, overrides input_shape.
backbone_levels (list): The backbone levels to be used.
to create the feature pyramid. Defaults to ['C3', 'C4', 'C5'].
pyramid_levels (list): The pyramid levels to attach regression and
classification heads to. Defaults to ['P3', 'P4', 'P5', 'P6', 'P7'].
norm_method (str): ImageNormalization mode to use.
location (bool): Whether to include location data.
use_imagenet (bool): Whether to load imagenet-based pretrained weights.
crop_size (tuple): 2-length tuple for the x-y size of the crops.
Used to create default roi_submodels.
pooling (str): optional pooling mode for feature extraction
when include_top is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
mask_dtype (str): Dtype to use for mask tensors.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
frames_per_batch (int): Size of z axis in generated batches.
If equal to 1, assumes 2D data.
kwargs (dict): Other standard inputs for retinanet_mask.
Returns:
tensorflow.keras.Model: RetinaNet model with a backbone.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
if inputs is None:
if frames_per_batch > 1:
if channel_axis == 1:
input_shape_with_time = tuple(
[input_shape[0], frames_per_batch] + list(input_shape)[1:])
else:
input_shape_with_time = tuple([frames_per_batch] +
list(input_shape))
inputs = Input(shape=input_shape_with_time)
else:
inputs = Input(shape=input_shape)
if location:
if frames_per_batch > 1:
# TODO: TimeDistributed is incompatible with channels_first
loc = TimeDistributed(Location2D(in_shape=input_shape))(inputs)
else:
loc = Location2D(in_shape=input_shape)(inputs)
concat = Concatenate(axis=channel_axis)([inputs, loc])
else:
concat = inputs
# force the channel size for backbone input to be `required_channels`
if frames_per_batch > 1:
norm = TimeDistributed(
ImageNormalization2D(norm_method=norm_method))(concat)
fixed_inputs = TimeDistributed(TensorProduct(required_channels))(norm)
else:
norm = ImageNormalization2D(norm_method=norm_method)(concat)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
axis = 0 if K.image_data_format() == 'channels_first' else -1
fixed_input_shape = list(input_shape)
fixed_input_shape[axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
_, backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=frames_per_batch,
return_dict=True,
**model_kwargs)
# create the full model
return retinanet_mask(inputs=inputs,
num_classes=num_classes,
backbone_dict=backbone_dict,
crop_size=crop_size,
backbone_levels=backbone_levels,
pyramid_levels=pyramid_levels,
name='{}_retinanet_mask'.format(backbone),
mask_dtype=mask_dtype,
frames_per_batch=frames_per_batch,
**kwargs)
def PanopticNet(backbone,
input_shape,
inputs=None,
backbone_levels=['C3', 'C4', 'C5'],
pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'],
create_pyramid_features=__create_pyramid_features,
create_semantic_head=__create_semantic_head,
frames_per_batch=1,
temporal_mode=None,
num_semantic_heads=1,
num_semantic_classes=[3],
required_channels=3,
norm_method='whole_image',
pooling=None,
location=True,
use_imagenet=True,
name='panopticnet',
**kwargs):
"""Constructs a mrcnn model using a backbone from keras-applications.
Args:
backbone (str): Name of backbone to use.
input_shape (tuple): The shape of the input data.
backbone_levels (list): The backbone levels to be used.
to create the feature pyramid. Defaults to ['C3', 'C4', 'C5'].
pyramid_levels (list): Pyramid levels to use. Defaults to ['P3','P4','P5','P6','P7']
create_pyramid_features (function): Function to get the pyramid
features from the backbone.
create_semantic_head (function): Function to get to build a
semantic head submodel.
frames_per_batch (int): Defaults to 1.
temporal_mode: Mode of temporal convolution. Choose from {'conv','lstm','gru', None}.
Defaults to None.
num_semantic_heads (int): Defaults to 1.
num_semantic_classes (list): Defaults to [3].
norm_method (str): ImageNormalization mode to use. Defaults to 'whole_image'
location (bool): Whether to include location data.
use_imagenet (bool): Whether to load imagenet-based pretrained weights.
pooling (str): optional pooling mode for feature extraction
when include_top is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
kwargs (dict): Other standard inputs for retinanet_mask.
Raises:
ValueError: temporal_mode not 'conv', 'lstm', 'gru' or None
Returns:
tensorflow.keras.Model: Panoptic model with a backbone.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
# Check input to __merge_temporal_features
acceptable_modes = {'conv', 'lstm', 'gru', None}
if temporal_mode is not None:
temporal_mode = str(temporal_mode).lower()
if temporal_mode not in acceptable_modes:
raise ValueError('Mode {} not supported. Please choose from {}.'.format(
temporal_mode, str(acceptable_modes)))
if inputs is None:
if frames_per_batch > 1:
if channel_axis == 1:
input_shape_with_time = tuple(
[input_shape[0], frames_per_batch] + list(input_shape)[1:])
else:
input_shape_with_time = tuple(
[frames_per_batch] + list(input_shape))
inputs = Input(shape=input_shape_with_time)
else:
inputs = Input(shape=input_shape)
# force the channel size for backbone input to be `required_channels`
if norm_method is None:
norm = inputs
else:
if frames_per_batch > 1:
norm = TimeDistributed(ImageNormalization2D(norm_method=norm_method))(inputs)
else:
norm = ImageNormalization2D(norm_method=norm_method)(inputs)
if location:
if frames_per_batch > 1:
# TODO: TimeDistributed is incompatible with channels_first
loc = TimeDistributed(Location2D(in_shape=input_shape))(norm)
else:
loc = Location2D(in_shape=input_shape)(norm)
concat = Concatenate(axis=channel_axis)([norm, loc])
else:
concat = norm
if frames_per_batch > 1:
fixed_inputs = TimeDistributed(TensorProduct(required_channels))(concat)
else:
fixed_inputs = TensorProduct(required_channels)(concat)
# force the input shape
axis = 0 if K.image_data_format() == 'channels_first' else -1
fixed_input_shape = list(input_shape)
fixed_input_shape[axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
_, backbone_dict = get_backbone(backbone, fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=frames_per_batch,
return_dict=True, **model_kwargs)
backbone_dict_reduced = {k: backbone_dict[k] for k in backbone_dict
if k in backbone_levels}
ndim = 2 if frames_per_batch == 1 else 3
pyramid_dict = create_pyramid_features(backbone_dict_reduced, ndim=ndim)
features = [pyramid_dict[key] for key in pyramid_levels]
if frames_per_batch > 1:
temporal_features = [__merge_temporal_features(
feature, mode=temporal_mode) for feature in features]
for f, k in zip(temporal_features, pyramid_dict.keys()):
pyramid_dict[k] = f
semantic_levels = [int(re.findall(r'\d+', k)[0]) for k in pyramid_dict]
target_level = min(semantic_levels)
semantic_head_list = []
for i in range(num_semantic_heads):
semantic_head_list.append(create_semantic_head(
pyramid_dict, n_classes=num_semantic_classes[i],
input_target=inputs, target_level=target_level,
semantic_id=i, ndim=ndim, **kwargs))
outputs = semantic_head_list
model = Model(inputs=inputs, outputs=outputs, name=name)
return model
def LabelDetectionModel(input_shape=(None, None, 1),
inputs=None,
backbone='mobilenetv2',
use_pretrained_weights=True):
"""Classify a microscopy image as Nuclear, Cytoplasm, or Phase.
This can be helpful in determining the type of data (nuclear, cytoplasm,
etc.) so that this data can be forwared to the correct segmenation model.
Based on a standard backbone with an intiial ImageNormalization2D and final
AveragePooling2D, TensorProduct, and Softmax layers.
Args:
input_shape (tuple): a 3-length tuple of the input data shape.
inputs (tensorflow.keras.Layer): Optional input layer of the model.
If not provided, creates a Layer based on input_shape.
backbone (str): name of the backbone to use for the model.
use_pretrained_weights (bool): whether to load pre-trained weights.
Only supports the MobileNetV2 backbone.
"""
required_channels = 3 # required for most backbones
if inputs is None:
inputs = keras.layers.Input(shape=input_shape)
if keras.backend.image_data_format() == 'channels_first':
channel_axis = 0
else:
channel_axis = -1
norm = ImageNormalization2D(norm_method='whole_image')(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[channel_axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
backbone_model = get_backbone(
backbone,
fixed_inputs,
use_imagenet=False,
return_dict=False,
include_top=False,
weights=None,
input_shape=fixed_input_shape,
pooling=None)
x = keras.layers.AveragePooling2D(4)(backbone_model.outputs[0])
x = TensorProduct(256)(x)
x = TensorProduct(3)(x)
x = keras.layers.Flatten()(x)
outputs = keras.layers.Activation('softmax')(x)
model = keras.Model(inputs=backbone_model.inputs, outputs=outputs)
if use_pretrained_weights:
local_name = 'LabelDetectionModel_{}.h5'.format(backbone)
if backbone.lower() in {'mobilenetv2' or 'mobilenet_v2'}:
weights_path = get_file(
local_name,
MOBILENETV2_WEIGHTS_PATH,
cache_subdir='models',
file_hash='14d4b2f7c77d334c958d2dde79972e6e')
else:
raise ValueError('Backbone %s does not have a weights file.' %
backbone)
model.load_weights(weights_path)
return model
def RetinaMask(backbone,
num_classes,
input_shape,
inputs=None,
backbone_levels=['C3', 'C4', 'C5'],
pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'],
norm_method='whole_image',
location=False,
use_imagenet=False,
crop_size=(14, 14),
pooling=None,
mask_dtype=K.floatx(),
required_channels=3,
frames_per_batch=1,
**kwargs):
"""Constructs a mrcnn model using a backbone from keras-applications.
Args:
backbone (str): Name of backbone to use.
num_classes (int): Number of classes to classify.
input_shape (tuple): The shape of the input data.
weights (str): one of None (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
pooling (str): optional pooling mode for feature extraction
when include_top is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
Returns:
tensorflow.keras.Model: RetinaNet model with a backbone.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
if inputs is None:
if frames_per_batch > 1:
if channel_axis == 1:
input_shape_with_time = tuple(
[input_shape[0], frames_per_batch] + list(input_shape)[1:])
else:
input_shape_with_time = tuple([frames_per_batch] +
list(input_shape))
inputs = Input(shape=input_shape_with_time)
else:
inputs = Input(shape=input_shape)
if location:
if frames_per_batch > 1:
# TODO: TimeDistributed is incompatible with channels_first
loc = TimeDistributed(Location2D(in_shape=input_shape))(inputs)
else:
loc = Location2D(in_shape=input_shape)(inputs)
concat = Concatenate(axis=channel_axis)([inputs, loc])
else:
concat = inputs
# force the channel size for backbone input to be `required_channels`
if frames_per_batch > 1:
norm = TimeDistributed(
ImageNormalization2D(norm_method=norm_method))(concat)
fixed_inputs = TimeDistributed(TensorProduct(required_channels))(norm)
else:
norm = ImageNormalization2D(norm_method=norm_method)(concat)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
axis = 0 if K.image_data_format() == 'channels_first' else -1
fixed_input_shape = list(input_shape)
fixed_input_shape[axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
_, backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=frames_per_batch,
return_dict=True,
**model_kwargs)
# create the full model
return retinanet_mask(inputs=inputs,
num_classes=num_classes,
backbone_dict=backbone_dict,
crop_size=crop_size,
backbone_levels=backbone_levels,
pyramid_levels=pyramid_levels,
name='{}_retinanet_mask'.format(backbone),
mask_dtype=mask_dtype,
frames_per_batch=frames_per_batch,
**kwargs)
def PanopticNet(backbone,
input_shape,
backbone_levels=['C3', 'C4', 'C5'],
create_pyramid_features=__create_pyramid_features,
create_semantic_head=__create_semantic_head,
num_semantic_heads=1,
num_semantic_classes=[3],
required_channels=3,
norm_method='whole_image',
pooling=None,
location=True,
use_imagenet=True,
name='panopticnet',
**kwargs):
"""Constructs a mrcnn model using a backbone from keras-applications.
Args:
backbone (str): Name of backbone to use.
input_shape (tuple): The shape of the input data.
backbone_levels (list): The backbone levels to be used.
to create the feature pyramid. Defaults to ['C3', 'C4', 'C5'].
create_pyramid_features (function): Function to get the pyramid
features from the backbone.
create_semantic_head (function): Function to get to build a
semantic head submodel.
norm_method (str): ImageNormalization mode to use.
location (bool): Whether to include location data.
use_imagenet (bool): Whether to load imagenet-based pretrained weights.
pooling (str): optional pooling mode for feature extraction
when include_top is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
kwargs (dict): Other standard inputs for retinanet_mask.
Returns:
tensorflow.keras.Model: Panoptic model with a backbone.
"""
inputs = Input(shape=input_shape)
norm = ImageNormalization2D(norm_method=norm_method)(inputs)
if location:
loc = Location2D(in_shape=input_shape)(norm)
concat = Concatenate(axis=-1)([norm, loc])
else:
concat = norm
fixed_inputs = TensorProduct(required_channels)(concat)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[-1] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
_, backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=1,
return_dict=True,
**model_kwargs)
backbone_dict_reduced = {
k: backbone_dict[k]
for k in backbone_dict if k in backbone_levels
}
pyramid_dict = create_pyramid_features(backbone_dict_reduced, ndim=2)
semantic_levels = [int(re.findall(r'\d+', k)[0]) for k in pyramid_dict]
target_level = min(semantic_levels)
semantic_head_list = []
for i in range(num_semantic_heads):
semantic_head_list.append(
create_semantic_head(pyramid_dict,
n_classes=num_semantic_classes[i],
input_target=inputs,
target_level=target_level,
semantic_id=i,
ndim=2,
**kwargs))
model = Model(inputs=inputs, outputs=semantic_head_list, name=name)
return model
def RetinaNet(backbone,
num_classes,
input_shape,
inputs=None,
norm_method='whole_image',
location=False,
use_imagenet=False,
pooling=None,
required_channels=3,
frames_per_batch=1,
**kwargs):
"""Constructs a RetinaNet model using a backbone from
``keras-applications``.
Args:
backbone (str): Name of backbone to use.
num_classes (int): Number of classes to classify.
input_shape (tuple): The shape of the input data.
inputs (tensor): Optional input tensor, overrides ``input_shape``.
norm_method (str): Normalization method to use with the
:mod:`deepcell.layers.normalization.ImageNormalization2D` layer.
location (bool): Whether to include a
:mod:`deepcell.layers.location.Location2D` layer.
use_imagenet (bool): Whether to load imagenet-based pretrained weights.
pooling (str): Pooling mode for feature extraction
when ``include_top`` is ``False``.
- None means that the output of the model will be
the 4D tensor output of the last convolutional layer.
- 'avg' means that global average pooling will be applied to
the output of the last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will be applied.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
frames_per_batch (int): Size of z axis in generated batches.
If equal to 1, assumes 2D data.
kwargs (dict): Other standard inputs for `~retinanet`.
Returns:
tensorflow.keras.Model: RetinaNet model with a backbone.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
if inputs is None:
if frames_per_batch > 1:
if channel_axis == 1:
input_shape_with_time = tuple(
[input_shape[0], frames_per_batch] + list(input_shape)[1:])
else:
input_shape_with_time = tuple([frames_per_batch] +
list(input_shape))
inputs = Input(shape=input_shape_with_time, name='input')
else:
inputs = Input(shape=input_shape, name='input')
if location:
if frames_per_batch > 1:
# TODO: TimeDistributed is incompatible with channels_first
loc = TimeDistributed(Location2D(in_shape=input_shape))(inputs)
else:
loc = Location2D(in_shape=input_shape)(inputs)
concat = Concatenate(axis=channel_axis)([inputs, loc])
else:
concat = inputs
# force the channel size for backbone input to be `required_channels`
if frames_per_batch > 1:
norm = TimeDistributed(
ImageNormalization2D(norm_method=norm_method))(concat)
fixed_inputs = TimeDistributed(TensorProduct(required_channels))(norm)
else:
norm = ImageNormalization2D(norm_method=norm_method)(concat)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
axis = 0 if K.image_data_format() == 'channels_first' else -1
fixed_input_shape = list(input_shape)
fixed_input_shape[axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
_, backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=frames_per_batch,
return_dict=True,
**model_kwargs)
# create the full model
return retinanet(inputs=inputs,
num_classes=num_classes,
backbone_dict=backbone_dict,
frames_per_batch=frames_per_batch,
name='{}_retinanet'.format(backbone),
**kwargs)
def ScaleDetectionModel(input_shape=(None, None, 1),
inputs=None,
backbone='mobilenetv2',
use_pretrained_weights=True):
"""Create a ScaleDetectionModel for detecting scales of input data.
This enables data to be scaled appropriately for other segmentation models
which may not be resolution tolerant.
Based on a standard backbone with an intiial ImageNormalization2D and final
AveragePooling2D and TensorProduct layers.
Args:
input_shape (tuple): a 3-length tuple of the input data shape.
inputs (tensorflow.keras.Layer): Optional input layer of the model.
If not provided, creates a Layer based on input_shape.
backbone (str): name of the backbone to use for the model.
use_pretrained_weights (bool): whether to load pre-trained weights.
Only supports the MobileNetV2 backbone.
"""
required_channels = 3 # required for most backbones
if inputs is None:
inputs = keras.layers.Input(shape=input_shape)
if keras.backend.image_data_format() == 'channels_first':
channel_axis = 0
else:
channel_axis = -1
norm = ImageNormalization2D(norm_method='whole_image')(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[channel_axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
backbone_model = get_backbone(
backbone,
fixed_inputs,
use_imagenet=False,
return_dict=False,
include_top=False,
weights=None,
input_shape=fixed_input_shape,
pooling=None)
x = keras.layers.AveragePooling2D(4)(backbone_model.outputs[0])
x = TensorProduct(256, activation='relu')(x)
x = TensorProduct(1)(x)
outputs = keras.layers.Flatten()(x)
model = keras.Model(inputs=backbone_model.inputs, outputs=outputs)
if use_pretrained_weights:
local_name = 'ScaleDetectionModel_{}.h5'.format(backbone)
if backbone.lower() in {'mobilenetv2' or 'mobilenet_v2'}:
weights_path = get_file(
local_name,
MOBILENETV2_WEIGHTS_PATH,
cache_subdir='models',
file_hash='aa78e6b9a4551289dd967f1f5ca83fed')
else:
raise ValueError('Backbone %s does not have a weights file.' %
backbone)
model.load_weights(weights_path)
return model
def FPNet(backbone,
input_shape,
inputs=None,
norm_method='whole_image',
use_imagenet=False,
pooling=None,
required_channels=3,
n_classes=3,
name='fpnet',
frames_per_batch=1,
**kwargs):
"""Creates a Feature Pyramid Network with a semantic segmentation head
Args:
backbone (str): A name of a supported backbone from [deepcell, resnet50]
input_shape (tuple): Shape of the input image.
inputs (keras.Layer): Optional preexisting layers.
norm_method (str): Normalization method, defaults to 'whole_image'
use_imagenet (bool): Whether to load imagenet-based pretrained weights.
pooling (str): Optional pooling mode for feature extraction
when include_top is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
n_classes (int): The number of classes to be predicted
name (str): Name to use for the model.
frames_per_batch (int): Size of z axis in generated batches.
If equal to 1, assumes 2D data.
Returns:
tensorflow.keras.models.Model: Feature pyramid network with a semantic
segmentation head as the output
"""
if inputs is None:
inputs = Input(shape=input_shape)
# force the channel size for backbone input to be required_channels
norm = ImageNormalization2D(norm_method=norm_method)(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[-1] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
# Get backbone outputs
_, backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=frames_per_batch,
return_dict=True,
**model_kwargs)
# Construct feature pyramid network
pyramid_dict = __create_pyramid_features(backbone_dict)
levels = [int(re.findall(r'\d+', k)[0]) for k in pyramid_dict]
target_level = min(levels)
x = __create_semantic_head(pyramid_dict,
n_classes=n_classes,
input_target=inputs,
target_level=target_level,
ndim=len(input_shape) - 1)
return Model(inputs=inputs, outputs=x, name=name)
def test_invalid_backbone(self):
inputs = Input(shape=(4, 2, 3))
with self.assertRaises(ValueError):
backbone_utils.get_backbone('bad', inputs, return_dict=True)
def MaskRCNN(backbone,
num_classes,
input_shape,
backbone_levels=['C3', 'C4', 'C5'],
pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'],
norm_method='whole_image',
location=False,
use_imagenet=False,
crop_size=(14, 14),
pooling=None,
mask_dtype=K.floatx(),
required_channels=3,
**kwargs):
"""Constructs a mrcnn model using a backbone from keras-applications.
Args:
backbone: string, name of backbone to use.
num_classes: Number of classes to classify.
input_shape: The shape of the input data.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
pooling: optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
required_channels: integer, the required number of channels of the
backbone. 3 is the default for all current backbones.
Returns:
RetinaNet model with a backbone.
"""
inputs = Input(shape=input_shape)
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
if location:
location = Location2D(in_shape=input_shape)(inputs)
inputs = Concatenate(axis=channel_axis)([inputs, location])
# force the channel size for backbone input to be `required_channels`
norm = ImageNormalization2D(norm_method=norm_method)(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[-1] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
**model_kwargs)
# create the full model
return retinanet_mask(inputs=inputs,
num_classes=num_classes,
backbone_dict=backbone_dict,
crop_size=crop_size,
backbone_levels=backbone_levels,
pyramid_levels=pyramid_levels,
name='{}_retinanet_mask'.format(backbone),
mask_dtype=mask_dtype,
**kwargs)
def PanopticNet(backbone,
input_shape,
inputs=None,
backbone_levels=['C3', 'C4', 'C5'],
pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'],
create_pyramid_features=__create_pyramid_features,
create_semantic_head=__create_semantic_head,
frames_per_batch=1,
temporal_mode=None,
num_semantic_heads=1,
num_semantic_classes=[3],
required_channels=3,
norm_method='whole_image',
pooling=None,
location=True,
use_imagenet=True,
lite=False,
upsample_type='upsampling2d',
interpolation='bilinear',
name='panopticnet',
**kwargs):
"""Constructs a mrcnn model using a backbone from keras-applications.
Args:
backbone (str): Name of backbone to use.
input_shape (tuple): The shape of the input data.
backbone_levels (list): The backbone levels to be used.
to create the feature pyramid. Defaults to ['C3', 'C4', 'C5'].
pyramid_levels (list): Pyramid levels to use. Defaults to
['P3','P4','P5','P6','P7']
create_pyramid_features (function): Function to get the pyramid
features from the backbone.
create_semantic_head (function): Function to build a semantic head
submodel.
frames_per_batch (int): Defaults to 1.
temporal_mode: Mode of temporal convolution. Choose from
{'conv','lstm','gru', None}. Defaults to None.
num_semantic_heads (int): Defaults to 1.
num_semantic_classes (list): Defaults to [3].
norm_method (str): ImageNormalization mode to use.
Defaults to 'whole_image'.
location (bool): Whether to include location data. Defaults to True
use_imagenet (bool): Whether to load imagenet-based pretrained weights.
lite (bool): Whether to use a depthwise conv in the feature pyramid
rather than regular conv. Defaults to False.
upsample_type (str): Choice of upsampling layer to use from
['upsamplelike', 'upsampling2d', 'upsampling3d']. Defaults to
'upsampling2d'.
interpolation (str): Choice of interpolation mode for upsampling
layers from ['bilinear', 'nearest']. Defaults to bilinear.
pooling (str): optional pooling mode for feature extraction
when include_top is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
kwargs (dict): Other standard inputs for retinanet_mask.
Raises:
ValueError: temporal_mode not 'conv', 'lstm', 'gru' or None
Returns:
tensorflow.keras.Model: Panoptic model with a backbone.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
conv = Conv3D if frames_per_batch > 1 else Conv2D
conv_kernel = (1, 1, 1) if frames_per_batch > 1 else (1, 1)
# Check input to __merge_temporal_features
acceptable_modes = {'conv', 'lstm', 'gru', None}
if temporal_mode is not None:
temporal_mode = str(temporal_mode).lower()
if temporal_mode not in acceptable_modes:
raise ValueError('temporal_mode {} not supported. Please choose '
'from {}.'.format(temporal_mode, acceptable_modes))
# TODO only works for 2D: do we check for 3D as well?
# What are the requirements for 3D data?
img_shape = input_shape[1:] if channel_axis == 1 else input_shape[:-1]
if img_shape[0] != img_shape[1]:
raise ValueError('Input data must be square, got dimensions {}'.format(
img_shape))
if not math.log(img_shape[0], 2).is_integer():
raise ValueError('Input data dimensions must be a power of 2, '
'got {}'.format(img_shape[0]))
# Check input to interpolation
acceptable_interpolation = {'bilinear', 'nearest'}
if interpolation not in acceptable_interpolation:
raise ValueError('Interpolation mode "{}" not supported. '
'Choose from {}.'.format(
interpolation, list(acceptable_interpolation)))
if inputs is None:
if frames_per_batch > 1:
if channel_axis == 1:
input_shape_with_time = tuple(
[input_shape[0], frames_per_batch] + list(input_shape)[1:])
else:
input_shape_with_time = tuple(
[frames_per_batch] + list(input_shape))
inputs = Input(shape=input_shape_with_time, name='input_0')
else:
inputs = Input(shape=input_shape, name='input_0')
# Normalize input images
if norm_method is None:
norm = inputs
else:
if frames_per_batch > 1:
norm = TimeDistributed(ImageNormalization2D(
norm_method=norm_method, name='norm'), name='td_norm')(inputs)
else:
norm = ImageNormalization2D(norm_method=norm_method,
name='norm')(inputs)
# Add location layer
if location:
if frames_per_batch > 1:
# TODO: TimeDistributed is incompatible with channels_first
loc = TimeDistributed(Location2D(in_shape=input_shape,
name='location'), name='td_location')(norm)
else:
loc = Location2D(in_shape=input_shape, name='location')(norm)
concat = Concatenate(axis=channel_axis,
name='concatenate_location')([norm, loc])
else:
concat = norm
# Force the channel size for backbone input to be `required_channels`
fixed_inputs = conv(required_channels, conv_kernel, strides=1,
padding='same', name='conv_channels')(concat)
# Force the input shape
axis = 0 if K.image_data_format() == 'channels_first' else -1
fixed_input_shape = list(input_shape)
fixed_input_shape[axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
_, backbone_dict = get_backbone(backbone, fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=frames_per_batch,
return_dict=True,
**model_kwargs)
backbone_dict_reduced = {k: backbone_dict[k] for k in backbone_dict
if k in backbone_levels}
ndim = 2 if frames_per_batch == 1 else 3
pyramid_dict = create_pyramid_features(backbone_dict_reduced,
ndim=ndim,
lite=lite,
interpolation=interpolation,
upsample_type=upsample_type)
features = [pyramid_dict[key] for key in pyramid_levels]
if frames_per_batch > 1:
temporal_features = [__merge_temporal_features(f, mode=temporal_mode,
frames_per_batch=frames_per_batch)
for f in features]
for f, k in zip(temporal_features, pyramid_levels):
pyramid_dict[k] = f
semantic_levels = [int(re.findall(r'\d+', k)[0]) for k in pyramid_dict]
target_level = min(semantic_levels)
semantic_head_list = []
for i in range(num_semantic_heads):
semantic_head_list.append(create_semantic_head(
pyramid_dict, n_classes=num_semantic_classes[i],
input_target=inputs, target_level=target_level,
semantic_id=i, ndim=ndim, upsample_type=upsample_type,
interpolation=interpolation, **kwargs))
outputs = semantic_head_list
model = Model(inputs=inputs, outputs=outputs, name=name)
return model
def RetinaNet(backbone,
num_classes,
input_shape,
inputs=None,
norm_method='whole_image',
location=False,
use_imagenet=False,
pooling=None,
required_channels=3,
**kwargs):
"""Constructs a retinanet model using a backbone from keras-applications.
Args:
backbone (str): Name of backbone to use.
num_classes (int): Number of classes to classify.
input_shape (tuple): The shape of the input data.
weights (str): one of None (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
pooling (str): optional pooling mode for feature extraction
when 'include_top' is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
Returns:
tensorflow.keras.Model: RetinaNet model with a backbone.
"""
if inputs is None:
inputs = Input(shape=input_shape)
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
if location:
location = Location2D(in_shape=input_shape)(inputs)
concat = Concatenate(axis=channel_axis)([inputs, location])
else:
concat = inputs
# force the channel size for backbone input to be `required_channels`
norm = ImageNormalization2D(norm_method=norm_method)(concat)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
axis = 0 if K.image_data_format() == 'channels_first' else -1
fixed_input_shape = list(input_shape)
fixed_input_shape[axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
**model_kwargs)
# create the full model
return retinanet(inputs=inputs,
num_classes=num_classes,
backbone_dict=backbone_dict,
name='{}_retinanet'.format(backbone),
**kwargs)
def FPNet(backbone,
input_shape,
inputs=None,
norm_method='whole_image',
use_imagenet=False,
pooling=None,
required_channels=3,
n_classes=3,
name='fpnet',
**kwargs):
"""
Creates a Feature Pyramid Network with a semantic segmentation head
Args:
backbone (str): A name of a supported backbone from [deepcell, resnet50]
input_shape (tuple): Shape of the input image
input (keras layer, optional): Defaults to None. Method to pass in preexisting layers
norm_method (str, optional): Defaults to 'whole_image'. Normalization method
weights (str, optional): Defaults to None. one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
pooling (str, optional): Defaults to None. optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
required_channels (int, optional): Defaults to 3. The required number of channels of the
backbone. 3 is the default for all current backbones.
n_classes (int, optional): Defaults to 3. The number of classes to be predicted
name (str, optional): Defaults to 'fpnet'. Name to use for the model.
Returns:
Model with a feature pyramid network with a semantic segmentation
head as the output
"""
if inputs is None:
inputs = Input(shape=input_shape)
# force the channel size for backbone input to be `required_channels`
norm = ImageNormalization2D(norm_method=norm_method)(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[-1] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
# Get backbone outputs
backbone_dict = get_backbone(
backbone, fixed_inputs, use_imagenet=use_imagenet, **model_kwargs)
# Construct feature pyramid network
pyramid_dict = __create_pyramid_features(backbone_dict)
levels = [int(re.findall(r'\d+', k)[0]) for k in pyramid_dict]
target_level = min(levels)
x = __create_semantic_head(pyramid_dict, n_classes=n_classes,
input_target=inputs, target_level=target_level)
return Model(inputs=inputs, outputs=x, name=name)