def residual_block(x, **metaparameters):
""" Construct a Residual Block
x : input to the block
n_filters: number of filters in convolution layer in residual block
"""
if 'n_filters' in metaparameters:
n_filters = metaparameters['n_filters']
del metaparameters['n_filters']
else:
n_filters = DenseNet.n_filters
# Remember input tensor into residual block
shortcut = x
# BN-RE-Conv pre-activation form of convolutions
# Dimensionality expansion, expand filters by 4 (DenseNet-B)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x, 4 * n_filters, (1, 1), strides=(1, 1), use_bias=False,
**metaparameters)
# Bottleneck convolution
# 3x3 convolution with padding=same to preserve same shape of feature maps
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x, n_filters, (3, 3), strides=(1, 1), padding='same', use_bias=False,
**metaparameters)
# Concatenate the input (identity) with the output of the residual block
# Concatenation (vs. merging) provides Feature Reuse between layers
x = Concatenate()([shortcut, x])
return x
def identity_block(x, **metaparameters):
""" Construct a ResNeXT block with identity link
x : input to block
filters_in : number of filters (channels) at the input convolution
filters_out : number of filters (channels) at the output convolution
cardinality : width of group convolution
"""
filters_in = metaparameters['filters_in']
filters_out = metaparameters['filters_out']
if 'cardinality' in metaparameters:
cardinality = metaparameters['cardinality']
else:
cardinality = ResNeXt.cardinality
# Remember the input
shortcut = x
# Dimensionality Reduction
x = Composable.Conv2D(x,
filters_in, (1, 1),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Cardinality (Wide) Layer (split-transform)
filters_card = filters_in // cardinality
groups = []
for i in range(cardinality):
group = Lambda(lambda z: z[:, :, :, i * filters_card:i *
filters_card + filters_card])(x)
groups.append(
Composable.Conv2D(group,
filters_card, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters))
# Concatenate the outputs of the cardinality layer together (merge)
x = Concatenate()(groups)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Dimensionality restoration
x = Composable.Conv2D(x,
filters_out, (1, 1),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
# Identity Link: Add the shortcut (input) to the output of the block
x = Add()([shortcut, x])
x = Composable.ReLU(x)
return x
def projection_block(x, strides=(2, 2), **metaparameters):
""" Create Bottleneck Residual Block with Projection Shortcut
Increase the number of filters by 4X
x : input into the block
strides : whether entry convolution is strided (i.e., (2, 2) vs (1, 1))
n_filters: number of filters
"""
n_filters = metaparameters['n_filters']
del metaparameters['n_filters']
# Construct the projection shortcut
# Increase filters by 4X to match shape when added to output of block
shortcut = Composable.Conv2D(x,
4 * n_filters, (1, 1),
strides=strides,
use_bias=False,
**metaparameters)
shortcut = BatchNormalization()(shortcut)
## Construct the 1x1, 3x3, 1x1 residual block (fig 3c)
# Dimensionality reduction
# Feature pooling when strides=(2, 2)
x = Composable.Conv2D(x,
n_filters, (1, 1),
strides=strides,
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Bottleneck layer
x = Composable.Conv2D(x,
n_filters, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Dimensionality restoration - increase the number of filters by 4X
x = Composable.Conv2D(x,
4 * n_filters, (1, 1),
strides=(1, 1),
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
# Pass the output through the squeeze and excitation block
x = SEResNet.squeeze_excite_block(x, **metaparameters)
# Add the projection shortcut link to the output of the residual block
x = Add()([x, shortcut])
x = Composable.ReLU(x)
return x
def projection_block(x, strides=(2, 2), **metaparameters):
""" Construct a Bottleneck Residual Block of Convolutions with Projection Shortcut
Increase the number of filters by 4X
x : input into the block
strides : whether the first convolution is strided
n_filters: number of filters
reg : kernel regularizer
"""
n_filters = metaparameters['n_filters']
del metaparameters['n_filters']
# Construct the projection shortcut
# Increase filters by 4X to match shape when added to output of block
shortcut = BatchNormalization()(x)
shortcut = Composable.Conv2D(shortcut,
4 * n_filters, (1, 1),
strides=strides,
use_bias=False,
**metaparameters)
## Construct the 1x1, 3x3, 1x1 convolution block
# Dimensionality reduction
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x,
n_filters, (1, 1),
strides=(1, 1),
use_bias=False,
**metaparameters)
# Bottleneck layer
# Feature pooling when strides=(2, 2)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x,
n_filters, (3, 3),
strides=strides,
padding='same',
use_bias=False,
**metaparameters)
# Dimensionality restoration - increase the number of filters by 4X
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x,
4 * n_filters, (1, 1),
strides=(1, 1),
use_bias=False,
**metaparameters)
# Add the projection shortcut to the output of the residual block
x = Add()([x, shortcut])
return x
def stem(inputs):
""" Create the stem entry into the neural network
inputs : input tensor to neural network
"""
# Strided convolution - dimensionality reduction
# Reduce feature maps by 75%
x = Composable.Conv2D(inputs, 32, (3, 3), strides=(2, 2), **metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Convolution - dimensionality expansion
# Double the number of filters
x = Composable.Conv2D(x, 64, (3, 3), strides=(1, 1), **metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
return x
def projection_block(x, **metaparameters):
""" Create a residual block using Depthwise Separable Convolutions with Projection shortcut
x : input into residual block
n_filters: number of filters
"""
n_filters = metaparameters['n_filters']
del metaparameters['n_filters']
# Remember the input
shortcut = x
# Strided convolution to double number of filters in identity link to
# match output of residual block for the add operation (projection shortcut)
shortcut = Composable.Conv2D(x, n_filters, (1, 1), strides=(2, 2), padding='same', **metaparameters)
shortcut = BatchNormalization()(shortcut)
# First Depthwise Separable Convolution
x = Composable.SeparableConv2D(x, n_filters, (3, 3), padding='same', **metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Second depthwise Separable Convolution
x = Composable.SeparableConv2D(x, n_filters, (3, 3), padding='same', **metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Create pooled feature maps, reduce size by 75%
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
# Add the projection shortcut to the output of the block
x = Add()([x, shortcut])
return x
def depthwise_block(x, strides, **metaparameters):
""" Construct a Depthwise Separable Convolution block
x : input to the block
strides : strides
n_filters : number of filters
alpha : width multiplier
"""
n_filters = metaparameters['n_filters']
alpha = metaparameters['alpha']
del metaparameters['n_filters']
# Apply the width filter to the number of feature maps
filters = int(n_filters * alpha)
# Strided convolution to match number of filters
if strides == (2, 2):
x = ZeroPadding2D(padding=((0, 1), (0, 1)))(x)
padding = 'valid'
else:
padding = 'same'
# Depthwise Convolution
x = Composable.DepthwiseConv2D(x, (3, 3), strides, padding=padding, use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Pointwise Convolution
x = Composable.Conv2D(x, filters, (1, 1), strides=(1, 1), padding='same', use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
return x
def trans_block(x, **metaparameters):
""" Construct a Transition Block
x : input layer
reduction: percentage of reduction of feature maps
"""
if 'reduction' in metaparameters:
reduction = metaparameters['reduction']
else:
reduction = DenseNet.reduction
del metaparameters['n_filters']
# Reduce (compress) the number of feature maps (DenseNet-C)
# shape[n] returns a class object. We use int() to cast it into the dimension size
n_filters = int( int(x.shape[3]) * reduction)
# BN-LI-Conv pre-activation form of convolutions
# Use 1x1 linear projection convolution
x = BatchNormalization()(x)
x = Composable.Conv2D(x, n_filters, (1, 1), strides=(1, 1), use_bias=False,
**metaparameters)
# Use mean value (average) instead of max value sampling when pooling reduce by 75%
x = AveragePooling2D((2, 2), strides=(2, 2))(x)
return x
def identity_block(x, **metaparameters):
""" Create a Bottleneck Residual Block with Identity Link
x : input into the block
n_filters: number of filters
"""
n_filters = metaparameters['n_filters']
del metaparameters['n_filters']
# Save input vector (feature maps) for the identity link
shortcut = x
## Construct the 1x1, 3x3, 1x1 residual block (fig 3c)
# Dimensionality reduction
x = Composable.Conv2D(x,
n_filters, (1, 1),
strides=(1, 1),
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Bottleneck layer
x = Composable.Conv2D(x,
n_filters, (3, 3),
strides=(1, 1),
padding="same",
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Dimensionality restoration - increase the number of output filters by 4X
x = Composable.Conv2D(x,
n_filters * 4, (1, 1),
strides=(1, 1),
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
# Pass the output through the squeeze and excitation block
x = SEResNet.squeeze_excite_block(x, **metaparameters)
# Add the identity link (input) to the output of the residual block
x = Add()([shortcut, x])
x = Composable.ReLU(x)
return x
def projection_block(x, **metaparameters):
""" Construct a B(3,3) style block
x : input into the block
n_filters: number of filters
k : width factor
strides : whether the projection shortcut is strided
"""
n_filters = metaparameters['n_filters']
strides = metaparameters['strides']
k = metaparameters['k']
del metaparameters['n_filters']
del metaparameters['strides']
# Save input vector (feature maps) for the identity link
shortcut = BatchNormalization()(x)
shortcut = Composable.Conv2D(shortcut,
n_filters * k, (3, 3),
strides=strides,
padding='same',
use_bias=False,
**metaparameters)
## Construct the 3x3, 3x3 convolution block
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x,
n_filters * k, (3, 3),
strides=strides,
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x,
n_filters * k, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
# Add the identity link (input) to the output of the residual block
x = Add()([shortcut, x])
return x
def identity_block(x, **metaparameters):
""" Construct a B(3,3) style block
x : input into the block
n_filters: number of filters
k : width factor
dropout : dropout rate
"""
n_filters = metaparameters['n_filters']
k = metaparameters['k']
dropout = metaparameters['dropout']
del metaparameters['n_filters']
del metaparameters['strides']
# Save input vector (feature maps) for the identity link
shortcut = x
## Construct the 3x3, 3x3 convolution block
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x,
n_filters * k, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
# dropout only in identity link (not projection)
if dropout > 0:
x = Dropout(dropout)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Composable.Conv2D(x,
n_filters * k, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
# Add the identity link (input) to the output of the residual block
x = Add()([shortcut, x])
return x
def auxiliary(x, n_classes, **metaparameters):
""" Construct the auxiliary classier
x : input to the auxiliary classifier
n_classes: number of output classes
"""
x = AveragePooling2D((5, 5), strides=(3, 3))(x)
x = Composable.Conv2D(x, 128, (1, 1), strides=(1, 1), padding='same', use_bias=False, **metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
x = Flatten()(x)
x = Composable.Dense(x, 1024, activation=Composable.ReLU, **metaparameters)
x = Dropout(0.7)(x)
output = Composable.Dense(x, n_classes, activation='softmax', **metaparameters)
return output
def exitFlow(x, n_classes, **metaparameters):
""" Create the exit flow section
x : input to the exit flow section
n_classes : number of output classes
"""
# Remember the input
shortcut = x
# Strided convolution to double number of filters in identity link to
# match output of residual block for the add operation (projection shortcut)
shortcut = Composable.Conv2D(x, 1024, (1, 1), strides=(2, 2), padding='same', **metaparameters)
shortcut = BatchNormalization()(shortcut)
# First Depthwise Separable Convolution
# Dimensionality reduction - reduce number of filters
x = Composable.SeparableConv2D(x, 728, (3, 3), padding='same', **metaparameters)
x = BatchNormalization()(x)
# Second Depthwise Separable Convolution
# Dimensionality restoration
x = Composable.SeparableConv2D(x, 1024, (3, 3), padding='same', **metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Create pooled feature maps, reduce size by 75%
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
# Add the projection shortcut to the output of the pooling layer
x = Add()([x, shortcut])
# Third Depthwise Separable Convolution
x = Composable.SeparableConv2D(x, 1556, (3, 3), padding='same', **metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Fourth Depthwise Separable Convolution
x = Composable.SeparableConv2D(x, 2048, (3, 3), padding='same', **metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Create classifier section
x = Composable.classifier(x, n_classes, **metaparameters)
return x
def group(x, **metaparameters):
""" Construct a Convolutional Group
x : input to the group
n_layers : number of convolutional layers
n_filters: number of filters
"""
n_filters = metaparameters['n_filters']
n_layers = metaparameters['n_layers']
del metaparameters['n_filters']
# Block of convolutional layers
for n in range(n_layers):
x = Composable.Conv2D(x,
n_filters, (3, 3),
strides=(1, 1),
padding="same",
activation=Composable.ReLU,
**metaparameters)
# Max pooling at the end of the block
x = MaxPooling2D(2, strides=(2, 2))(x)
return x
def pw_group_conv(x, n_partitions, n_filters, **metaparameters):
''' A Pointwise Group Convolution
x : input tensor
n_partitions: number of groups to partition feature maps (channels) into.
n_filters : number of filters
'''
# Calculate the number of input filters (channels)
in_filters = x.shape[3]
# Derive the number of input filters (channels) per group
grp_in_filters = in_filters // n_partitions
# Derive the number of output filters per group (Note the rounding up)
grp_out_filters = int(n_filters / n_partitions + 0.5)
# Perform convolution across each channel group
groups = []
for i in range(n_partitions):
# Slice the input across channel group
group = Lambda(lambda x: x[:, :, :, grp_in_filters * i:
grp_in_filters * (i + 1)])(x)
# Perform convolution on channel group
conv = Composable.Conv2D(group,
grp_out_filters, (1, 1),
padding='same',
strides=1,
use_bias=False,
**metaparameters)
# Maintain the point-wise group convolutions in a list
groups.append(conv)
# Concatenate the outputs of the group pointwise convolutions together
x = Concatenate()(groups)
# Do batch normalization of the concatenated filters (feature maps)
x = BatchNormalization()(x)
return x
def inception_block(x, f1x1, f3x3, f5x5, fpool, **metaparameters):
""" Construct an Inception block (module)
x : input to the block
f1x1 : filters for 1x1 branch
f3x3 : filters for 3x3 branch
f5x5 : filters for 5x5 branch
fpool: filters for pooling branch
"""
# 1x1 branch
b1x1 = Composable.Conv2D(x, f1x1[0], (1, 1), strides=1, padding='same', use_bias=False, **metaparameters)
b1x1 = BatchNormalization()(b1x1)
b1x1 = Composable.ReLU(b1x1)
# 3x3 branch
# 3x3 reduction
b3x3 = Composable.Conv2D(x, f3x3[0], (1, 1), strides=1, padding='same', use_bias=False, **metaparameters)
b3x3 = BatchNormalization()(b3x3)
b3x3 = Composable.ReLU(b3x3)
b3x3 = ZeroPadding2D((1,1))(b3x3)
b3x3 = Composable.Conv2D(b3x3, f3x3[1], (3, 3), strides=1, padding='valid', use_bias=False, **metaparameters)
b3x3 = BatchNormalization()(b3x3)
b3x3 = Composable.ReLU(b3x3)
# 5x5 branch
# 5x5 reduction
b5x5 = Composable.Conv2D(x, f5x5[0], (1, 1), strides=1, padding='same', use_bias=False, **metaparameters)
b5x5 = BatchNormalization()(b5x5)
b5x5 = Composable.ReLU(b5x5)
b5x5 = ZeroPadding2D((1,1))(b5x5)
b5x5 = Composable.Conv2D(b5x5, f5x5[1], (3, 3), strides=1, padding='valid', use_bias=False, **metaparameters)
b5x5 = BatchNormalization()(b5x5)
b5x5 = Composable.ReLU(b5x5)
# Pooling branch
bpool = MaxPooling2D((3, 3), strides=1, padding='same')(x)
# 1x1 projection
bpool = Composable.Conv2D(bpool, fpool[0], (1, 1), strides=1, padding='same', use_bias=False, **metaparameters)
bpool = BatchNormalization()(bpool)
bpool = Composable.ReLU(bpool)
# Concatenate the outputs (filters) of the branches
x = Concatenate()([b1x1, b3x3, b5x5, bpool])
return x
def projection_block(x, strides=1, **metaparameters):
""" Construct a ResNeXT block with projection shortcut
x : input to the block
strides : whether entry convolution is strided (i.e., (2, 2) vs (1, 1))
filters_in : number of filters (channels) at the input convolution
filters_out: number of filters (channels) at the output convolution
cardinality: width of cardinality layer
"""
filters_in = metaparameters['filters_in']
filters_out = metaparameters['filters_out']
cardinality = metaparameters['cardinality']
# Construct the projection shortcut
# Increase filters by 2X to match shape when added to output of block
shortcut = Composable.Conv2D(x,
filters_out,
kernel_size=(1, 1),
strides=strides,
padding='same',
**metaparameters)
shortcut = BatchNormalization()(shortcut)
# Dimensionality Reduction
x = Composable.Conv2D(x,
filters_in,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Cardinality (Wide) Layer (split-transform)
filters_card = filters_in // cardinality
groups = []
for i in range(cardinality):
group = Lambda(lambda z: z[:, :, :, i * filters_card:i *
filters_card + filters_card])(x)
groups.append(
Composable.Conv2D(group,
filters_card,
kernel_size=(3, 3),
strides=strides,
padding='same',
use_bias=False,
**metaparameters))
# Concatenate the outputs of the cardinality layer together (merge)
x = Concatenate()(groups)
x = BatchNormalization()(x)
x = Composable.ReLU(x)
# Dimensionality restoration
x = Composable.Conv2D(x,
filters_out,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
# Pass the output through the squeeze and excitation block
x = SEResNeXt.squeeze_excite_block(x, **metaparameters)
# Add the projection shortcut (input) to the output of the block
x = Add()([shortcut, x])
x = Composable.ReLU(x)
return x
def attention_block(x, strides=(1, 1), **metaparameters):
""" Construct an Attention Residual Block
x : input to the block
strides : strides
n_filters : number of filters
alpha : width multiplier
expansion : multiplier for expanding number of filters
squeeze : whether to include squeeze
activation: type of activation function
"""
n_filters = metaparameters['n_filters']
expansion = metaparameters['expansion']
alpha = metaparameters['alpha']
if 'alpha' in metaparameters:
alpha = metaparameters['alpha']
else:
alpha = MobileNetV3.alpha
if 'squeeze' in metaparameters:
squeeze = metaparameters['squeeze']
else:
squeeze = False
if 'activation' in metaparameters:
activation = metaparameters['activation']
else:
activation = ReLU6
del metaparameters['n_filters']
del metaparameters['kernel_size']
# Remember input
shortcut = x
# Apply the width filter to the number of feature maps for the pointwise convolution
filters = int(n_filters * alpha)
n_channels = int(x.shape[3])
# Dimensionality Expansion
# 1x1 linear convolution
x = Composable.Conv2D(x,
expansion, (1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = activation(x)
# Depthwise Convolution
x = Composable.DepthwiseConv2D(x, (3, 3),
strides,
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
x = activation(x)
# Add squeeze (dimensionality reduction)
if squeeze:
x = MobileNetV3.squeeze(x, **metaparameters)
# Linear Pointwise Convolution
x = Composable.Conv2D(x,
filters, (1, 1),
strides=(1, 1),
padding='same',
use_bias=False,
**metaparameters)
x = BatchNormalization()(x)
# Number of input filters matches the number of output filters
if n_channels == filters and strides == (1, 1):
x = Add()([shortcut, x])
return x
