def activation_block(inputs, act_fn, trainable=True, block_name='activation'):
with tf.variable_scope(block_name):
if act_fn == "relu":
return layers.relu(inputs)
if act_fn == "leaky_relu":
return layers.leaky_relu(inputs, alpha=0.2)
if act_fn == "prelu_shared":
return layers.prelu(inputs,
channel_shared=True,
trainable=trainable)
if act_fn == "prelu_not_shared":
return layers.prelu(inputs,
channel_shared=False,
trainable=trainable)
if act_fn == "selu":
return layers.selu(inputs)
if act_fn == "crelu":
return layers.crelu(inputs)
if act_fn == "elu":
return layers.elu(inputs)
raise ValueError("Unknown activation function: %s - Authorized: %s" %
(act_fn, authorized_activation_fn))
def squeeze_excitation_layer(
inputs,
ratio,
training=True,
data_format='NCHW',
kernel_initializer=tf.compat.v1.variance_scaling_initializer(),
bias_initializer=tf.zeros_initializer(),
name="squeeze_excitation_layer"):
if data_format not in ['NHWC', 'NCHW']:
raise ValueError(
"Unknown data format: `%s` (accepted: ['NHWC', 'NCHW'])" %
data_format)
in_shape = inputs.get_shape()
num_channels = in_shape[1] if data_format == "NCHW" else in_shape[-1]
with tf.variable_scope(name):
net = inputs
# squeeze
squeeze = layers.reduce_mean(net,
keepdims=False,
data_format=data_format,
name='squeeze_spatial_mean')
# fc + relu
excitation = layers.dense(inputs=squeeze,
units=num_channels // ratio,
use_bias=True,
trainable=training,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)
excitation = layers.relu(excitation)
# fc + sigmoid
excitation = layers.dense(inputs=excitation,
units=num_channels,
use_bias=True,
trainable=training,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)
excitation = layers.sigmoid(excitation)
out_shape = [-1, num_channels, 1, 1
] if data_format == "NCHW" else [-1, 1, 1, num_channels]
excitation = tf.reshape(excitation, out_shape)
net = net * excitation
return net
def conv2d_block(inputs,
n_channels,
kernel_size=(3, 3),
strides=(2, 2),
mode='SAME',
use_batch_norm=True,
activation='relu',
is_training=True,
data_format='NHWC',
conv2d_hparams=None,
batch_norm_hparams=None,
name='conv2d'):
if not isinstance(conv2d_hparams, tf.contrib.training.HParams):
raise ValueError(
"The paramater `conv2d_hparams` is not of type `HParams`")
if not isinstance(batch_norm_hparams,
tf.contrib.training.HParams) and use_batch_norm:
raise ValueError(
"The paramater `conv2d_hparams` is not of type `HParams`")
with tf.variable_scope(name):
if mode != 'SAME_RESNET':
net = layers.conv2d(
inputs,
n_channels=n_channels,
kernel_size=kernel_size,
strides=strides,
padding=mode,
data_format=data_format,
use_bias=not use_batch_norm,
trainable=is_training,
kernel_initializer=conv2d_hparams.kernel_initializer,
bias_initializer=conv2d_hparams.bias_initializer,
)
else: # Special padding mode for ResNet models
if strides == (1, 1):
net = layers.conv2d(
inputs,
n_channels=n_channels,
kernel_size=kernel_size,
strides=strides,
padding='SAME',
data_format=data_format,
use_bias=not use_batch_norm,
trainable=is_training,
kernel_initializer=conv2d_hparams.kernel_initializer,
bias_initializer=conv2d_hparams.bias_initializer,
)
else:
rate = 1 # Unused (for 'a trous' convolutions)
kernel_height_effective = kernel_size[0] + (kernel_size[0] -
1) * (rate - 1)
pad_h_beg = (kernel_height_effective - 1) // 2
pad_h_end = kernel_height_effective - 1 - pad_h_beg
kernel_width_effective = kernel_size[1] + (kernel_size[1] -
1) * (rate - 1)
pad_w_beg = (kernel_width_effective - 1) // 2
pad_w_end = kernel_width_effective - 1 - pad_w_beg
padding = [[0, 0], [pad_h_beg, pad_h_end],
[pad_w_beg, pad_w_end], [0, 0]]
if data_format == 'NCHW':
padding = [padding[0], padding[3], padding[1], padding[2]]
padded_inputs = tf.pad(inputs, padding)
net = layers.conv2d(
padded_inputs, # inputs,
n_channels=n_channels,
kernel_size=kernel_size,
strides=strides,
padding='VALID',
data_format=data_format,
use_bias=not use_batch_norm,
trainable=is_training,
kernel_initializer=conv2d_hparams.kernel_initializer,
bias_initializer=conv2d_hparams.bias_initializer,
)
if use_batch_norm:
net = layers.batch_norm(
net,
decay=batch_norm_hparams.decay,
epsilon=batch_norm_hparams.epsilon,
scale=batch_norm_hparams.scale,
center=batch_norm_hparams.center,
is_training=is_training,
data_format=data_format,
param_initializers=batch_norm_hparams.param_initializers)
if activation == 'relu':
net = layers.relu(net, name='relu')
elif activation == 'tanh':
net = layers.tanh(net, name='tanh')
elif activation != 'linear' and activation is not None:
raise KeyError('Invalid activation type: `%s`' % activation)
return net
def bottleneck_block(inputs,
depth,
depth_bottleneck,
stride,
training=True,
data_format='NCHW',
conv2d_hparams=None,
batch_norm_hparams=None,
block_name="bottleneck_block"):
if data_format not in ['NHWC', 'NCHW']:
raise ValueError(
"Unknown data format: `%s` (accepted: ['NHWC', 'NCHW'])" %
data_format)
if not isinstance(conv2d_hparams, tf.contrib.training.HParams):
raise ValueError(
"The paramater `conv2d_hparams` is not of type `HParams`")
if not isinstance(batch_norm_hparams, tf.contrib.training.HParams):
raise ValueError(
"The paramater `conv2d_hparams` is not of type `HParams`")
in_shape = inputs.get_shape()
in_size = in_shape[1] if data_format == "NCHW" else in_shape[-1]
with tf.variable_scope(block_name):
with tf.variable_scope("shortcut"):
if depth == in_size:
if stride == 1:
shortcut = tf.identity(inputs)
else:
shortcut = layers.average_pooling2d(
inputs,
pool_size=(1, 1),
strides=(stride, stride),
padding='valid',
data_format='channels_first'
if data_format == 'NCHW' else 'channels_last',
name="average_pooling2d",
)
else:
shortcut = blocks.conv2d_block(
inputs,
n_channels=depth,
kernel_size=(1, 1),
strides=(stride, stride),
mode='SAME',
use_batch_norm=True,
activation=
None, # Applied at the end after addition with bottleneck
is_training=training,
data_format=data_format,
conv2d_hparams=conv2d_hparams,
batch_norm_hparams=batch_norm_hparams)
bottleneck = blocks.conv2d_block(inputs,
n_channels=depth_bottleneck,
kernel_size=(1, 1),
strides=(1, 1),
mode='SAME',
use_batch_norm=True,
activation='relu',
is_training=training,
data_format=data_format,
conv2d_hparams=conv2d_hparams,
batch_norm_hparams=batch_norm_hparams,
name='bottleneck_1')
bottleneck = blocks.conv2d_block(bottleneck,
n_channels=depth_bottleneck,
kernel_size=(3, 3),
strides=(stride, stride),
mode='SAME_RESNET',
use_batch_norm=True,
activation='relu',
is_training=training,
data_format=data_format,
conv2d_hparams=conv2d_hparams,
batch_norm_hparams=batch_norm_hparams,
name='bottleneck_2')
bottleneck = blocks.conv2d_block(
bottleneck,
n_channels=depth,
kernel_size=(1, 1),
strides=(1, 1),
mode='SAME',
use_batch_norm=True,
activation=None, # Applied at the end after addition with shortcut
is_training=training,
data_format=data_format,
conv2d_hparams=conv2d_hparams,
batch_norm_hparams=batch_norm_hparams,
name='bottleneck_3')
return layers.relu(shortcut + bottleneck, name='relu')
def conv2d_block(
inputs,
n_channels,
kernel_size=(3, 3),
strides=(2, 2),
mode='SAME',
use_batch_norm=True,
activation='relu',
is_training=True,
data_format='NHWC',
conv2d_hparams=None,
batch_norm_hparams=None,
name='conv2d',
cardinality=1,
):
if not isinstance(conv2d_hparams, tf.contrib.training.HParams):
raise ValueError(
"The paramater `conv2d_hparams` is not of type `HParams`")
if not isinstance(batch_norm_hparams,
tf.contrib.training.HParams) and use_batch_norm:
raise ValueError(
"The paramater `conv2d_hparams` is not of type `HParams`")
with tf.variable_scope(name):
if cardinality == 1:
net = layers.conv2d(
inputs,
n_channels=n_channels,
kernel_size=kernel_size,
strides=strides,
padding=mode,
data_format=data_format,
use_bias=not use_batch_norm,
trainable=is_training,
kernel_initializer=conv2d_hparams.kernel_initializer,
bias_initializer=conv2d_hparams.bias_initializer)
else:
group_filter = tf.get_variable(
name=name + 'group_filter',
shape=[3, 3, n_channels // cardinality, n_channels],
trainable=is_training,
dtype=tf.float32)
net = tf.nn.conv2d(inputs,
group_filter,
strides=strides,
padding='SAME',
data_format=data_format)
if use_batch_norm:
net = layers.batch_norm(
net,
decay=batch_norm_hparams.decay,
epsilon=batch_norm_hparams.epsilon,
scale=batch_norm_hparams.scale,
center=batch_norm_hparams.center,
is_training=is_training,
data_format=data_format,
param_initializers=batch_norm_hparams.param_initializers)
if activation == 'relu':
net = layers.relu(net, name='relu')
elif activation == 'tanh':
net = layers.tanh(net, name='tanh')
elif activation != 'linear' and activation is not None:
raise KeyError('Invalid activation type: `%s`' % activation)
return net