def _inverted_res_block(inputs, expansion, stride, alpha, filters, block_id):
channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1
in_channels = backend.int_shape(inputs)[channel_axis]
pointwise_conv_filters = int(filters * alpha)
pointwise_filters = _make_divisible(pointwise_conv_filters, 8)
x = inputs
prefix = 'block_{}_'.format(block_id)
if block_id:
# Expand
x = layers.Conv2D(round(expansion * in_channels),
kernel_size=1,
padding='same',
use_bias=False,
activation=None,
kernel_regularizer=regularizers.l2(l2_reg),
name=prefix + 'expand')(x)
x = layers.BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name=prefix + 'expand_BN')(x)
x = layers.ReLU(6., name=prefix + 'expand_relu')(x)
else:
prefix = 'expanded_conv_'
# Depthwise
if stride == 2:
x = layers.ZeroPadding2D(padding=correct_pad(backend, x, 3),
name=prefix + 'pad')(x)
x = layers.DepthwiseConv2D(kernel_size=3,
strides=stride,
activation=None,
use_bias=False,
depthwise_regularizer=regularizers.l2(l2_reg),
padding='same' if stride == 1 else 'valid',
name=prefix + 'depthwise')(x)
x = layers.BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name=prefix + 'depthwise_BN')(x)
x = layers.ReLU(6., name=prefix + 'depthwise_relu')(x)
# Project
x = layers.Conv2D(pointwise_filters,
kernel_size=1,
padding='same',
use_bias=False,
activation=None,
kernel_regularizer=regularizers.l2(l2_reg),
name=prefix + 'project')(x)
x = layers.BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name=prefix + 'project_BN')(x)
if in_channels == pointwise_filters and stride == 1:
return layers.Add(name=prefix + 'add')([inputs, x])
return x
def _depthwise_conv_block(inputs,
pointwise_conv_filters,
alpha,
depth_multiplier=1,
strides=(1, 1),
block_id=1):
channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1
pointwise_conv_filters = int(pointwise_conv_filters * alpha)
if strides == (1, 1):
x = inputs
else:
x = layers.ZeroPadding2D(((0, 1), (0, 1)),
name='conv_pad_%d' % block_id)(inputs)
x = layers.DepthwiseConv2D(
(3, 3),
padding='same' if strides == (1, 1) else 'valid',
depth_multiplier=depth_multiplier,
strides=strides,
use_bias=False,
name='conv_dw_%d' % block_id)(x)
x = layers.BatchNormalization(axis=channel_axis,
name='conv_dw_%d_bn' % block_id)(x)
x = layers.ReLU(6., name='conv_dw_%d_relu' % block_id)(x)
x = layers.Conv2D(pointwise_conv_filters, (1, 1),
padding='same',
use_bias=False,
strides=(1, 1),
name='conv_pw_%d' % block_id)(x)
x = layers.BatchNormalization(axis=channel_axis,
name='conv_pw_%d_bn' % block_id)(x)
return layers.ReLU(6., name='conv_pw_%d_relu' % block_id)(x)
def convs(x, y):
a = layers.DepthwiseConv2D(
8,
2,
depth_multiplier=2,
use_bias=True,
depthwise_initializer=init_ops.constant_initializer(w1),
bias_initializer=init_ops.constant_initializer(0.5),
name='conv1')(x)
b = layers.DepthwiseConv2D(
4,
2,
depth_multiplier=2,
use_bias=True,
depthwise_initializer=init_ops.constant_initializer(w2),
bias_initializer=init_ops.constant_initializer(0.2),
name='conv2')(y)
return a, b
def MobilenetSeparableConv2D(filters,
kernel_size,
strides=(1, 1),
padding='valid',
use_bias=True):
return compose(
kl.DepthwiseConv2D(kernel_size,
padding=padding,
use_bias=use_bias,
strides=strides), kl.BatchNormalization(),
kl.LeakyReLU(),
kl.Conv2D(filters, 1, padding='same', use_bias=use_bias, strides=1),
kl.BatchNormalization(), kl.LeakyReLU())
def _inverted_res_block(x, expansion, filters, kernel_size, stride, se_ratio,
activation, block_id):
channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1
shortcut = x
prefix = 'expanded_conv/'
infilters = backend.int_shape(x)[channel_axis]
if block_id:
# Expand
prefix = 'expanded_conv_{}/'.format(block_id)
x = layers.Conv2D(_depth(infilters * expansion),
kernel_size=1,
padding='same',
use_bias=False,
name=prefix + 'expand')(x)
x = layers.BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name=prefix + 'expand/BatchNorm')(x)
x = activation(x)
if stride == 2:
x = layers.ZeroPadding2D(padding=imagenet_utils.correct_pad(
x, kernel_size),
name=prefix + 'depthwise/pad')(x)
x = layers.DepthwiseConv2D(kernel_size,
strides=stride,
padding='same' if stride == 1 else 'valid',
use_bias=False,
name=prefix + 'depthwise')(x)
x = layers.BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name=prefix + 'depthwise/BatchNorm')(x)
x = activation(x)
if se_ratio:
x = _se_block(x, _depth(infilters * expansion), se_ratio, prefix)
x = layers.Conv2D(filters,
kernel_size=1,
padding='same',
use_bias=False,
name=prefix + 'project')(x)
x = layers.BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name=prefix + 'project/BatchNorm')(x)
if stride == 1 and infilters == filters:
x = layers.Add(name=prefix + 'Add')([shortcut, x])
return x
def block(inputs):
if block_args.expand_ratio != 1:
x = KL.Conv2D(filters,
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding='same',
use_bias=False)(inputs)
x = KL.BatchNormalization(axis=channel_axis,
momentum=batch_norm_momentum,
epsilon=batch_norm_epsilon)(x)
x = Swish()(x)
else:
x = inputs
x = KL.DepthwiseConv2D([kernel_size, kernel_size],
strides=block_args.strides,
depthwise_initializer=conv_kernel_initializer,
padding='same',
use_bias=False)(x)
x = KL.BatchNormalization(axis=channel_axis,
momentum=batch_norm_momentum,
epsilon=batch_norm_epsilon)(x)
x = Swish()(x)
if has_se:
x = SEBlock(block_args, global_params)(x)
# output phase
x = KL.Conv2D(block_args.output_filters,
kernel_size=[1, 1],
strides=[1, 1],
kernel_initializer=conv_kernel_initializer,
padding='same',
use_bias=False)(x)
x = KL.BatchNormalization(axis=channel_axis,
momentum=batch_norm_momentum,
epsilon=batch_norm_epsilon)(x)
if block_args.id_skip:
if all(s == 1 for s in block_args.strides
) and block_args.input_filters == block_args.output_filters:
# only apply drop_connect if skip presents.
if drop_connect_rate:
x = DropConnect(drop_connect_rate)(x)
x = KL.Add()([x, inputs])
return x
def get_model_depthwise_conv(input_shape):
inputs = tf.keras.Input(shape=input_shape[1:], name='input')
conv1 = layers.Conv2D(8, 1, name='conv1', kernel_initializer='Ones',
bias_initializer='Ones')
conv2 = layers.Conv2D(128, 3, name='conv2', kernel_initializer='Ones',
bias_initializer='Ones')
conv_depthwise = layers.DepthwiseConv2D(3, name='conv4', kernel_initializer='Ones',
bias_initializer='Ones')
conv3 = layers.Conv2D(8, 3, name='conv3', kernel_initializer='Ones',
bias_initializer='Ones')
flatten = layers.Flatten()
linear = layers.Dense(128)
x = conv1(inputs)
x = conv2(x)
x = conv_depthwise(x)
x = conv3(x)
x = linear(flatten(x))
return tf.keras.Model(inputs=inputs, outputs=[x])
def seperable_conv2d(x,
filters,
name,
kernel_size=1,
stride=1,
padding='same'):
"""
:param x:
:param filters:
:param name:
:param kernel_size:
:param stride:
:param padding:
:return:
"""
prefix = name
channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1
x = layers.DepthwiseConv2D(kernel_size=kernel_size,
strides=stride,
activation=None,
use_bias=False,
padding=padding,
depthwise_regularizer=regularizers.l2(l2_reg),
name=prefix + 'depthwise')(x)
x = layers.BatchNormalization(axis=channel_axis,
epsilon=1e-3,
momentum=0.999,
name=prefix + 'depthwise_BN')(x)
x = layers.ReLU(6., name=prefix + 'depthwise_relu')(x)
# Project
x = layers.Conv2D(filters,
kernel_size=1,
padding='same',
use_bias=False,
activation=None,
kernel_regularizer=regularizers.l2(l2_reg),
name=prefix + 'project')(x)
return x
def Conv(x,
config,
num_filter=1,
kernel_size=(1, 1),
stride=(1, 1),
padding="same",
group=False,
name=None,
suffix=''):
conv_name = "%s%s_conv2d" % (name, suffix)
if group:
x = layers.DepthwiseConv2D(kernel_size=kernel_size,
strides=stride,
padding=padding,
use_bias=False,
name=conv_name)(x)
else:
x = layers.Conv2D(filters=num_filter,
kernel_size=kernel_size,
strides=stride,
padding=padding,
use_bias=False,
name=conv_name)(x)
x = layers.BatchNormalization(momentum=config["bn_mom"],
name='%s%s_batchnorm' % (name, suffix))(x)
if config["net_act"] == "relu":
activation = tf.nn.relu_layer
elif config["net_act"] == "prelu":
activation = tf.nn.leaky_relu
else:
activation = None
print("can not get activation layer from {}".format(config["net_act"]))
exit(0)
x = layers.Activation(activation=activation,
name='%s%s_relu' % (name, suffix))(x)
return x
def block3(x,
filters,
kernel_size=3,
stride=1,
groups=32,
conv_shortcut=True,
name=None):
"""A residual block.
Arguments:
x: input tensor.
filters: integer, filters of the bottleneck layer.
kernel_size: default 3, kernel size of the bottleneck layer.
stride: default 1, stride of the first layer.
groups: default 32, group size for grouped convolution.
conv_shortcut: default True, use convolution shortcut if True,
otherwise identity shortcut.
name: string, block label.
Returns:
Output tensor for the residual block.
"""
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
if conv_shortcut:
shortcut = layers.Conv2D((64 // groups) * filters,
1,
strides=stride,
use_bias=False,
name=name + '_0_conv')(x)
shortcut = tf.keras.layers.experimental.SyncBatchNormalization(
axis=bn_axis, name=name + '_0_bn')(shortcut)
else:
shortcut = x
x = layers.Conv2D(filters, 1, use_bias=False, name=name + '_1_conv')(x)
x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis,
name=name +
'_1_bn')(x)
x = layers.Activation('relu', name=name + '_1_relu')(x)
c = filters // groups
x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x)
x = layers.DepthwiseConv2D(kernel_size,
strides=stride,
depth_multiplier=c,
use_bias=False,
name=name + '_2_conv')(x)
x_shape = backend.int_shape(x)[1:-1]
x = layers.Reshape(x_shape + (groups, c, c))(x)
x = layers.Lambda(lambda x: sum(x[:, :, :, :, i] for i in range(c)),
name=name + '_2_reduce')(x)
x = layers.Reshape(x_shape + (filters, ))(x)
x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis,
name=name +
'_2_bn')(x)
x = layers.Activation('relu', name=name + '_2_relu')(x)
x = layers.Conv2D((64 // groups) * filters,
1,
use_bias=False,
name=name + '_3_conv')(x)
x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis,
name=name +
'_3_bn')(x)
x = layers.Add(name=name + '_add')([shortcut, x])
x = layers.Activation('relu', name=name + '_out')(x)
return x
def _depthwise_conv_block(inputs, pointwise_conv_filters, alpha,
depth_multiplier=1, strides=(1, 1), block_id=1):
"""Adds a depthwise convolution block.
A depthwise convolution block consists of a depthwise conv,
batch normalization, relu6, pointwise convolution,
batch normalization and relu6 activation.
# Arguments
inputs: Input tensor of shape `(rows, cols, channels)`
(with `channels_last` data format) or
(channels, rows, cols) (with `channels_first` data format).
pointwise_conv_filters: Integer, the dimensionality of the output space
(i.e. the number of output filters in the pointwise convolution).
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
depth_multiplier: The number of depthwise convolution output channels
for each input channel.
The total number of depthwise convolution output
channels will be equal to `filters_in * depth_multiplier`.
strides: An integer or tuple/list of 2 integers,
specifying the strides of the convolution
along the width and height.
Can be a single integer to specify the same value for
all spatial dimensions.
Specifying any stride value != 1 is incompatible with specifying
any `dilation_rate` value != 1.
block_id: Integer, a unique identification designating
the block number.
# Input shape
4D tensor with shape:
`(batch, channels, rows, cols)` if data_format='channels_first'
or 4D tensor with shape:
`(batch, rows, cols, channels)` if data_format='channels_last'.
# Output shape
4D tensor with shape:
`(batch, filters, new_rows, new_cols)`
if data_format='channels_first'
or 4D tensor with shape:
`(batch, new_rows, new_cols, filters)`
if data_format='channels_last'.
`rows` and `cols` values might have changed due to stride.
# Returns
Output tensor of block.
"""
# channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1
pointwise_conv_filters = int(pointwise_conv_filters * alpha)
if strides == (1, 1):
x = inputs
else:
x = layers.ZeroPadding2D(((1, 1), (1, 1)), name='conv_pad_%d' % block_id)(inputs)
x = layers.DepthwiseConv2D((3, 3),
padding='same' if strides == (1, 1) else 'valid',
depth_multiplier=depth_multiplier,
strides=strides,
use_bias=False,
name='conv_dw_%d' % block_id)(x)
x = layers.BatchNormalization(name='conv_dw_%d_bn' % block_id)(x)
x = layers.ReLU(name='conv_dw_%d_relu' % block_id)(x)
x = layers.Conv2D(pointwise_conv_filters, (1, 1),
padding='same',
use_bias=False,
strides=(1, 1),
name='conv_pw_%d' % block_id)(x)
x = layers.BatchNormalization(name='conv_pw_%d_bn' % block_id)(x)
return layers.LeakyReLU(name='conv_pw_%d_relu' % block_id)(x)
def block(inputs,
activation='swish',
drop_rate=0.,
name='',
filters_in=32,
filters_out=16,
kernel_size=3,
strides=1,
expand_ratio=1,
se_ratio=0.,
id_skip=True):
"""An inverted residual block.
Arguments:
inputs: input tensor.
activation: activation function.
drop_rate: float between 0 and 1, fraction of the input units to drop.
name: string, block label.
filters_in: integer, the number of input filters.
filters_out: integer, the number of output filters.
kernel_size: integer, the dimension of the convolution window.
strides: integer, the stride of the convolution.
expand_ratio: integer, scaling coefficient for the input filters.
se_ratio: float between 0 and 1, fraction to squeeze the input filters.
id_skip: boolean.
Returns:
output tensor for the block.
"""
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
# Expansion phase
filters = filters_in * expand_ratio
if expand_ratio != 1:
x = layers.Conv2D(
filters,
1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'expand_conv')(
inputs)
x = layers.BatchNormalization(axis=bn_axis, name=name + 'expand_bn')(x)
x = layers.Activation(activation, name=name + 'expand_activation')(x)
else:
x = inputs
# Depthwise Convolution
if strides == 2:
x = layers.ZeroPadding2D(
padding=imagenet_utils.correct_pad(x, kernel_size),
name=name + 'dwconv_pad')(x)
conv_pad = 'valid'
else:
conv_pad = 'same'
x = layers.DepthwiseConv2D(
kernel_size,
strides=strides,
padding=conv_pad,
use_bias=False,
depthwise_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'dwconv')(x)
x = layers.BatchNormalization(axis=bn_axis, name=name + 'bn')(x)
x = layers.Activation(activation, name=name + 'activation')(x)
# Squeeze and Excitation phase
if 0 < se_ratio <= 1:
filters_se = max(1, int(filters_in * se_ratio))
se = layers.GlobalAveragePooling2D(name=name + 'se_squeeze')(x)
se = layers.Reshape((1, 1, filters), name=name + 'se_reshape')(se)
se = layers.Conv2D(
filters_se,
1,
padding='same',
activation=activation,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'se_reduce')(
se)
se = layers.Conv2D(
filters,
1,
padding='same',
activation='sigmoid',
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'se_expand')(se)
x = layers.multiply([x, se], name=name + 'se_excite')
# Output phase
x = layers.Conv2D(
filters_out,
1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + 'project_conv')(x)
x = layers.BatchNormalization(axis=bn_axis, name=name + 'project_bn')(x)
if id_skip and strides == 1 and filters_in == filters_out:
if drop_rate > 0:
x = layers.Dropout(
drop_rate, noise_shape=(None, 1, 1, 1), name=name + 'drop')(x)
x = layers.add([x, inputs], name=name + 'add')
return x
def conv2d(x,
in_channels,
out_channels,
kernel_size,
strides=1,
padding=0,
dilation=1,
groups=1,
use_bias=True,
name="conv2d"):
"""
Convolution 2D layer wrapper.
Parameters:
----------
x : keras.backend tensor/variable/symbol
Input tensor/variable/symbol.
in_channels : int
Number of input channels.
out_channels : int
Number of output channels.
kernel_size : int or tuple/list of 2 int
Convolution window size.
strides : int or tuple/list of 2 int
Strides of the convolution.
padding : int or tuple/list of 2 int
Padding value for convolution layer.
dilation : int or tuple/list of 2 int, default 1
Dilation value for convolution layer.
groups : int, default 1
Number of groups.
use_bias : bool, default False
Whether the layer uses a bias vector.
name : str, default 'conv2d'
Layer name.
Returns
-------
keras.backend tensor/variable/symbol
Resulted tensor/variable/symbol.
"""
if isinstance(kernel_size, int):
kernel_size = (kernel_size, kernel_size)
if isinstance(strides, int):
strides = (strides, strides)
if isinstance(padding, int):
padding = (padding, padding)
if isinstance(dilation, int):
dilation = (dilation, dilation)
extra_pad = False
if K.backend() == "tensorflow":
if (padding[0] > 0) or (padding[1] > 0):
import tensorflow as tf
x = nn.Lambda((lambda z: tf.pad(z, [[0, 0], [
0, 0
], list(padding), list(padding)])) if is_channels_first() else (
lambda z: tf.pad(z, [[0, 0],
list(padding),
list(padding), [0, 0]])))(x)
if not ((padding[0] == padding[1]) and
(kernel_size[0] == kernel_size[1]) and
(kernel_size[0] // 2 == padding[0])):
extra_pad = True
padding_ke = "valid"
else:
if (padding[0] == padding[1]) and (padding[0] == 0):
padding_ke = "valid"
elif (padding[0]
== padding[1]) and (kernel_size[0]
== kernel_size[1]) and (kernel_size[0] // 2
== padding[0]):
padding_ke = "same"
else:
x = nn.ZeroPadding2D(padding=padding, name=name + "/pad")(x)
padding_ke = "valid"
extra_pad = True
if groups == 1:
if extra_pad:
name = name + "/conv"
x = nn.Conv2D(filters=out_channels,
kernel_size=kernel_size,
strides=strides,
padding=padding_ke,
dilation_rate=dilation,
use_bias=use_bias,
name=name)(x)
elif (groups == out_channels) and (out_channels == in_channels):
assert (dilation[0] == 1) and (dilation[1] == 1)
if extra_pad:
name = name + "/conv"
x = nn.DepthwiseConv2D(kernel_size=kernel_size,
strides=strides,
padding=padding_ke,
use_bias=use_bias,
name=name)(x)
else:
assert (in_channels % groups == 0)
assert (out_channels % groups == 0)
none_batch = (x.shape[0] is None)
in_group_channels = in_channels // groups
out_group_channels = out_channels // groups
group_list = []
for gi in range(groups):
xi = nn.Lambda((lambda z: z[:, gi * in_group_channels:
(gi + 1) * in_group_channels, :, :]
) if is_channels_first() else (
lambda z: z[:, :, :, gi * in_group_channels:
(gi + 1) * in_group_channels]))(x)
xi = nn.Conv2D(filters=out_group_channels,
kernel_size=kernel_size,
strides=strides,
padding=padding_ke,
dilation_rate=dilation,
use_bias=use_bias,
name=name + "/convgroup{}".format(gi + 1))(xi)
group_list.append(xi)
x = nn.concatenate(group_list,
axis=get_channel_axis(),
name=name + "/concat")
if none_batch and (x.shape[0] is not None):
x.shape = (None, ) + x.shape[1:]
return x
def mbconvblock(inputs,
activations='swish',
droprate=0.,
name='',
filters_in=32,
filters_out=16,
kernel_size=3,
strides=1,
expand_ratio=1,
se_ratio=0.,
id_skip=True,
attn=True):
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
#expansion phase
filters = filters_in * expand_ratio
if expand_ratio != 1:
x = layers.Conv2D(filters,
kernel_size=1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER)(inputs)
x = layers.BatchNormalization(axis=bn_axis)(x)
x = layers.Activation(activations)(x) #swish activation
else:
x = inputs
#depthwise conv
if strides == 2:
x = layers.ZeroPadding2D(
padding=imagenet_utils.correct_pad(x, kernel_size))(x)
conv_pad = 'valid'
else:
conv_pad = 'same'
x = layers.DepthwiseConv2D(
kernel_size=kernel_size,
strides=strides,
padding=conv_pad,
use_bias=False,
depthwise_initializer=CONV_KERNEL_INITIALIZER)(x)
x = layers.BatchNormalization(axis=bn_axis)(x)
x = layers.Activation(activations)(x)
#squeeze and excitation
if 0 < se_ratio <= 1:
filters_se = max(1, int(filters_in * se_ratio))
se = layers.GlobalAveragePooling2D()(x)
se = layers.Reshape((1, 1, filters))(se)
se = layers.Conv2D(filters_se,
kernel_size=1,
padding='same',
activation=activations,
kernel_initializer=CONV_KERNEL_INITIALIZER)(
se) #reduce
se = layers.Conv2D(filters,
kernel_size=1,
padding='same',
activation='sigmoid',
kernel_initializer=CONV_KERNEL_INITIALIZER)(
se) #expand
x = layers.multiply([x, se])
if attn:
avgp = layers.Lambda(lambda x: K.mean(x, axis=3, keepdims=True))(x)
maxp = layers.Lambda(lambda x: K.max(x, axis=3, keepdims=True))(x)
conc = layers.Concatenate(axis=3)([avgp, maxp])
attn_mask = layers.Conv2D(
filters=1,
kernel_size=1,
activation='sigmoid',
padding='same',
name=name + '_Attn',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER)(conc)
#Output phase
x = layers.Conv2D(filters_out,
kernel_size=1,
padding='same',
use_bias=False,
kernel_initializer=CONV_KERNEL_INITIALIZER)(x)
x = layers.BatchNormalization(axis=bn_axis)(x)
if id_skip and strides == 1 and filters_in == filters_out:
if droprate > 0:
x = layers.Dropout(droprate, noise_shape=(None, 1, 1, 1))(x)
x = layers.add([x, inputs])
if attn:
x = layers.multiply([x, attn_mask])
return x
convolution_5_output = convolution_5(convolution_layer_1_output)
convolution_6_output = convolution_6(convolution_layer_1_output)
convolution_7_output = convolution_7(convolution_layer_1_output)
convolution_8_output = convolution_8(convolution_layer_1_output)
convolution_layer_2_output = concatenate([
convolution_5_output, convolution_6_output, convolution_7_output,
convolution_8_output
])
reshape_1 = layers.Reshape((1, word_length, 128))
reshape_1_output = reshape_1(convolution_layer_2_output)
depthwise_convolution_1 = layers.DepthwiseConv2D((1, word_length),
depth_multiplier=8,
activation="tanh")
depthwise_convolution_1_output = depthwise_convolution_1(reshape_1_output)
flatten_output_1 = flatten(depthwise_convolution_1_output)
dense_1 = layers.Dense(embedding_size, activation="tanh")
dense_1_output = dense_1(flatten_output_1)
encoder = models.Model(inputs=[input_1], outputs=[dense_1_output])
encoder.summary()
input_2 = layers.Input(shape=(embedding_size, ))
dense_2 = layers.Dense(word_length * 32, activation="tanh")
