def _inverted_res_block(inputs, expansion, stride, alpha, filters, block_id): channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 in_channels = backend.int_shape(inputs)[channel_axis] pointwise_conv_filters = int(filters * alpha) pointwise_filters = _make_divisible(pointwise_conv_filters, 8) x = inputs prefix = 'block_{}_'.format(block_id) if block_id: # Expand x = layers.Conv2D(round(expansion * in_channels), kernel_size=1, padding='same', use_bias=False, activation=None, kernel_regularizer=regularizers.l2(l2_reg), name=prefix + 'expand')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'expand_BN')(x) x = layers.ReLU(6., name=prefix + 'expand_relu')(x) else: prefix = 'expanded_conv_' # Depthwise if stride == 2: x = layers.ZeroPadding2D(padding=correct_pad(backend, x, 3), name=prefix + 'pad')(x) x = layers.DepthwiseConv2D(kernel_size=3, strides=stride, activation=None, use_bias=False, depthwise_regularizer=regularizers.l2(l2_reg), padding='same' if stride == 1 else 'valid', name=prefix + 'depthwise')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise_BN')(x) x = layers.ReLU(6., name=prefix + 'depthwise_relu')(x) # Project x = layers.Conv2D(pointwise_filters, kernel_size=1, padding='same', use_bias=False, activation=None, kernel_regularizer=regularizers.l2(l2_reg), name=prefix + 'project')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'project_BN')(x) if in_channels == pointwise_filters and stride == 1: return layers.Add(name=prefix + 'add')([inputs, x]) return x
def _depthwise_conv_block(inputs, pointwise_conv_filters, alpha, depth_multiplier=1, strides=(1, 1), block_id=1): channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 pointwise_conv_filters = int(pointwise_conv_filters * alpha) if strides == (1, 1): x = inputs else: x = layers.ZeroPadding2D(((0, 1), (0, 1)), name='conv_pad_%d' % block_id)(inputs) x = layers.DepthwiseConv2D( (3, 3), padding='same' if strides == (1, 1) else 'valid', depth_multiplier=depth_multiplier, strides=strides, use_bias=False, name='conv_dw_%d' % block_id)(x) x = layers.BatchNormalization(axis=channel_axis, name='conv_dw_%d_bn' % block_id)(x) x = layers.ReLU(6., name='conv_dw_%d_relu' % block_id)(x) x = layers.Conv2D(pointwise_conv_filters, (1, 1), padding='same', use_bias=False, strides=(1, 1), name='conv_pw_%d' % block_id)(x) x = layers.BatchNormalization(axis=channel_axis, name='conv_pw_%d_bn' % block_id)(x) return layers.ReLU(6., name='conv_pw_%d_relu' % block_id)(x)
def convs(x, y): a = layers.DepthwiseConv2D( 8, 2, depth_multiplier=2, use_bias=True, depthwise_initializer=init_ops.constant_initializer(w1), bias_initializer=init_ops.constant_initializer(0.5), name='conv1')(x) b = layers.DepthwiseConv2D( 4, 2, depth_multiplier=2, use_bias=True, depthwise_initializer=init_ops.constant_initializer(w2), bias_initializer=init_ops.constant_initializer(0.2), name='conv2')(y) return a, b
def MobilenetSeparableConv2D(filters, kernel_size, strides=(1, 1), padding='valid', use_bias=True): return compose( kl.DepthwiseConv2D(kernel_size, padding=padding, use_bias=use_bias, strides=strides), kl.BatchNormalization(), kl.LeakyReLU(), kl.Conv2D(filters, 1, padding='same', use_bias=use_bias, strides=1), kl.BatchNormalization(), kl.LeakyReLU())
def _inverted_res_block(x, expansion, filters, kernel_size, stride, se_ratio, activation, block_id): channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 shortcut = x prefix = 'expanded_conv/' infilters = backend.int_shape(x)[channel_axis] if block_id: # Expand prefix = 'expanded_conv_{}/'.format(block_id) x = layers.Conv2D(_depth(infilters * expansion), kernel_size=1, padding='same', use_bias=False, name=prefix + 'expand')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'expand/BatchNorm')(x) x = activation(x) if stride == 2: x = layers.ZeroPadding2D(padding=imagenet_utils.correct_pad( x, kernel_size), name=prefix + 'depthwise/pad')(x) x = layers.DepthwiseConv2D(kernel_size, strides=stride, padding='same' if stride == 1 else 'valid', use_bias=False, name=prefix + 'depthwise')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise/BatchNorm')(x) x = activation(x) if se_ratio: x = _se_block(x, _depth(infilters * expansion), se_ratio, prefix) x = layers.Conv2D(filters, kernel_size=1, padding='same', use_bias=False, name=prefix + 'project')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'project/BatchNorm')(x) if stride == 1 and infilters == filters: x = layers.Add(name=prefix + 'Add')([shortcut, x]) return x
def block(inputs): if block_args.expand_ratio != 1: x = KL.Conv2D(filters, kernel_size=[1, 1], strides=[1, 1], kernel_initializer=conv_kernel_initializer, padding='same', use_bias=False)(inputs) x = KL.BatchNormalization(axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon)(x) x = Swish()(x) else: x = inputs x = KL.DepthwiseConv2D([kernel_size, kernel_size], strides=block_args.strides, depthwise_initializer=conv_kernel_initializer, padding='same', use_bias=False)(x) x = KL.BatchNormalization(axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon)(x) x = Swish()(x) if has_se: x = SEBlock(block_args, global_params)(x) # output phase x = KL.Conv2D(block_args.output_filters, kernel_size=[1, 1], strides=[1, 1], kernel_initializer=conv_kernel_initializer, padding='same', use_bias=False)(x) x = KL.BatchNormalization(axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon)(x) if block_args.id_skip: if all(s == 1 for s in block_args.strides ) and block_args.input_filters == block_args.output_filters: # only apply drop_connect if skip presents. if drop_connect_rate: x = DropConnect(drop_connect_rate)(x) x = KL.Add()([x, inputs]) return x
def get_model_depthwise_conv(input_shape): inputs = tf.keras.Input(shape=input_shape[1:], name='input') conv1 = layers.Conv2D(8, 1, name='conv1', kernel_initializer='Ones', bias_initializer='Ones') conv2 = layers.Conv2D(128, 3, name='conv2', kernel_initializer='Ones', bias_initializer='Ones') conv_depthwise = layers.DepthwiseConv2D(3, name='conv4', kernel_initializer='Ones', bias_initializer='Ones') conv3 = layers.Conv2D(8, 3, name='conv3', kernel_initializer='Ones', bias_initializer='Ones') flatten = layers.Flatten() linear = layers.Dense(128) x = conv1(inputs) x = conv2(x) x = conv_depthwise(x) x = conv3(x) x = linear(flatten(x)) return tf.keras.Model(inputs=inputs, outputs=[x])
def seperable_conv2d(x, filters, name, kernel_size=1, stride=1, padding='same'): """ :param x: :param filters: :param name: :param kernel_size: :param stride: :param padding: :return: """ prefix = name channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 x = layers.DepthwiseConv2D(kernel_size=kernel_size, strides=stride, activation=None, use_bias=False, padding=padding, depthwise_regularizer=regularizers.l2(l2_reg), name=prefix + 'depthwise')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise_BN')(x) x = layers.ReLU(6., name=prefix + 'depthwise_relu')(x) # Project x = layers.Conv2D(filters, kernel_size=1, padding='same', use_bias=False, activation=None, kernel_regularizer=regularizers.l2(l2_reg), name=prefix + 'project')(x) return x
def Conv(x, config, num_filter=1, kernel_size=(1, 1), stride=(1, 1), padding="same", group=False, name=None, suffix=''): conv_name = "%s%s_conv2d" % (name, suffix) if group: x = layers.DepthwiseConv2D(kernel_size=kernel_size, strides=stride, padding=padding, use_bias=False, name=conv_name)(x) else: x = layers.Conv2D(filters=num_filter, kernel_size=kernel_size, strides=stride, padding=padding, use_bias=False, name=conv_name)(x) x = layers.BatchNormalization(momentum=config["bn_mom"], name='%s%s_batchnorm' % (name, suffix))(x) if config["net_act"] == "relu": activation = tf.nn.relu_layer elif config["net_act"] == "prelu": activation = tf.nn.leaky_relu else: activation = None print("can not get activation layer from {}".format(config["net_act"])) exit(0) x = layers.Activation(activation=activation, name='%s%s_relu' % (name, suffix))(x) return x
def block3(x, filters, kernel_size=3, stride=1, groups=32, conv_shortcut=True, name=None): """A residual block. Arguments: x: input tensor. filters: integer, filters of the bottleneck layer. kernel_size: default 3, kernel size of the bottleneck layer. stride: default 1, stride of the first layer. groups: default 32, group size for grouped convolution. conv_shortcut: default True, use convolution shortcut if True, otherwise identity shortcut. name: string, block label. Returns: Output tensor for the residual block. """ bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 if conv_shortcut: shortcut = layers.Conv2D((64 // groups) * filters, 1, strides=stride, use_bias=False, name=name + '_0_conv')(x) shortcut = tf.keras.layers.experimental.SyncBatchNormalization( axis=bn_axis, name=name + '_0_bn')(shortcut) else: shortcut = x x = layers.Conv2D(filters, 1, use_bias=False, name=name + '_1_conv')(x) x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis, name=name + '_1_bn')(x) x = layers.Activation('relu', name=name + '_1_relu')(x) c = filters // groups x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x) x = layers.DepthwiseConv2D(kernel_size, strides=stride, depth_multiplier=c, use_bias=False, name=name + '_2_conv')(x) x_shape = backend.int_shape(x)[1:-1] x = layers.Reshape(x_shape + (groups, c, c))(x) x = layers.Lambda(lambda x: sum(x[:, :, :, :, i] for i in range(c)), name=name + '_2_reduce')(x) x = layers.Reshape(x_shape + (filters, ))(x) x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis, name=name + '_2_bn')(x) x = layers.Activation('relu', name=name + '_2_relu')(x) x = layers.Conv2D((64 // groups) * filters, 1, use_bias=False, name=name + '_3_conv')(x) x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis, name=name + '_3_bn')(x) x = layers.Add(name=name + '_add')([shortcut, x]) x = layers.Activation('relu', name=name + '_out')(x) return x
def _depthwise_conv_block(inputs, pointwise_conv_filters, alpha, depth_multiplier=1, strides=(1, 1), block_id=1): """Adds a depthwise convolution block. A depthwise convolution block consists of a depthwise conv, batch normalization, relu6, pointwise convolution, batch normalization and relu6 activation. # Arguments inputs: Input tensor of shape `(rows, cols, channels)` (with `channels_last` data format) or (channels, rows, cols) (with `channels_first` data format). pointwise_conv_filters: Integer, the dimensionality of the output space (i.e. the number of output filters in the pointwise convolution). alpha: controls the width of the network. - If `alpha` < 1.0, proportionally decreases the number of filters in each layer. - If `alpha` > 1.0, proportionally increases the number of filters in each layer. - If `alpha` = 1, default number of filters from the paper are used at each layer. depth_multiplier: The number of depthwise convolution output channels for each input channel. The total number of depthwise convolution output channels will be equal to `filters_in * depth_multiplier`. strides: An integer or tuple/list of 2 integers, specifying the strides of the convolution along the width and height. Can be a single integer to specify the same value for all spatial dimensions. Specifying any stride value != 1 is incompatible with specifying any `dilation_rate` value != 1. block_id: Integer, a unique identification designating the block number. # Input shape 4D tensor with shape: `(batch, channels, rows, cols)` if data_format='channels_first' or 4D tensor with shape: `(batch, rows, cols, channels)` if data_format='channels_last'. # Output shape 4D tensor with shape: `(batch, filters, new_rows, new_cols)` if data_format='channels_first' or 4D tensor with shape: `(batch, new_rows, new_cols, filters)` if data_format='channels_last'. `rows` and `cols` values might have changed due to stride. # Returns Output tensor of block. """ # channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 pointwise_conv_filters = int(pointwise_conv_filters * alpha) if strides == (1, 1): x = inputs else: x = layers.ZeroPadding2D(((1, 1), (1, 1)), name='conv_pad_%d' % block_id)(inputs) x = layers.DepthwiseConv2D((3, 3), padding='same' if strides == (1, 1) else 'valid', depth_multiplier=depth_multiplier, strides=strides, use_bias=False, name='conv_dw_%d' % block_id)(x) x = layers.BatchNormalization(name='conv_dw_%d_bn' % block_id)(x) x = layers.ReLU(name='conv_dw_%d_relu' % block_id)(x) x = layers.Conv2D(pointwise_conv_filters, (1, 1), padding='same', use_bias=False, strides=(1, 1), name='conv_pw_%d' % block_id)(x) x = layers.BatchNormalization(name='conv_pw_%d_bn' % block_id)(x) return layers.LeakyReLU(name='conv_pw_%d_relu' % block_id)(x)
def block(inputs, activation='swish', drop_rate=0., name='', filters_in=32, filters_out=16, kernel_size=3, strides=1, expand_ratio=1, se_ratio=0., id_skip=True): """An inverted residual block. Arguments: inputs: input tensor. activation: activation function. drop_rate: float between 0 and 1, fraction of the input units to drop. name: string, block label. filters_in: integer, the number of input filters. filters_out: integer, the number of output filters. kernel_size: integer, the dimension of the convolution window. strides: integer, the stride of the convolution. expand_ratio: integer, scaling coefficient for the input filters. se_ratio: float between 0 and 1, fraction to squeeze the input filters. id_skip: boolean. Returns: output tensor for the block. """ bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 # Expansion phase filters = filters_in * expand_ratio if expand_ratio != 1: x = layers.Conv2D( filters, 1, padding='same', use_bias=False, kernel_initializer=CONV_KERNEL_INITIALIZER, name=name + 'expand_conv')( inputs) x = layers.BatchNormalization(axis=bn_axis, name=name + 'expand_bn')(x) x = layers.Activation(activation, name=name + 'expand_activation')(x) else: x = inputs # Depthwise Convolution if strides == 2: x = layers.ZeroPadding2D( padding=imagenet_utils.correct_pad(x, kernel_size), name=name + 'dwconv_pad')(x) conv_pad = 'valid' else: conv_pad = 'same' x = layers.DepthwiseConv2D( kernel_size, strides=strides, padding=conv_pad, use_bias=False, depthwise_initializer=CONV_KERNEL_INITIALIZER, name=name + 'dwconv')(x) x = layers.BatchNormalization(axis=bn_axis, name=name + 'bn')(x) x = layers.Activation(activation, name=name + 'activation')(x) # Squeeze and Excitation phase if 0 < se_ratio <= 1: filters_se = max(1, int(filters_in * se_ratio)) se = layers.GlobalAveragePooling2D(name=name + 'se_squeeze')(x) se = layers.Reshape((1, 1, filters), name=name + 'se_reshape')(se) se = layers.Conv2D( filters_se, 1, padding='same', activation=activation, kernel_initializer=CONV_KERNEL_INITIALIZER, name=name + 'se_reduce')( se) se = layers.Conv2D( filters, 1, padding='same', activation='sigmoid', kernel_initializer=CONV_KERNEL_INITIALIZER, name=name + 'se_expand')(se) x = layers.multiply([x, se], name=name + 'se_excite') # Output phase x = layers.Conv2D( filters_out, 1, padding='same', use_bias=False, kernel_initializer=CONV_KERNEL_INITIALIZER, name=name + 'project_conv')(x) x = layers.BatchNormalization(axis=bn_axis, name=name + 'project_bn')(x) if id_skip and strides == 1 and filters_in == filters_out: if drop_rate > 0: x = layers.Dropout( drop_rate, noise_shape=(None, 1, 1, 1), name=name + 'drop')(x) x = layers.add([x, inputs], name=name + 'add') return x
def conv2d(x, in_channels, out_channels, kernel_size, strides=1, padding=0, dilation=1, groups=1, use_bias=True, name="conv2d"): """ Convolution 2D layer wrapper. Parameters: ---------- x : keras.backend tensor/variable/symbol Input tensor/variable/symbol. in_channels : int Number of input channels. out_channels : int Number of output channels. kernel_size : int or tuple/list of 2 int Convolution window size. strides : int or tuple/list of 2 int Strides of the convolution. padding : int or tuple/list of 2 int Padding value for convolution layer. dilation : int or tuple/list of 2 int, default 1 Dilation value for convolution layer. groups : int, default 1 Number of groups. use_bias : bool, default False Whether the layer uses a bias vector. name : str, default 'conv2d' Layer name. Returns ------- keras.backend tensor/variable/symbol Resulted tensor/variable/symbol. """ if isinstance(kernel_size, int): kernel_size = (kernel_size, kernel_size) if isinstance(strides, int): strides = (strides, strides) if isinstance(padding, int): padding = (padding, padding) if isinstance(dilation, int): dilation = (dilation, dilation) extra_pad = False if K.backend() == "tensorflow": if (padding[0] > 0) or (padding[1] > 0): import tensorflow as tf x = nn.Lambda((lambda z: tf.pad(z, [[0, 0], [ 0, 0 ], list(padding), list(padding)])) if is_channels_first() else ( lambda z: tf.pad(z, [[0, 0], list(padding), list(padding), [0, 0]])))(x) if not ((padding[0] == padding[1]) and (kernel_size[0] == kernel_size[1]) and (kernel_size[0] // 2 == padding[0])): extra_pad = True padding_ke = "valid" else: if (padding[0] == padding[1]) and (padding[0] == 0): padding_ke = "valid" elif (padding[0] == padding[1]) and (kernel_size[0] == kernel_size[1]) and (kernel_size[0] // 2 == padding[0]): padding_ke = "same" else: x = nn.ZeroPadding2D(padding=padding, name=name + "/pad")(x) padding_ke = "valid" extra_pad = True if groups == 1: if extra_pad: name = name + "/conv" x = nn.Conv2D(filters=out_channels, kernel_size=kernel_size, strides=strides, padding=padding_ke, dilation_rate=dilation, use_bias=use_bias, name=name)(x) elif (groups == out_channels) and (out_channels == in_channels): assert (dilation[0] == 1) and (dilation[1] == 1) if extra_pad: name = name + "/conv" x = nn.DepthwiseConv2D(kernel_size=kernel_size, strides=strides, padding=padding_ke, use_bias=use_bias, name=name)(x) else: assert (in_channels % groups == 0) assert (out_channels % groups == 0) none_batch = (x.shape[0] is None) in_group_channels = in_channels // groups out_group_channels = out_channels // groups group_list = [] for gi in range(groups): xi = nn.Lambda((lambda z: z[:, gi * in_group_channels: (gi + 1) * in_group_channels, :, :] ) if is_channels_first() else ( lambda z: z[:, :, :, gi * in_group_channels: (gi + 1) * in_group_channels]))(x) xi = nn.Conv2D(filters=out_group_channels, kernel_size=kernel_size, strides=strides, padding=padding_ke, dilation_rate=dilation, use_bias=use_bias, name=name + "/convgroup{}".format(gi + 1))(xi) group_list.append(xi) x = nn.concatenate(group_list, axis=get_channel_axis(), name=name + "/concat") if none_batch and (x.shape[0] is not None): x.shape = (None, ) + x.shape[1:] return x
def mbconvblock(inputs, activations='swish', droprate=0., name='', filters_in=32, filters_out=16, kernel_size=3, strides=1, expand_ratio=1, se_ratio=0., id_skip=True, attn=True): bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 #expansion phase filters = filters_in * expand_ratio if expand_ratio != 1: x = layers.Conv2D(filters, kernel_size=1, padding='same', use_bias=False, kernel_initializer=CONV_KERNEL_INITIALIZER)(inputs) x = layers.BatchNormalization(axis=bn_axis)(x) x = layers.Activation(activations)(x) #swish activation else: x = inputs #depthwise conv if strides == 2: x = layers.ZeroPadding2D( padding=imagenet_utils.correct_pad(x, kernel_size))(x) conv_pad = 'valid' else: conv_pad = 'same' x = layers.DepthwiseConv2D( kernel_size=kernel_size, strides=strides, padding=conv_pad, use_bias=False, depthwise_initializer=CONV_KERNEL_INITIALIZER)(x) x = layers.BatchNormalization(axis=bn_axis)(x) x = layers.Activation(activations)(x) #squeeze and excitation if 0 < se_ratio <= 1: filters_se = max(1, int(filters_in * se_ratio)) se = layers.GlobalAveragePooling2D()(x) se = layers.Reshape((1, 1, filters))(se) se = layers.Conv2D(filters_se, kernel_size=1, padding='same', activation=activations, kernel_initializer=CONV_KERNEL_INITIALIZER)( se) #reduce se = layers.Conv2D(filters, kernel_size=1, padding='same', activation='sigmoid', kernel_initializer=CONV_KERNEL_INITIALIZER)( se) #expand x = layers.multiply([x, se]) if attn: avgp = layers.Lambda(lambda x: K.mean(x, axis=3, keepdims=True))(x) maxp = layers.Lambda(lambda x: K.max(x, axis=3, keepdims=True))(x) conc = layers.Concatenate(axis=3)([avgp, maxp]) attn_mask = layers.Conv2D( filters=1, kernel_size=1, activation='sigmoid', padding='same', name=name + '_Attn', use_bias=False, kernel_initializer=CONV_KERNEL_INITIALIZER)(conc) #Output phase x = layers.Conv2D(filters_out, kernel_size=1, padding='same', use_bias=False, kernel_initializer=CONV_KERNEL_INITIALIZER)(x) x = layers.BatchNormalization(axis=bn_axis)(x) if id_skip and strides == 1 and filters_in == filters_out: if droprate > 0: x = layers.Dropout(droprate, noise_shape=(None, 1, 1, 1))(x) x = layers.add([x, inputs]) if attn: x = layers.multiply([x, attn_mask]) return x
convolution_5_output = convolution_5(convolution_layer_1_output) convolution_6_output = convolution_6(convolution_layer_1_output) convolution_7_output = convolution_7(convolution_layer_1_output) convolution_8_output = convolution_8(convolution_layer_1_output) convolution_layer_2_output = concatenate([ convolution_5_output, convolution_6_output, convolution_7_output, convolution_8_output ]) reshape_1 = layers.Reshape((1, word_length, 128)) reshape_1_output = reshape_1(convolution_layer_2_output) depthwise_convolution_1 = layers.DepthwiseConv2D((1, word_length), depth_multiplier=8, activation="tanh") depthwise_convolution_1_output = depthwise_convolution_1(reshape_1_output) flatten_output_1 = flatten(depthwise_convolution_1_output) dense_1 = layers.Dense(embedding_size, activation="tanh") dense_1_output = dense_1(flatten_output_1) encoder = models.Model(inputs=[input_1], outputs=[dense_1_output]) encoder.summary() input_2 = layers.Input(shape=(embedding_size, )) dense_2 = layers.Dense(word_length * 32, activation="tanh")