def densenet_base(img_input, blocks=[6, 12, 24, 16], **kwargs): bn_axis = 3 x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)))(img_input) x = layers.Conv2D(64, 7, strides=2, use_bias=False, kernel_regularizer=regularizers.l2(l2_reg), name='conv1/conv')(x) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='conv1/bn')(x) x = layers.Activation('relu', name='conv1/relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(x) x = layers.MaxPooling2D(3, strides=2, name='pool1')(x) x = dense_block(x, blocks[0], name='conv2') _, x = transition_block(x, 0.5, name='pool2') x = dense_block(x, blocks[1], name='conv3') _, x = transition_block(x, 0.5, name='pool3') x = dense_block(x, blocks[2], name='conv4') f1, x = transition_block(x, 0.5, name='pool4') x = dense_block(x, blocks[3], name='conv5') x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='bn')(x) x = layers.Activation('relu', name='relu')(x) f2 = x return f1, f2
def small_densenet(self, img_input_shape=(64, 64, 3), blocks=[6, 12, 24, 16], weight_decay=1e-4, kernel_initializer='he_normal', init_filters=None, reduction=None, growth_rate=None, init_stride=None): img_input = layers.Input(shape=(img_input_shape)) x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)))(img_input) x = layers.Conv2D(init_filters, 3, strides=init_stride, use_bias=False, kernel_initializer=kernel_initializer, kernel_regularizer=l2(weight_decay), name='conv1/conv')(x) x = layers.BatchNormalization(axis=3, epsilon=1.001e-5, name='conv1/bn')(x) x = layers.Activation('relu', name='conv1/relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(x) x = layers.AveragePooling2D(3, strides=2, name='pool1')(x) for i, block in enumerate(blocks): scope_num_str = str(i + 2) x = self.dense_block(x, block, name='conv' + scope_num_str, growth_rate=growth_rate, weight_decay=weight_decay, kernel_initializer=kernel_initializer) if i != len(blocks) - 1: x = self.transition_block( x, reduction, name='pool' + scope_num_str, weight_decay=weight_decay, kernel_initializer=kernel_initializer) x = layers.BatchNormalization(axis=3, epsilon=1.001e-5, name='bn')(x) x = layers.Activation('relu', name='relu')(x) x = layers.GlobalAveragePooling2D(name='avg_pool')(x) x = layers.Dense(self.cat_max, activation='softmax', kernel_initializer=kernel_initializer, name='fc')(x) model = Model(img_input, x) model.compile(optimizer=Adam(lr=self.lr), loss='categorical_crossentropy', metrics=['categorical_accuracy']) return model
def layer(input_tensor): x = input_tensor residual = input_tensor # bottleneck x = layers.Conv2D(filters // 2, (1, 1), kernel_initializer='he_uniform', use_bias=False)(x) x = layers.BatchNormalization(**bn_params)(x) x = layers.Activation('relu')(x) x = layers.ZeroPadding2D(1)(x) x = GroupConv2D(filters, (3, 3), strides=strides, groups=groups, kernel_initializer='he_uniform', use_bias=False, **kwargs)(x) x = layers.BatchNormalization(**bn_params)(x) x = layers.Activation('relu')(x) x = layers.Conv2D(filters, (1, 1), kernel_initializer='he_uniform', use_bias=False)(x) x = layers.BatchNormalization(**bn_params)(x) # if number of filters or spatial dimensions changed # make same manipulations with residual connection x_channels = get_num_channels(x) r_channels = get_num_channels(residual) if strides != 1 or x_channels != r_channels: if padding: residual = layers.ZeroPadding2D(1)(residual) residual = layers.Conv2D(x_channels, downsample_kernel_size, strides=strides, kernel_initializer='he_uniform', use_bias=False)(residual) residual = layers.BatchNormalization(**bn_params)(residual) # apply attention module x = ChannelSE(reduction=reduction, **kwargs)(x) # add residual connection x = layers.Add()([x, residual]) x = layers.Activation('relu')(x) return x
def resnet50v2(num_classes, batch_size=None): """Instantiates the ResNet50V2 architecture. Args: num_classes: `int` number of classes for image classification. batch_size: Size of the batches for each step. Returns: A Keras model instance. """ input_shape = (224, 224, 3) img_input = layers.Input(shape=input_shape, batch_size=batch_size) x = img_input if backend.image_data_format() == 'channels_first': x = layers.Permute((3, 1, 2))(x) bn_axis = 1 else: # channels_last bn_axis = -1 x = layers.ZeroPadding2D(padding=3)(x) x = layers.Conv2D(filters=64, kernel_size=(7, 7), strides=(2, 2), kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name='conv1_conv')(x) x = layers.ZeroPadding2D(padding=(1, 1))(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x) x = resnet_block(x, size=3, filters=256, stage=2, stride=1) x = resnet_block(x, size=4, filters=512, stage=3) x = resnet_block(x, size=6, filters=1024, stage=4) x = resnet_block(x, size=3, filters=2048, stage=5) x = layers.BatchNormalization(axis=bn_axis, name='post_bn')(x) x = layers.Activation('relu', name='post_relu')(x) x = layers.GlobalAveragePooling2D()(x) x = layers.Dense(num_classes, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name='fc1000')(x) # A softmax that is followed by the model loss must be done cannot be done # in float16 due to numeric issues. So we pass dtype=float32. x = layers.Activation('softmax', dtype='float32')(x) # Create model. return models.Model(img_input, x, name='resnet50v2')
def block3(x, filters, kernel_size=3, stride=1, conv_shortcut=True, name=None): """A residual block. Arguments: x: input tensor. filters: integer, filters of the bottleneck layer. kernel_size: default 3, kernel size of the bottleneck layer. stride: default 1, stride of the first layer. conv_shortcut: default True, use convolution shortcut if True, otherwise identity shortcut. name: string, block label. Returns: Output tensor for the residual block. """ bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 if conv_shortcut: shortcut = layers.Conv2D( filters, 1, strides=stride, use_bias=False, name=name + '_0_conv')(x) shortcut = layers.BatchNormalization( axis=bn_axis, epsilon=1.001e-5, name=name + '_0_bn')(shortcut) else: shortcut = x x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_1_pad')(x) x = layers.Conv2D( filters, kernel_size=kernel_size, strides=stride, use_bias=False, name=name + '_1_conv')(x) x = layers.BatchNormalization( axis=bn_axis, epsilon=1.001e-5, name=name + '_1_bn')(x) x = layers.Activation('relu', name=name + '_1_relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x) x = layers.Conv2D( filters, kernel_size=kernel_size, use_bias=False, name=name + '_2_conv')(x) x = layers.BatchNormalization( axis=bn_axis, epsilon=1.001e-5, name=name + '_2_bn')(x) x = layers.Add(name=name + '_add')([shortcut, x]) x = layers.Activation('relu', name=name + '_out')(x) return x
def ResNet50(input_tensor=None, pooling=None, **kwargs): """Instantiates the ResNet50 architecture. # Arguments # Returns A Keras model instance. """ # Input arguments include_top = get_varargin(kwargs, 'include_top', True) nb_classes = get_varargin(kwargs, 'nb_classes', 1000) default_input_shape = _obtain_input_shape(None, default_size=224, min_size=197, data_format=K.image_data_format(), require_flatten=include_top) input_shape = get_varargin(kwargs, 'input_shape', default_input_shape) if input_tensor is None: img_input = KL.Input(shape=input_shape) else: if not K.is_keras_tensor(input_tensor): img_input = KL.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor bn_axis = 3 if K.image_data_format() == 'channels_last' else 1 x = KL.ZeroPadding2D((3, 3))(img_input) x = KL.Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x) x = KL.BatchNormalization(axis=bn_axis, name='bn_conv1')(x) x = KL.Activation('relu')(x) x = KL.MaxPooling2D((3, 3), strides=(2, 2))(x) for stage, nb_block in zip([2,3,4,5], [3,4,6,3]): for blk in range(nb_block): conv_block = True if blk == 0 else False strides = (2,2) if stage>2 and blk==0 else (1,1) x = identity_block(x, stage = stage, block_id = blk + 1, conv_block = conv_block, strides = strides) x = KL.AveragePooling2D((7, 7), name='avg_pool')(x) if include_top: x = KL.Flatten()(x) x = KL.Dense(nb_classes, activation='softmax', name='fc1000')(x) else: if pooling == 'avg': x = KL.GlobalAveragePooling2D()(x) elif pooling == 'max': x = KL.GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='resnet50') return model
def _inverted_res_block(inputs, expansion, stride, alpha, filters, block_id): channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 in_channels = backend.int_shape(inputs)[channel_axis] pointwise_conv_filters = int(filters * alpha) pointwise_filters = _make_divisible(pointwise_conv_filters, 8) x = inputs prefix = 'block_{}_'.format(block_id) if block_id: # Expand x = layers.Conv2D(round(expansion * in_channels), kernel_size=1, padding='same', use_bias=False, activation=None, kernel_regularizer=regularizers.l2(l2_reg), name=prefix + 'expand')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'expand_BN')(x) x = layers.ReLU(6., name=prefix + 'expand_relu')(x) else: prefix = 'expanded_conv_' # Depthwise if stride == 2: x = layers.ZeroPadding2D(padding=correct_pad(backend, x, 3), name=prefix + 'pad')(x) x = layers.DepthwiseConv2D(kernel_size=3, strides=stride, activation=None, use_bias=False, depthwise_regularizer=regularizers.l2(l2_reg), padding='same' if stride == 1 else 'valid', name=prefix + 'depthwise')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise_BN')(x) x = layers.ReLU(6., name=prefix + 'depthwise_relu')(x) # Project x = layers.Conv2D(pointwise_filters, kernel_size=1, padding='same', use_bias=False, activation=None, kernel_regularizer=regularizers.l2(l2_reg), name=prefix + 'project')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'project_BN')(x) if in_channels == pointwise_filters and stride == 1: return layers.Add(name=prefix + 'add')([inputs, x]) return x
def _depthwise_conv_block(inputs, pointwise_conv_filters, alpha, depth_multiplier=1, strides=(1, 1), block_id=1): channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 pointwise_conv_filters = int(pointwise_conv_filters * alpha) if strides == (1, 1): x = inputs else: x = layers.ZeroPadding2D(((0, 1), (0, 1)), name='conv_pad_%d' % block_id)(inputs) x = layers.DepthwiseConv2D( (3, 3), padding='same' if strides == (1, 1) else 'valid', depth_multiplier=depth_multiplier, strides=strides, use_bias=False, name='conv_dw_%d' % block_id)(x) x = layers.BatchNormalization(axis=channel_axis, name='conv_dw_%d_bn' % block_id)(x) x = layers.ReLU(6., name='conv_dw_%d_relu' % block_id)(x) x = layers.Conv2D(pointwise_conv_filters, (1, 1), padding='same', use_bias=False, strides=(1, 1), name='conv_pw_%d' % block_id)(x) x = layers.BatchNormalization(axis=channel_axis, name='conv_pw_%d_bn' % block_id)(x) return layers.ReLU(6., name='conv_pw_%d_relu' % block_id)(x)
def define_discriminator(in_shape=(28, 28, 1), nclasses=10): img = layers.Input(shape=in_shape) x = layers.Conv2D(filters=16, kernel_size=3, strides=2, padding="same")(img) x = layers.LeakyReLU(alpha=0.2)(x) x = layers.Dropout(0.25)(x) x = layers.Conv2D(filters=32, kernel_size=3, strides=2, padding="same")(x) x = layers.ZeroPadding2D(padding=((0, 1), (0, 1)))(x) x = layers.LeakyReLU(alpha=0.2)(x) x = layers.Dropout(0.25)(x) x = layers.BatchNormalization(momentum=0.8)(x) x = layers.Conv2D(filters=64, kernel_size=3, strides=2, padding="same")(x) x = layers.LeakyReLU(alpha=0.2)(x) x = layers.Dropout(0.25)(x) x = layers.BatchNormalization(momentum=0.8)(x) x = layers.Conv2D(filters=128, kernel_size=3, strides=2, padding="same")(x) x = layers.LeakyReLU(alpha=0.2)(x) x = layers.Dropout(0.25)(x) x = layers.Flatten()(x) out = layers.Dense(1, activation="sigmoid")(x) label = layers.Dense(nclasses+1, activation="softmax")(out) model = tf.keras.Model(img, [out, label]) return model
def resnet(num_blocks, img_input=None, classes=10, training=None): """Instantiates the ResNet architecture. Arguments: num_blocks: integer, the number of conv/identity blocks in each block. The ResNet contains 3 blocks with each block containing one conv block followed by (layers_per_block - 1) number of idenity blocks. Each conv/idenity block has 2 convolutional layers. With the input convolutional layer and the pooling layer towards the end, this brings the total size of the network to (6*num_blocks + 2) classes: optional number of classes to classify images into training: Only used if training keras model with Estimator. In other scenarios it is handled automatically. Returns: A Keras model instance. """ if backend.image_data_format() == 'channels_first': x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)), name='transpose')(img_input) bn_axis = 1 else: # channel_last x = img_input bn_axis = 3 x = layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(x) x = layers.Conv2D(16, (3, 3), strides=(1, 1), padding='valid', use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name='conv1')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name='bn_conv1',)(x, training=training) x = layers.Activation('relu')(x) x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[16, 16], stage=2, conv_strides=(1, 1), training=training) x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[32, 32], stage=3, conv_strides=(2, 2), training=training) x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[64, 64], stage=4, conv_strides=(2, 2), training=training) rm_axes = [1, 2] if backend.image_data_format() == 'channels_last' else [2, 3] x = layers.Lambda(lambda x: backend.mean(x, rm_axes), name='reduce_mean')(x) x = layers.Dense(classes, activation='softmax', kernel_initializer=initializers.RandomNormal(stddev=0.01), kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name='fc10')(x) inputs = img_input # Create model. model = tf.keras.models.Model(inputs, x, name='resnet56') return model
def resnet50(inputs): bn_axis = 3 # x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(inputs) x = layers.Conv2D(64, (7, 7), strides=(2, 2), padding='valid', name='conv1')(x) x = BatchNorm(axis=bn_axis, name='bn_conv1')(x) x = layers.Activation('relu')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') # x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') # x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') # x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') # model = Model(input, x, name='resnet50') return x
def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), block_id=None): """Adds 2 blocks of [relu-separable conv-batchnorm]. Arguments: ip: Input tensor filters: Number of output filters per layer kernel_size: Kernel size of separable convolutions strides: Strided convolution for downsampling block_id: String block_id Returns: A Keras tensor """ channel_dim = 1 if backend.image_data_format() == 'channels_first' else -1 with backend.name_scope('separable_conv_block_%s' % block_id): x = layers.Activation('relu')(ip) if strides == (2, 2): x = layers.ZeroPadding2D( padding=imagenet_utils.correct_pad(x, kernel_size), name='separable_conv_1_pad_%s' % block_id)(x) conv_pad = 'valid' else: conv_pad = 'same' x = layers.SeparableConv2D( filters, kernel_size, strides=strides, name='separable_conv_1_%s' % block_id, padding=conv_pad, use_bias=False, kernel_initializer='he_normal')( x) x = layers.BatchNormalization( axis=channel_dim, momentum=0.9997, epsilon=1e-3, name='separable_conv_1_bn_%s' % (block_id))( x) x = layers.Activation('relu')(x) x = layers.SeparableConv2D( filters, kernel_size, name='separable_conv_2_%s' % block_id, padding='same', use_bias=False, kernel_initializer='he_normal')( x) x = layers.BatchNormalization( axis=channel_dim, momentum=0.9997, epsilon=1e-3, name='separable_conv_2_bn_%s' % (block_id))( x) return x
def define_model(self): input_img = Input(shape=(self.model_parameters.img_height, self.model_parameters.img_width, self.model_parameters.num_channels)) x = layers.Conv2D(filters=64, kernel_size=(4, 4), strides=(2, 2), padding='same')(input_img) x = layers.LeakyReLU()(x) x = layers.Conv2D(filters=128, kernel_size=(4, 4), strides=(2, 2), padding='same')(x) x = tfa.layers.InstanceNormalization(axis=-1)(x) x = layers.LeakyReLU()(x) x = layers.Conv2D( filters=256, kernel_size=(4, 4), strides=(2, 2), padding='same', )(x) x = tfa.layers.InstanceNormalization(axis=-1)(x) x = layers.LeakyReLU()(x) x = layers.ZeroPadding2D()(x) x = layers.Conv2D(filters=512, kernel_size=(4, 4), strides=(1, 1), padding='valid')(x) x = tfa.layers.InstanceNormalization(axis=-1)(x) x = layers.LeakyReLU()(x) x = layers.ZeroPadding2D()(x) x = layers.Conv2D(filters=1, kernel_size=(4, 4), strides=(1, 1), padding='valid')(x) model = Model(name=self.model_name, inputs=input_img, outputs=x) return model
def SplitAttentionConv2D(x, filters, kernel_size, stride=1, padding=(0, 0), groups=1, use_bias=True, radix=2, name=None): bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 reduction_factor = 4 inter_filters = max(filters * radix // reduction_factor, 32) x = layers.ZeroPadding2D((padding, padding), name=name + '_splat_pad')(x) x = layers.Conv2D(filters * radix, kernel_size=kernel_size, strides=stride, groups=groups * radix, use_bias=use_bias, name=name + '_0_splat_conv')(x) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_0_splat_bn')(x) x = layers.Activation('relu', name=name + '_0_splat_relu')(x) splits = layers.Lambda(lambda x: tf.split(x, radix, bn_axis), name=name + '_0_splat_split')(x) x = layers.Add(name=name + '_0_splat_add')(splits) x = layers.GlobalAveragePooling2D(name=name + '_0_splat_pool')(x) shape = (1, 1, filters) if bn_axis == 3 else (filters, 1, 1) x = layers.Reshape(shape, name=name + '_0_splat_reshape')(x) x = layers.Conv2D(inter_filters, kernel_size=1, groups=groups, name=name + '_1_splat_conv')(x) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_1_splat_bn')(x) x = layers.Activation('relu', name=name + '_1_splat_relu')(x) # Attention x = layers.Conv2D(filters * radix, kernel_size=1, groups=groups, name=name + '_2_splat_conv')(x) x = RSoftmax(x, filters * radix, radix, groups, name=name) x = layers.Lambda(lambda x: tf.split(x, radix, bn_axis), name=name + '_1_splat_split')(x) x = layers.Lambda( lambda x: [tf.stack(x[0], axis=bn_axis), tf.stack(x[1], axis=bn_axis)], name=name + '_splat_stack')([splits, x]) x = layers.Multiply(name=name + '_splat_mult')(x) x = layers.Lambda(lambda x: tf.unstack(x, axis=bn_axis), name=name + '_splat_unstack')(x) x = layers.Add(name=name + '_splat_add')(x) return x
def block2(x, filters, kernel_size=3, stride=1, conv_shortcut=False, name=None): """A residual block. Arguments: x: input tensor. filters: integer, filters of the bottleneck layer. kernel_size: default 3, kernel size of the bottleneck layer. stride: default 1, stride of the first layer. conv_shortcut: default False, use convolution shortcut if True, otherwise identity shortcut. name: string, block label. Returns: Output tensor for the residual block. """ bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 preact = tf.keras.layers.experimental.SyncBatchNormalization( axis=bn_axis, name=name + '_preact_bn')(x) preact = layers.Activation('relu', name=name + '_preact_relu')(preact) if conv_shortcut: shortcut = layers.Conv2D(4 * filters, 1, strides=stride, name=name + '_0_conv')(preact) else: shortcut = layers.MaxPooling2D(1, strides=stride)(x) if stride > 1 else x x = layers.Conv2D(filters, 1, strides=1, use_bias=False, name=name + '_1_conv')(preact) x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis, name=name + '_1_bn')(x) x = layers.Activation('relu', name=name + '_1_relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x) x = layers.Conv2D(filters, kernel_size, strides=stride, use_bias=False, name=name + '_2_conv')(x) x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis, name=name + '_2_bn')(x) x = layers.Activation('relu', name=name + '_2_relu')(x) x = layers.Conv2D(4 * filters, 1, name=name + '_3_conv')(x) x = layers.Add(name=name + '_out')([shortcut, x]) return x
def _conv_block(inputs, filters, alpha, kernel=(3, 3), strides=(1, 1)): """Adds an initial convolution layer (with batch normalization and relu6). # Arguments inputs: Input tensor of shape `(rows, cols, 3)` (with `channels_last` data format) or (3, rows, cols) (with `channels_first` data format). It should have exactly 3 inputs channels, and width and height should be no smaller than 32. E.g. `(224, 224, 3)` would be one valid value. filters: Integer, the dimensionality of the output space (i.e. the number of output filters in the convolution). alpha: controls the width of the network. - If `alpha` < 1.0, proportionally decreases the number of filters in each layer. - If `alpha` > 1.0, proportionally increases the number of filters in each layer. - If `alpha` = 1, default number of filters from the paper are used at each layer. kernel: An integer or tuple/list of 2 integers, specifying the width and height of the 2D convolution window. Can be a single integer to specify the same value for all spatial dimensions. strides: An integer or tuple/list of 2 integers, specifying the strides of the convolution along the width and height. Can be a single integer to specify the same value for all spatial dimensions. Specifying any stride value != 1 is incompatible with specifying any `dilation_rate` value != 1. # Input shape 4D tensor with shape: `(samples, channels, rows, cols)` if data_format='channels_first' or 4D tensor with shape: `(samples, rows, cols, channels)` if data_format='channels_last'. # Output shape 4D tensor with shape: `(samples, filters, new_rows, new_cols)` if data_format='channels_first' or 4D tensor with shape: `(samples, new_rows, new_cols, filters)` if data_format='channels_last'. `rows` and `cols` values might have changed due to stride. # Returns Output tensor of block. """ channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 filters = int(filters * alpha) x = layers.ZeroPadding2D(padding=((0, 1), (0, 1)), name='conv1_pad')(inputs) x = layers.Conv2D(filters, kernel, padding='valid', use_bias=False, strides=strides, name='conv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='conv1_bn')(x) return layers.ReLU(6., name='conv1_relu')(x)
def make_discriminator_model(input_tensor=None, input_shape=(28, 28, 1)): """ Returns: tf.keras.Model """ if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor x = layers.Conv2D(16, (3, 3), strides=(2, 2), padding="same", use_bias=False, name='conv1')(img_input) x = layers.LeakyReLU(alpha=0.2, name='act1')(x) x = layers.Dropout(rate=0.25, name='drop1')(x) x = layers.Conv2D(32, (3, 3), strides=(2, 2), padding="same", use_bias=False, name='conv2')(x) x = layers.ZeroPadding2D(padding=((0, 1), (0, 1)), name='zeropadding1')(x) x = layers.LeakyReLU(alpha=0.2, name='act2')(x) x = layers.Dropout(rate=0.25, name='drop2')(x) x = layers.BatchNormalization(momentum=0.8, name='bn1')(x) x = layers.Conv2D(64, (3, 3), strides=(2, 2), padding="same", use_bias=False, name='conv3')(x) x = layers.LeakyReLU(alpha=0.2, name='act3')(x) x = layers.Dropout(rate=0.25, name='drop3')(x) x = layers.BatchNormalization(momentum=0.8, name='bn3')(x) x = layers.Conv2D(128, (3, 3), strides=(1, 1), padding="same", use_bias=False, name='conv4')(x) x = layers.LeakyReLU(alpha=0.2, name='act4')(x) x = layers.Dropout(rate=0.25, name='drop4')(x) x = layers.Flatten(name='flatten')(x) # Determine validity and label of the image validity = layers.Dense(1, activation=tf.nn.sigmoid)(x) label = layers.Dense(num_classes, activation=tf.nn.softmax)(validity) return models.Model(x, [validity, label])
def resblock_body(x, num_filters, num_blocks): '''A series of resblocks starting with a downsampling Convolution2D''' # Darknet uses left and top padding instead of 'same' mode x = kl.ZeroPadding2D(((1, 0), (1, 0)))(x) x = DarknetConv2D_BN_Leaky(num_filters, (3, 3), strides=(2, 2))(x) for i in range(num_blocks): y = compose(DarknetConv2D_BN_Leaky(num_filters // 2, (1, 1)), DarknetConv2D_BN_Leaky(num_filters, (3, 3)))(x) x = kl.Add()([x, y]) return x
def ResNet50(input_tensor): """ ResNet50 :param input_tensor: :return: """ img_input = input_tensor if backend.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(img_input) x = layers.Conv2D(64, (7, 7), strides=(2, 2), kernel_initializer='he_normal', name='conv1')(x) x = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x) x = layers.Activation('relu')(x) x = layers.ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x) x = _conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) x = _identity_block(x, 3, [64, 64, 256], stage=2, block='b') x = _identity_block(x, 3, [64, 64, 256], stage=2, block='c') x = _conv_block(x, 3, [128, 128, 512], stage=3, block='a') x = _identity_block(x, 3, [128, 128, 512], stage=3, block='b') x = _identity_block(x, 3, [128, 128, 512], stage=3, block='c') x = _identity_block(x, 3, [128, 128, 512], stage=3, block='d') x = _conv_block(x, 3, [256, 256, 1024], stage=4, block='a') x = _identity_block(x, 3, [256, 256, 1024], stage=4, block='b') x = _identity_block(x, 3, [256, 256, 1024], stage=4, block='c') x = _identity_block(x, 3, [256, 256, 1024], stage=4, block='d') x = _identity_block(x, 3, [256, 256, 1024], stage=4, block='e') x = _identity_block(x, 3, [256, 256, 1024], stage=4, block='f') x = _conv_block(x, 3, [512, 512, 2048], stage=5, block='a') x = _identity_block(x, 3, [512, 512, 2048], stage=5, block='b') x = _identity_block(x, 3, [512, 512, 2048], stage=5, block='c') return x
def DenseNet121(input_shape=None): input_shape = imagenet_utils.obtain_input_shape( input_shape, default_size=224, min_size=32, data_format=backend.image_data_format(), require_flatten=True) img_input = layers.Input(shape=input_shape) bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)))(img_input) x = layers.Conv2D(64, 7, strides=2, use_bias=False, name='conv1/conv')(x) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='conv1/bn')(x) x = layers.Activation('relu', name='conv1/relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(x) x = layers.MaxPooling2D(3, strides=2, name='pool1')(x) x = dense_block(x, 6, name='conv2') x = transition_block(x, 0.5, name='pool2') x = dense_block(x, 12, name='conv3') x = transition_block(x, 0.5, name='pool3') x = dense_block(x, 24, name='conv4') x = transition_block(x, 0.5, name='pool4') x = dense_block(x, 16, name='conv5') x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='bn')(x) x = layers.Activation('relu', name='relu')(x) x = layers.GlobalAveragePooling2D(name='avg_pool')(x) imagenet_utils.validate_activation('softmax', None) x = layers.Dense(NUM_CLASSES, activation='softmax', name='predictions')(x) # Create model. model = training.Model(img_input, x, name='densenet121') return model
def _conv_block(inputs, filters, alpha, kernel=(3, 3), strides=(1, 1)): channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 filters = int(filters * alpha) x = layers.ZeroPadding2D(padding=((0, 1), (0, 1)), name='conv1_pad')(inputs) x = layers.Conv2D(filters, kernel, padding='valid', use_bias=False, strides=strides, name='conv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='conv1_bn')(x) return layers.ReLU(6., name='conv1_relu')(x)
def _inverted_res_block(x, expansion, filters, kernel_size, stride, se_ratio, activation, block_id): channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 shortcut = x prefix = 'expanded_conv/' infilters = backend.int_shape(x)[channel_axis] if block_id: # Expand prefix = 'expanded_conv_{}/'.format(block_id) x = layers.Conv2D(_depth(infilters * expansion), kernel_size=1, padding='same', use_bias=False, name=prefix + 'expand')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'expand/BatchNorm')(x) x = activation(x) if stride == 2: x = layers.ZeroPadding2D(padding=imagenet_utils.correct_pad( x, kernel_size), name=prefix + 'depthwise/pad')(x) x = layers.DepthwiseConv2D(kernel_size, strides=stride, padding='same' if stride == 1 else 'valid', use_bias=False, name=prefix + 'depthwise')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'depthwise/BatchNorm')(x) x = activation(x) if se_ratio: x = _se_block(x, _depth(infilters * expansion), se_ratio, prefix) x = layers.Conv2D(filters, kernel_size=1, padding='same', use_bias=False, name=prefix + 'project')(x) x = layers.BatchNormalization(axis=channel_axis, epsilon=1e-3, momentum=0.999, name=prefix + 'project/BatchNorm')(x) if stride == 1 and infilters == filters: x = layers.Add(name=prefix + 'Add')([shortcut, x]) return x
def DenseNet(blocks, input_tensor=None): img_input = input_tensor bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)))(img_input) x = layers.Conv2D(64, 7, strides=2, use_bias=False, name='conv1/conv')(x) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='conv1/bn')(x) x = layers.Activation('relu', name='conv1/relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(x) x = layers.MaxPooling2D(3, strides=2, name='pool1')(x) x = _dense_block(x, blocks[0], name='conv2') x = _transition_block(x, 0.5, name='pool2') x = _dense_block(x, blocks[1], name='conv3') x = _transition_block(x, 0.5, name='pool3') x = _dense_block(x, blocks[2], name='conv4') x = _transition_block(x, 0.5, name='pool4') x = _dense_block(x, blocks[3], name='conv5') x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='bn')(x) x = layers.Activation('relu', name='relu')(x) return x
def ResNet50V2(input_shape=None): input_shape = imagenet_utils.obtain_input_shape( input_shape, default_size=224, min_size=32, data_format=backend.image_data_format(), require_flatten=True) img_input = layers.Input(shape=input_shape) bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)), name='conv1_pad')(img_input) x = layers.Conv2D(64, 7, strides=2, use_bias=True, name='conv1_conv')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name='pool1_pad')(x) x = layers.MaxPooling2D(3, strides=2, name='pool1_pool')(x) x = stack2(x, 64, 3, name='conv2') x = stack2(x, 128, 4, name='conv3') x = stack2(x, 256, 6, name='conv4') x = stack2(x, 512, 3, stride1=1, name='conv5') x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='post_bn')(x) x = layers.Activation('relu', name='post_relu')(x) x = layers.GlobalAveragePooling2D(name='avg_pool')(x) imagenet_utils.validate_activation('softmax', None) x = layers.Dense(NUM_CLASSES, activation='softmax', name='predictions')(x) # Create model. model = training.Model(img_input, x, name="resnet50v2") return model
def small_densenet(self, img_input_shape=(64, 64, 3), blocks=[6, 12, 24, 16]): img_input = Input(shape=(img_input_shape)) x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)))(img_input) x = layers.Conv2D(64, 7, strides=2, use_bias=False, name='conv1/conv')(x) x = layers.BatchNormalization(axis=3, epsilon=1.001e-5, name='conv1/bn')(x) x = layers.Activation('relu', name='conv1/relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(x) x = layers.MaxPooling2D(3, strides=2, name='pool1')(x) x = self.dense_block(x, blocks[0], name='conv2') x = self.transition_block(x, 0.5, name='pool2') x = self.dense_block(x, blocks[1], name='conv3') x = self.transition_block(x, 0.5, name='pool3') x = self.dense_block(x, blocks[2], name='conv4') x = self.transition_block(x, 0.5, name='pool4') x = self.dense_block(x, blocks[3], name='conv5') x = layers.BatchNormalization(axis=3, epsilon=1.001e-5, name='bn')(x) x = layers.Activation('relu', name='relu')(x) x = layers.GlobalAveragePooling2D(name='avg_pool')(x) # x = Lambda(lambda x: x, name = 'densenet_features')(x) x = layers.Dense(self.cat_max, activation='softmax', name='fc')(x) model = Model(img_input, x) # print (model.summary()) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['categorical_accuracy']) return model
def block2(x, filters, kernel_size=3, stride=1, conv_shortcut=False, name=None): bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 preact = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_preact_bn')(x) preact = layers.Activation('relu', name=name + '_preact_relu')(preact) if conv_shortcut: shortcut = layers.Conv2D(4 * filters, 1, strides=stride, name=name + '_0_conv')(preact) else: shortcut = layers.MaxPooling2D(1, strides=stride)(x) if stride > 1 else x x = layers.Conv2D(filters, 1, strides=1, use_bias=False, name=name + '_1_conv')(preact) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_1_bn')(x) x = layers.Activation('relu', name=name + '_1_relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x) x = layers.Conv2D(filters, kernel_size, strides=stride, use_bias=False, name=name + '_2_conv')(x) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_2_bn')(x) x = layers.Activation('relu', name=name + '_2_relu')(x) x = layers.Conv2D(4 * filters, 1, name=name + '_3_conv')(x) x = layers.Add(name=name + '_out')([shortcut, x]) return x
def effi_init_block(x, in_channels, out_channels, bn_epsilon, activation, tf_mode, name="effi_init_block"): """ EfficientNet specific initial block. Parameters: ---------- x : keras.backend tensor/variable/symbol Input tensor/variable/symbol. in_channels : int Number of input channels. out_channels : int Number of output channels. bn_epsilon : float Small float added to variance in Batch norm. activation : str Name of activation function. tf_mode : bool Whether to use TF-like mode. name : str, default 'effi_init_block' Block name. Returns ------- keras.backend tensor/variable/symbol Resulted tensor/variable/symbol. """ if tf_mode: x = nn.ZeroPadding2D(padding=calc_tf_padding(x, kernel_size=3, strides=2), name=name + "/conv_pad")(x) x = conv3x3_block(x=x, in_channels=in_channels, out_channels=out_channels, strides=1, padding=(0 if tf_mode else 1), bn_epsilon=bn_epsilon, activation=activation, name=name + "/conv") return x
def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), block_id=None): channel_dim = -1 with backend.name_scope('separable_conv_block_%s' % block_id): x = layers.Activation('relu')(ip) if strides == (2, 2): x = layers.ZeroPadding2D( padding=imagenet_utils.correct_pad(x, kernel_size), name='separable_conv_1_pad_%s' % block_id)(x) conv_pad = 'valid' else: conv_pad = 'same' x = layers.SeparableConv2D(filters, kernel_size, strides=strides, name='separable_conv_1_%s' % block_id, padding=conv_pad, use_bias=False, kernel_initializer='he_normal')(x) x = layers.BatchNormalization(axis=channel_dim, momentum=0.9997, epsilon=1e-3, name='separable_conv_1_bn_%s' % (block_id))(x) x = layers.Activation('relu')(x) x = layers.SeparableConv2D(filters, kernel_size, name='separable_conv_2_%s' % block_id, padding='same', use_bias=False, kernel_initializer='he_normal')(x) x = layers.BatchNormalization(axis=channel_dim, momentum=0.9997, epsilon=1e-3, name='separable_conv_2_bn_%s' % (block_id))(x) return x
def ResNet(stack_fn, preact, use_bias, model_name='resnet', include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000, classifier_activation='softmax', **kwargs): """Instantiates the ResNet, ResNetV2, and ResNeXt architecture. Optionally loads weights pre-trained on ImageNet. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. Caution: Be sure to properly pre-process your inputs to the application. Please see `applications.resnet.preprocess_input` for an example. Arguments: stack_fn: a function that returns output tensor for the stacked residual blocks. preact: whether to use pre-activation or not (True for ResNetV2, False for ResNet and ResNeXt). use_bias: whether to use biases for convolutional layers or not (True for ResNet and ResNetV2, False for ResNeXt). model_name: string, model name. include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization), 'imagenet' (pre-training on ImageNet), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or `(3, 224, 224)` (with `channels_first` data format). It should have exactly 3 inputs channels. pooling: optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. classifier_activation: A `str` or callable. The activation function to use on the "top" layer. Ignored unless `include_top=True`. Set `classifier_activation=None` to return the logits of the "top" layer. **kwargs: For backwards compatibility only. Returns: A `keras.Model` instance. Raises: ValueError: in case of invalid argument for `weights`, or invalid input shape. ValueError: if `classifier_activation` is not `softmax` or `None` when using a pretrained top layer. """ if 'layers' in kwargs: global layers layers = kwargs.pop('layers') if kwargs: raise ValueError('Unknown argument(s): %s' % (kwargs, )) if not (weights in {'imagenet', None} or os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `imagenet` ' '(pre-training on ImageNet), ' 'or the path to the weights file to be loaded.') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError( 'If using `weights` as `"imagenet"` with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape input_shape = imagenet_utils.obtain_input_shape( input_shape, default_size=224, min_size=32, data_format=backend.image_data_format(), require_flatten=include_top, weights=weights) if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)), name='conv1_pad')(img_input) x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv')(x) if not preact: x = tf.keras.layers.experimental.SyncBatchNormalization( axis=bn_axis, name='conv1_bn')(x) x = layers.Activation('relu', name='conv1_relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name='pool1_pad')(x) x = layers.MaxPooling2D(3, strides=2, name='pool1_pool')(x) x = stack_fn(x) if preact: x = tf.keras.layers.experimental.SyncBatchNormalization( axis=bn_axis, name='post_bn')(x) x = layers.Activation('relu', name='post_relu')(x) if include_top: x = layers.GlobalAveragePooling2D(name='avg_pool')(x) imagenet_utils.validate_activation(classifier_activation, weights) x = layers.Dense(classes, activation=classifier_activation, name='predictions')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling2D(name='avg_pool')(x) elif pooling == 'max': x = layers.GlobalMaxPooling2D(name='max_pool')(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = layer_utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = training.Model(inputs, x, name=model_name) # Load weights. if (weights == 'imagenet') and (model_name in WEIGHTS_HASHES): if include_top: file_name = model_name + '_weights_tf_dim_ordering_tf_kernels.h5' file_hash = WEIGHTS_HASHES[model_name][0] else: file_name = model_name + '_weights_tf_dim_ordering_tf_kernels_notop.h5' file_hash = WEIGHTS_HASHES[model_name][1] weights_path = data_utils.get_file(file_name, BASE_WEIGHTS_PATH + file_name, cache_subdir='models', file_hash=file_hash) model.load_weights(weights_path) elif weights is not None: model.load_weights(weights) return model
def block3(x, filters, kernel_size=3, stride=1, groups=32, conv_shortcut=True, name=None): """A residual block. Arguments: x: input tensor. filters: integer, filters of the bottleneck layer. kernel_size: default 3, kernel size of the bottleneck layer. stride: default 1, stride of the first layer. groups: default 32, group size for grouped convolution. conv_shortcut: default True, use convolution shortcut if True, otherwise identity shortcut. name: string, block label. Returns: Output tensor for the residual block. """ bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 if conv_shortcut: shortcut = layers.Conv2D((64 // groups) * filters, 1, strides=stride, use_bias=False, name=name + '_0_conv')(x) shortcut = tf.keras.layers.experimental.SyncBatchNormalization( axis=bn_axis, name=name + '_0_bn')(shortcut) else: shortcut = x x = layers.Conv2D(filters, 1, use_bias=False, name=name + '_1_conv')(x) x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis, name=name + '_1_bn')(x) x = layers.Activation('relu', name=name + '_1_relu')(x) c = filters // groups x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x) x = layers.DepthwiseConv2D(kernel_size, strides=stride, depth_multiplier=c, use_bias=False, name=name + '_2_conv')(x) x_shape = backend.int_shape(x)[1:-1] x = layers.Reshape(x_shape + (groups, c, c))(x) x = layers.Lambda(lambda x: sum(x[:, :, :, :, i] for i in range(c)), name=name + '_2_reduce')(x) x = layers.Reshape(x_shape + (filters, ))(x) x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis, name=name + '_2_bn')(x) x = layers.Activation('relu', name=name + '_2_relu')(x) x = layers.Conv2D((64 // groups) * filters, 1, use_bias=False, name=name + '_3_conv')(x) x = tf.keras.layers.experimental.SyncBatchNormalization(axis=bn_axis, name=name + '_3_bn')(x) x = layers.Add(name=name + '_add')([shortcut, x]) x = layers.Activation('relu', name=name + '_out')(x) return x