def block_basic(name, x, fmaps, strides, dropout_rate=0.0): with tf.compat.v1.variable_scope('Block_{}'.format(name)): if x.shape[-1] == fmaps: r = layers.leaky_relu(batch_norm(x)) s = x if strides == 1 else tf.nn.max_pool2d( x, ksize=1, strides=2, padding='SAME') else: x = layers.leaky_relu(batch_norm(x)) with tf.compat.v1.variable_scope('Shortcut'): s = layers.conv2d(x, fmaps, kernel=1, strides=strides, regularizer_rate=regularizer_rate) with tf.compat.v1.variable_scope('Conv2D_0'): r = layers.leaky_relu( batch_norm( layers.conv2d(r if x.shape[-1] == fmaps else x, fmaps=fmaps, kernel=3, strides=strides, regularizer_rate=regularizer_rate))) if dropout_rate > 0: r = tf.cond(training, lambda: tf.nn.dropout(r, rate=dropout_rate), lambda: r, name='use_dropout') with tf.compat.v1.variable_scope('Conv2D_1'): r = layers.conv2d(r, fmaps=fmaps, kernel=3, regularizer_rate=regularizer_rate) return r + s
def block(inputs): with tf.variable_scope('MBConvBlock_{}'.format(name)): if expand_ratio != 1: with tf.variable_scope('Expand'): x = layers.conv2d(inputs, filters, kernel=1) x = layers.batch_norm(x, training) x = tf.nn.swish(x) else: x = inputs with tf.variable_scope('Depthwise'): x = layers.depthwise_conv2d(x, kernel=kernel_size, strides=strides) x = layers.batch_norm(x, training) x = tf.nn.swish(x) if has_se: x = SEBlock(name, input_filters, se_ratio, expand_ratio)(x) # output phase with tf.variable_scope('Output'): x = layers.conv2d(x, output_filters, kernel=1) x = layers.batch_norm(x, training) if id_skip: if strides == 1 and (input_filters == output_filters): x = x + inputs return x
def block_se(name, x, fmaps, se_ratio): squeezed_fmaps = max(1, int(se_ratio * fmaps)) with tf.compat.v1.variable_scope('SEBlock_{}'.format(name)): with tf.compat.v1.variable_scope('Squeeze'): s = tf.compat.v1.reduce_mean(x, axis=[1, 2], keepdims=True) s = tf.nn.swish(layers.conv2d(s, squeezed_fmaps, kernel=1)) with tf.compat.v1.variable_scope('Excite'): s = tf.nn.sigmoid(layers.conv2d(s, fmaps, kernel=1)) return s * x
def block(inputs): with tf.variable_scope('SEBlock'): x = tf.compat.v1.reduce_mean(inputs, axis=spatial_dims, keepdims=True) with tf.variable_scope('Conv2D_0'): x = layers.conv2d(x, num_reduced_filters, kernel=1) x = layers.bias(x) x = tf.nn.swish(x) # Excite with tf.variable_scope('Conv2D_1'): x = layers.conv2d(x, filters, kernel=1) x = layers.bias(x) x = tf.nn.sigmoid(x) return x * inputs
def from_rgb(x, number): with tf.compat.v1.variable_scope('Input_{}'.format(number), reuse=tf.compat.v1.AUTO_REUSE): x = layers.downscale2d(x, factor=2**(res_building - number)) x = layers.conv2d(x, fmaps=latent_size, kernel=1) x = BA(x) return x
def to_rgb(x, number): with tf.compat.v1.variable_scope('Output_{}'.format(number), reuse=tf.compat.v1.AUTO_REUSE): x = layers.upscale2d(x, factor=2**(res_building - number)) x = layers.conv2d(x, fmaps=3, kernel=1) x = layers.bias(x) return x
def conv_layer(name, x, fmaps, kernel=3, strides=1, padding='SAME'): with tf.compat.v1.variable_scope('Conv2D_{}'.format(name)): x = layers.conv2d(x, fmaps=fmaps, kernel=kernel, strides=strides, padding=padding, regularizer_rate=regularizer_rate) x = BAN(x, use_bias=False) return x
def deep_cnn_v2(inputs, training, nbof_labels, regularizer_rate=0, dropout_rate=0.): def BAN(x, use_bias=True, use_act=True, use_norm=True): if use_bias: x = layers.bias(x, regularizer_rate=regularizer_rate) if use_act: x = layers.leaky_relu(x, alpha=0.2) # if use_norm: # regularizer=tf.keras.regularizers.l2(regularizer_rate) # x = tf.layers.batch_normalization(x, training=training, momentum=0.9, gamma_regularizer=regularizer, beta_regularizer=regularizer) if use_norm: x = layers.batch_norm(x, training=training, decay=0.99, regularizer_rate=regularizer_rate) return x def conv_layer(name, x, fmaps, kernel=3, strides=1, padding='SAME'): with tf.compat.v1.variable_scope('Conv2D_{}'.format(name)): x = layers.conv2d(x, fmaps=fmaps, kernel=kernel, strides=strides, padding=padding, regularizer_rate=regularizer_rate) x = BAN(x, use_bias=False) return x def dense_layer(x, fmaps, name=0): with tf.compat.v1.variable_scope('Dense_{}'.format(name)): x = layers.dense(x, fmaps=fmaps, regularizer_rate=regularizer_rate) x = BAN(x, use_bias=False, use_act=False, use_norm=False) return x with tf.compat.v1.variable_scope('Conv2D_{}'.format('inputs')): x = layers.conv2d(inputs, fmaps=10, kernel=5, strides=1, padding='SAME', regularizer_rate=regularizer_rate) fmaps = [20, 40, 80, 160] for i in range(len(fmaps)): x = conv_layer(name=i * 2, x=x, strides=2, fmaps=fmaps[i]) # x = tf.nn.avg_pool2d(x, ksize=2, strides=1, padding='VALID') if dropout_rate > 0: x = tf.cond(training, lambda: tf.nn.dropout(x, rate=dropout_rate), lambda: x, name='use_dropout') logit = dense_layer(x, fmaps=nbof_labels, name='logit') return logit
def conv_layer(name, x, fmaps): with tf.compat.v1.variable_scope('Conv_{}'.format(name)): x = tf.compat.v1.layers.batch_normalization( tf.nn.elu( layers.conv2d(x, fmaps=fmaps, kernel=3, strides=1, regularizer_rate=regularizer_rate)), training=training, momentum=0.99, gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate), beta_regularizer=tf.keras.regularizers.l2(regularizer_rate)) return x
def conv_layer(x, number, fmaps): with tf.compat.v1.variable_scope('Conv2D_{}'.format(number)): x = layers.conv2d(x, fmaps=fmaps, kernel=3) x = BA(x) return x
def from_rgb(x, res): with tf.compat.v1.variable_scope('Input_{}'.format(res), reuse=tf.compat.v1.AUTO_REUSE): x = layers.conv2d(x, fmaps=nbof_fmaps(res), kernel=1) x = BA(x) return x
def to_rgb(x, res): with tf.compat.v1.variable_scope('Output_{}'.format(res), reuse=tf.compat.v1.AUTO_REUSE): x = layers.conv2d(x, fmaps=1, kernel=1) x = layers.bias(x) return x
def wideresnet_se(inputs, training, nbof_labels, regularizer_rate=0): def batch_norm(x): with tf.compat.v1.variable_scope('BN'): # x = layers.batch_norm(x, training=training, regularizer_rate=regularizer_rate) x = tf.compat.v1.layers.batch_normalization( x, training=training, momentum=0.99, gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate), beta_regularizer=tf.keras.regularizers.l2(regularizer_rate)) # if len(x.shape)>2: x = layers.pixel_norm(x) return x def block_se(name, x, fmaps, se_ratio): squeezed_fmaps = max(1, int(se_ratio * fmaps)) with tf.compat.v1.variable_scope('SEBlock_{}'.format(name)): with tf.compat.v1.variable_scope('Squeeze'): s = tf.compat.v1.reduce_mean(x, axis=[1, 2], keepdims=True) s = tf.nn.swish(layers.conv2d(s, squeezed_fmaps, kernel=1)) with tf.compat.v1.variable_scope('Excite'): s = tf.nn.sigmoid(layers.conv2d(s, fmaps, kernel=1)) return s * x act = layers.leaky_relu # act = tf.nn.selu def block_basic(name, x, fmaps, strides, dropout_rate=0.0): with tf.compat.v1.variable_scope('Block_{}'.format(name)): if x.shape[-1] == fmaps: r = act(batch_norm(x)) s = x if strides == 1 else tf.nn.max_pool2d( x, ksize=1, strides=2, padding='SAME') else: x = act(batch_norm(x)) with tf.compat.v1.variable_scope('Shortcut'): s = layers.conv2d(x, fmaps, kernel=1, strides=strides) # r = block_se(name, r, fmaps, 0.25) with tf.compat.v1.variable_scope('Conv2D_0'): r = act( batch_norm( layers.conv2d(r if x.shape[-1] == fmaps else x, fmaps=fmaps, kernel=3, strides=strides))) if dropout_rate > 0: r = tf.cond(training, lambda: tf.nn.dropout(r, rate=dropout_rate), lambda: r, name='use_dropout') with tf.compat.v1.variable_scope('Conv2D_1'): r = layers.conv2d(r, fmaps=fmaps, kernel=3) r = block_se(name, r, fmaps, 0.25) return r + s # Inputs with tf.compat.v1.variable_scope('Conv2D_1'): x = layers.conv2d(inputs, fmaps=16, kernel=3) # Middle layers fmaps = [160, 320, 640] nbof_unit = [4, 4, 4] # nbof_unit = [1,1,1] strides = [1, 2, 2] dropouts = [0., 0., 0.] # dropouts = [0.025,0.05,0.1] # dropouts = [0.1,0.2,0.3] # strides = [2,2,2] # nbof_unit = [2,3,4,6,3] for i in range(len(fmaps)): x = block_basic('{}_{}'.format(i, 0), x, fmaps[i], strides=strides[i], dropout_rate=dropouts[i]) for j in range(nbof_unit[i] - 1): x = block_basic('{}_{}'.format(i, j + 1), x, fmaps[i], strides=1, dropout_rate=dropouts[i]) # Output with tf.compat.v1.variable_scope('Output'): x = act(batch_norm(x)) x = layers.global_avg_pool(x) logit = layers.dense(x, fmaps=nbof_labels) return logit
def wideresnet(inputs, training, nbof_labels, regularizer_rate=0, fmaps=[160, 320, 640], nbof_unit=[4, 4, 4], strides=[1, 2, 2], dropouts=[0., 0., 0.]): def batch_norm(x): with tf.compat.v1.variable_scope('BN'): # x = layers.batch_norm(x, training=training, regularizer_rate=regularizer_rate) x = tf.compat.v1.layers.batch_normalization( x, training=training, momentum=0.99, gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate), beta_regularizer=tf.keras.regularizers.l2(regularizer_rate)) # x = tf.nn.l2_normalize(x, axis=-1) # if len(x.shape)>2: x = layers.pixel_norm(x) return x def block_basic(name, x, fmaps, strides, dropout_rate=0.0): with tf.compat.v1.variable_scope('Block_{}'.format(name)): if x.shape[-1] == fmaps: r = layers.leaky_relu(batch_norm(x)) s = x if strides == 1 else tf.nn.max_pool2d( x, ksize=1, strides=2, padding='SAME') else: x = layers.leaky_relu(batch_norm(x)) with tf.compat.v1.variable_scope('Shortcut'): s = layers.conv2d(x, fmaps, kernel=1, strides=strides, regularizer_rate=regularizer_rate) with tf.compat.v1.variable_scope('Conv2D_0'): r = layers.leaky_relu( batch_norm( layers.conv2d(r if x.shape[-1] == fmaps else x, fmaps=fmaps, kernel=3, strides=strides, regularizer_rate=regularizer_rate))) if dropout_rate > 0: r = tf.cond(training, lambda: tf.nn.dropout(r, rate=dropout_rate), lambda: r, name='use_dropout') with tf.compat.v1.variable_scope('Conv2D_1'): r = layers.conv2d(r, fmaps=fmaps, kernel=3, regularizer_rate=regularizer_rate) return r + s # Inputs with tf.compat.v1.variable_scope('Conv2D_1'): x = layers.conv2d(inputs, fmaps=fmaps[0], kernel=3, regularizer_rate=regularizer_rate) # Middle layers for i in range(len(fmaps)): x = block_basic('{}_{}'.format(i, 0), x, fmaps[i], strides=strides[i], dropout_rate=dropouts[i]) for j in range(nbof_unit[i] - 1): x = block_basic('{}_{}'.format(i, j + 1), x, fmaps[i], strides=1, dropout_rate=dropouts[i]) # Output with tf.compat.v1.variable_scope('Output'): x = layers.leaky_relu(batch_norm(x)) x = layers.global_avg_pool(x) logit = layers.dense(x, fmaps=nbof_labels, regularizer_rate=regularizer_rate) return logit
def EfficientNet( inputs, training, nbof_labels, width_coefficient=1.0, depth_coefficient=1.0, #dropout_rate=0., depth_divisor=8, min_depth=None): block_args_list = get_default_block_list() with tf.variable_scope('Input'): x = layers.conv2d(inputs, 32, kernel=1, strides=1) x = layers.batch_norm(x, training) x = tf.nn.swish(x) # Blocks part for i, block_args in enumerate(block_args_list): assert block_args.num_repeat > 0 # Update block input and output filters based on depth multiplier. block_args.input_filters = round_filters(block_args.input_filters, width_coefficient, depth_divisor, min_depth) block_args.output_filters = round_filters(block_args.output_filters, width_coefficient, depth_divisor, min_depth) block_args.num_repeat = round_repeats(block_args.num_repeat, depth_coefficient) # The first block needs to take care of stride and filter size increase. x = MBConvBlock('{}_0'.format(i), block_args.input_filters, block_args.output_filters, block_args.kernel_size, block_args.strides, block_args.expand_ratio, block_args.se_ratio, block_args.identity_skip, training)(x) if block_args.num_repeat > 1: block_args.input_filters = block_args.output_filters block_args.strides = 1 for j in range(block_args.num_repeat - 1): x = MBConvBlock('{}_{}'.format(i, j + 1), block_args.input_filters, block_args.output_filters, block_args.kernel_size, block_args.strides, block_args.expand_ratio, block_args.se_ratio, block_args.identity_skip, training)(x) # Head part with tf.variable_scope('HeadConv'): x = layers.conv2d(x, fmaps=round_filters(1280, width_coefficient, depth_coefficient, min_depth), kernel=1) x = layers.batch_norm(x, training) x = tf.nn.swish(x) with tf.variable_scope('HeadDense'): x = layers.global_avg_pool(x) logit = layers.dense(x=x, fmaps=nbof_labels) return logit
def from_rgb(x, number): with tf.compat.v1.variable_scope('Input_{}'.format(number), reuse=tf.compat.v1.AUTO_REUSE): x = layers.conv2d(x, fmaps=latent_size, kernel=1) x = BA(x) return x