def deep_cnn_v1(x, training, nbof_labels, regularizer_rate=0, fmaps=[32, 64, 128]): def conv_layer(name, x, fmaps): with tf.compat.v1.variable_scope('Conv_{}'.format(name)): x = tf.compat.v1.layers.batch_normalization( tf.nn.elu( layers.conv2d(x, fmaps=fmaps, kernel=3, strides=1, regularizer_rate=regularizer_rate)), training=training, momentum=0.99, gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate), beta_regularizer=tf.keras.regularizers.l2(regularizer_rate)) return x x = conv_layer('Input', x, fmaps[0]) x = tf.nn.max_pool2d(x, 2, 2, 'SAME') for i in range(len(fmaps)): x = conv_layer('{}_{}'.format(i, 0), x, fmaps[i]) x = conv_layer('{}_{}'.format(i, 1), x, fmaps[i]) x = tf.nn.max_pool2d(x, 2, 2, 'SAME') x = tf.cond(training, lambda: tf.nn.dropout(x, rate=0.2 + i * 0.1), lambda: x) x = layers.global_avg_pool(x) with tf.compat.v1.variable_scope('Output'): logit = layers.dense(x, fmaps=nbof_labels) return logit
def dense_layer(x, fmaps, number): with tf.compat.v1.variable_scope('Dense_{}'.format(number)): x = layers.dense(x, fmaps=fmaps) x = layers.bias(x) if fmaps > 1: x = layers.leaky_relu(x) return x
def dense_layer(name, x, fmaps): with tf.compat.v1.variable_scope('Dense_{}'.format(name)): x = layers.dense(x, fmaps=fmaps, regularizer_rate=regularizer_rate) x = BAN(x, use_bias=False, use_act=False, use_norm=False) return x
def dense_layer(x, number): with tf.compat.v1.variable_scope('Dense_0'): x = layers.dense(x, fmaps=latent_size*16) x = tf.compat.v1.reshape(x, [-1, 4, 4, latent_size]) x = BAN(x) return x
def wideresnet_se(inputs, training, nbof_labels, regularizer_rate=0): def batch_norm(x): with tf.compat.v1.variable_scope('BN'): # x = layers.batch_norm(x, training=training, regularizer_rate=regularizer_rate) x = tf.compat.v1.layers.batch_normalization( x, training=training, momentum=0.99, gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate), beta_regularizer=tf.keras.regularizers.l2(regularizer_rate)) # if len(x.shape)>2: x = layers.pixel_norm(x) return x def block_se(name, x, fmaps, se_ratio): squeezed_fmaps = max(1, int(se_ratio * fmaps)) with tf.compat.v1.variable_scope('SEBlock_{}'.format(name)): with tf.compat.v1.variable_scope('Squeeze'): s = tf.compat.v1.reduce_mean(x, axis=[1, 2], keepdims=True) s = tf.nn.swish(layers.conv2d(s, squeezed_fmaps, kernel=1)) with tf.compat.v1.variable_scope('Excite'): s = tf.nn.sigmoid(layers.conv2d(s, fmaps, kernel=1)) return s * x act = layers.leaky_relu # act = tf.nn.selu def block_basic(name, x, fmaps, strides, dropout_rate=0.0): with tf.compat.v1.variable_scope('Block_{}'.format(name)): if x.shape[-1] == fmaps: r = act(batch_norm(x)) s = x if strides == 1 else tf.nn.max_pool2d( x, ksize=1, strides=2, padding='SAME') else: x = act(batch_norm(x)) with tf.compat.v1.variable_scope('Shortcut'): s = layers.conv2d(x, fmaps, kernel=1, strides=strides) # r = block_se(name, r, fmaps, 0.25) with tf.compat.v1.variable_scope('Conv2D_0'): r = act( batch_norm( layers.conv2d(r if x.shape[-1] == fmaps else x, fmaps=fmaps, kernel=3, strides=strides))) if dropout_rate > 0: r = tf.cond(training, lambda: tf.nn.dropout(r, rate=dropout_rate), lambda: r, name='use_dropout') with tf.compat.v1.variable_scope('Conv2D_1'): r = layers.conv2d(r, fmaps=fmaps, kernel=3) r = block_se(name, r, fmaps, 0.25) return r + s # Inputs with tf.compat.v1.variable_scope('Conv2D_1'): x = layers.conv2d(inputs, fmaps=16, kernel=3) # Middle layers fmaps = [160, 320, 640] nbof_unit = [4, 4, 4] # nbof_unit = [1,1,1] strides = [1, 2, 2] dropouts = [0., 0., 0.] # dropouts = [0.025,0.05,0.1] # dropouts = [0.1,0.2,0.3] # strides = [2,2,2] # nbof_unit = [2,3,4,6,3] for i in range(len(fmaps)): x = block_basic('{}_{}'.format(i, 0), x, fmaps[i], strides=strides[i], dropout_rate=dropouts[i]) for j in range(nbof_unit[i] - 1): x = block_basic('{}_{}'.format(i, j + 1), x, fmaps[i], strides=1, dropout_rate=dropouts[i]) # Output with tf.compat.v1.variable_scope('Output'): x = act(batch_norm(x)) x = layers.global_avg_pool(x) logit = layers.dense(x, fmaps=nbof_labels) return logit
def wideresnet(inputs, training, nbof_labels, regularizer_rate=0, fmaps=[160, 320, 640], nbof_unit=[4, 4, 4], strides=[1, 2, 2], dropouts=[0., 0., 0.]): def batch_norm(x): with tf.compat.v1.variable_scope('BN'): # x = layers.batch_norm(x, training=training, regularizer_rate=regularizer_rate) x = tf.compat.v1.layers.batch_normalization( x, training=training, momentum=0.99, gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate), beta_regularizer=tf.keras.regularizers.l2(regularizer_rate)) # x = tf.nn.l2_normalize(x, axis=-1) # if len(x.shape)>2: x = layers.pixel_norm(x) return x def block_basic(name, x, fmaps, strides, dropout_rate=0.0): with tf.compat.v1.variable_scope('Block_{}'.format(name)): if x.shape[-1] == fmaps: r = layers.leaky_relu(batch_norm(x)) s = x if strides == 1 else tf.nn.max_pool2d( x, ksize=1, strides=2, padding='SAME') else: x = layers.leaky_relu(batch_norm(x)) with tf.compat.v1.variable_scope('Shortcut'): s = layers.conv2d(x, fmaps, kernel=1, strides=strides, regularizer_rate=regularizer_rate) with tf.compat.v1.variable_scope('Conv2D_0'): r = layers.leaky_relu( batch_norm( layers.conv2d(r if x.shape[-1] == fmaps else x, fmaps=fmaps, kernel=3, strides=strides, regularizer_rate=regularizer_rate))) if dropout_rate > 0: r = tf.cond(training, lambda: tf.nn.dropout(r, rate=dropout_rate), lambda: r, name='use_dropout') with tf.compat.v1.variable_scope('Conv2D_1'): r = layers.conv2d(r, fmaps=fmaps, kernel=3, regularizer_rate=regularizer_rate) return r + s # Inputs with tf.compat.v1.variable_scope('Conv2D_1'): x = layers.conv2d(inputs, fmaps=fmaps[0], kernel=3, regularizer_rate=regularizer_rate) # Middle layers for i in range(len(fmaps)): x = block_basic('{}_{}'.format(i, 0), x, fmaps[i], strides=strides[i], dropout_rate=dropouts[i]) for j in range(nbof_unit[i] - 1): x = block_basic('{}_{}'.format(i, j + 1), x, fmaps[i], strides=1, dropout_rate=dropouts[i]) # Output with tf.compat.v1.variable_scope('Output'): x = layers.leaky_relu(batch_norm(x)) x = layers.global_avg_pool(x) logit = layers.dense(x, fmaps=nbof_labels, regularizer_rate=regularizer_rate) return logit
def EfficientNet( inputs, training, nbof_labels, width_coefficient=1.0, depth_coefficient=1.0, #dropout_rate=0., depth_divisor=8, min_depth=None): block_args_list = get_default_block_list() with tf.variable_scope('Input'): x = layers.conv2d(inputs, 32, kernel=1, strides=1) x = layers.batch_norm(x, training) x = tf.nn.swish(x) # Blocks part for i, block_args in enumerate(block_args_list): assert block_args.num_repeat > 0 # Update block input and output filters based on depth multiplier. block_args.input_filters = round_filters(block_args.input_filters, width_coefficient, depth_divisor, min_depth) block_args.output_filters = round_filters(block_args.output_filters, width_coefficient, depth_divisor, min_depth) block_args.num_repeat = round_repeats(block_args.num_repeat, depth_coefficient) # The first block needs to take care of stride and filter size increase. x = MBConvBlock('{}_0'.format(i), block_args.input_filters, block_args.output_filters, block_args.kernel_size, block_args.strides, block_args.expand_ratio, block_args.se_ratio, block_args.identity_skip, training)(x) if block_args.num_repeat > 1: block_args.input_filters = block_args.output_filters block_args.strides = 1 for j in range(block_args.num_repeat - 1): x = MBConvBlock('{}_{}'.format(i, j + 1), block_args.input_filters, block_args.output_filters, block_args.kernel_size, block_args.strides, block_args.expand_ratio, block_args.se_ratio, block_args.identity_skip, training)(x) # Head part with tf.variable_scope('HeadConv'): x = layers.conv2d(x, fmaps=round_filters(1280, width_coefficient, depth_coefficient, min_depth), kernel=1) x = layers.batch_norm(x, training) x = tf.nn.swish(x) with tf.variable_scope('HeadDense'): x = layers.global_avg_pool(x) logit = layers.dense(x=x, fmaps=nbof_labels) return logit
def dense_layer(x, fmaps, name=0, use_bias=True): with tf.compat.v1.variable_scope('Dense_{}'.format(name)): x = layers.dense(x, fmaps=fmaps, regularizer_rate=regularizer_rate) if use_bias: x = layers.bias(x, regularizer_rate=regularizer_rate) return x