def conv_battery(x, global_step, convk=3, n=32, mpk=2, mpstride=1, dropout=0.0, is_training=False, name="C", verbose=False): # BATCH NORM SETTINGS bn_offset = 0.0 bn_scale = 1.0 # DROPOUT conv_dropout = tf.cond(is_training, lambda: tf.constant(dropout), lambda: tf.constant(0.0)) # CONV STACK with tf.name_scope(name) as scope: x = conv_layer(x, k=convk, n=n, bias=None, stride=1, name="conv") x = batchnormC(x, is_training=is_training, iteration=global_step, conv=True, offset=bn_offset, scale=bn_scale) x = tf.nn.dropout(x, keep_prob=1 - conv_dropout) x = max_pool_layer(x, k=mpk, stride=mpstride, name="maxpool") x = leaky_relu(x, name="relu") print_tensor_shape(x, name=scope, verbose=verbose) return x
def fc_battery(x, global_step, n=1024, bias=0.1, is_training=False, dropout=0.0, winit=None, verbose=False, name="FC"): # BATCH NORM SETTINGS bn_offset = 0.0 bn_scale = 1.0 # DROPOUT dropout = tf.cond(is_training, lambda: tf.constant(dropout), lambda: tf.constant(0.0)) # DEFAULT WEIGHTS INITIALIZATION if winit is None: winit = he_weights_initializer() # identity_weights_initializer() # FC STACK with tf.name_scope(name) as scope: x = fc_layer(x, n=n, bias=bias, winit=winit, name="FC") x = batchnormC(x, is_training=is_training, iteration=global_step, conv=False, offset=bn_offset, scale=bn_scale) x = tf.nn.dropout(x, keep_prob=1 - dropout) x = leaky_relu(x, rate=0.01, name="relu") print_tensor_shape(x, name=scope, verbose=verbose) return x
def model_a(x, is_training, global_step, settings=None, verbose=True): """ """ # BATCH NORM SETTINGS bn_offset = 0.0 bn_scale = 1.0 # MISC SETTINGS bval = 0.01 # Bias value leak = 0.01 # leakiness of leaky relus # WEIGHTS INITIALIZERS # st_winit = zero_weights_initializer() conv_winit = he_weights_initializer() fc_winit = he_weights_initializer() # identity_weights_initializer() # DROPOUT SETTINGS conv_dropout = tf.cond(is_training, lambda: tf.constant(settings.conv_dropout), lambda: tf.constant(0.0)) fc_dropout = tf.cond(is_training, lambda: tf.constant(settings.fc_dropout), lambda: tf.constant(0.0)) # -------------------------------------------------------------------------- # TRUNK # -------------------------------------------------------------------------- # CONV LAYERS x = conv_battery(x, global_step=global_step, convk=5, n=48, mpk=2, mpstride=2, is_training=is_training, verbose=verbose) x = conv_battery(x, global_step=global_step, convk=5, n=64, mpk=2, mpstride=1, is_training=is_training, verbose=verbose) x = conv_battery(x, global_step=global_step, convk=5, n=128, mpk=2, mpstride=2, is_training=is_training, verbose=verbose) x = conv_battery(x, global_step=global_step, convk=5, n=160, mpk=2, mpstride=1, is_training=is_training, verbose=verbose) x = conv_battery(x, global_step=global_step, convk=3, n=192, mpk=2, mpstride=2, is_training=is_training, verbose=verbose) x = conv_battery(x, global_step=global_step, convk=3, n=192, mpk=2, mpstride=1, is_training=is_training, verbose=verbose) x = conv_battery(x, global_step=global_step, convk=3, n=192, mpk=2, mpstride=2, is_training=is_training, verbose=verbose) x = conv_battery(x, global_step=global_step, convk=2, n=192, mpk=2, mpstride=1, is_training=is_training, verbose=verbose) # FC LAYER x = flatten(x) print_tensor_shape(x, verbose=verbose) x = fc_battery(x, global_step=global_step, n=1024, bias=None, is_training=is_training, dropout=settings.fc_dropout, winit=fc_winit, verbose=verbose, name="FC") # -------------------------------------------------------------------------- # DIGIT BRANCHES # -------------------------------------------------------------------------- max_digits = 5 d = [None] * max_digits for i in range(max_digits): d[i] = fc_layer(x, n=11, bias=0.1, winit=fc_winit, name="branch_{}".format(i + 1)) print_tensor_shape(d[i], verbose=verbose) digits = tf.stack(d, axis=0, name="digit_logits") print_tensor_shape(digits, verbose=verbose) # -------------------------------------------------------------------------- # BBOX BRANCHES # -------------------------------------------------------------------------- bboxes = fc_layer(x, n=24, bias=0.1, winit=fc_winit, name="bbox_logits") print_tensor_shape(bboxes, verbose=verbose) return digits, bboxes
def batchnormC(x, is_training, iteration, conv=False, offset=0.0, scale=1.0): """ Given some logits `x`, apply batch normalization to them. Parameters ---------- x is_training iteration conv: (boolean)(default=False) Applying it to a convolutional layer? Returns ------- Credits ------- This code is based on code written by Martin Gorner: - https://github.com/martin-gorner/tensorflow-mnist-tutorial/blob/master/mnist_4.2_batchnorm_convolutional.py https://www.youtube.com/watch?v=vq2nnJ4g6N0 """ # adding the iteration prevents from averaging across non-existing iterations exp_moving_avg = tf.train.ExponentialMovingAverage(0.9999, iteration) bnepsilon = 1e-5 # calculate mean and variance for batch of logits if conv: mean, variance = tf.nn.moments(x, [0, 1, 2]) else: # mean and variance along the batch mean, variance = tf.nn.moments(x, [0]) update_moving_averages = exp_moving_avg.apply([mean, variance]) tf.add_to_collection("update_moving_averages", update_moving_averages) # Mean and Variance (how it get it is dependent on whether it is training) # TODO: Change the following to use the `is_trianing` directly without logical_not() # to make it more intuitive. m = tf.cond(tf.logical_not(is_training), lambda: exp_moving_avg.average(mean), lambda: mean) v = tf.cond(tf.logical_not(is_training), lambda: exp_moving_avg.average(variance), lambda: variance) # Offset param_shape = mean.get_shape().as_list() beta_init = tf.constant_initializer(offset) beta = tf.Variable(initial_value=beta_init(param_shape), name="beta") # Scale gamma_init = tf.constant_initializer(scale) gamma = tf.Variable(initial_value=gamma_init(param_shape), name="gamma") # Apply Batch Norm Ybn = tf.nn.batch_normalization(x, m, v, offset=beta, scale=gamma, variance_epsilon=bnepsilon) return Ybn