Beispiel #1
0
def generator2(s, training=True, weight_decay=0.0001, batch_norm_decay=0.997,
        batch_norm_epsilon=1e-5, batch_norm_scale=True):
    batch_norm_params = {
        'is_training': training,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'updates_collections': ops.GraphKeys.UPDATE_OPS,
    }

    with arg_scope(
        [slim.conv2d],
        weights_regularizer=regularizers.l2_regularizer(weight_decay),
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=lrelu,
        normalizer_fn=layers.batch_norm,
        normalizer_params=batch_norm_params):

        s = slim.stack(s, slim.conv2d, [(64, [4,4], [2,2]), (128, [4,4], [2,2])], scope="convolution")

    # Res
    with arg_scope(
        [slim.conv2d],
        weights_regularizer=regularizers.l2_regularizer(weight_decay),
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=layers.batch_norm,
        normalizer_params=batch_norm_params):

        res = slim.conv2d(s, 256, [3,3], [1,1], scope="res1_1")
        res = slim.conv2d(res, 256, [3,3], [1,1], scope="res1_2")
        gen = s + res

        res = slim.conv2d(gen, 256, [3,3], [1,1], scope="res2_1")
        res = slim.conv2d(res, 256, [3,3], [1,1], scope="res2_2")
        gen = gen + res

    with arg_scope(
        [slim.conv2d],
        weights_regularizer=regularizers.l2_regularizer(weight_decay),
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=layers.batch_norm,
        normalizer_params=batch_norm_params):

        gen = tf.image.resize_nearest_neighbor(gen, [64,64])
        gen = slim.conv2d(gen, 1024, [3,3], [1,1], scope="convolution_2")

        gen = tf.image.resize_nearest_neighbor(gen, [128,128])
        gen = slim.conv2d(gen, 512, [3,3], [1,1], scope="convolution_2_1")

        # l = [(4096, [3,3], [2,2]), (2048, [3,3], [2,2]), (2048, [3,3], [2,2]),
        #     (2048, [3,3], [2,2]), (1024, [3,3], [2,2]), (512, [3,3], [2,2]), (256, [3,3], [2,2]), (128, [3,3], [1,1]), (3, [3,3], [1,1])]


    gen = slim.conv2d(gen, 3, [3,3], [1,1], weights_regularizer=regularizers.l2_regularizer(weight_decay),
    weights_initializer=initializers.variance_scaling_initializer(), activation_fn=tf.tanh, scope="convolutionend")


    return gen
 def test_wrong_dtype(self):
   with self.assertRaisesRegexp(
       TypeError, 'Cannot create initializer for non-floating point type.'):
     initializers.variance_scaling_initializer(dtype=dtypes.int32)
   initializer = initializers.variance_scaling_initializer()
   with self.assertRaisesRegexp(
       TypeError, 'Cannot create initializer for non-floating point type.'):
     initializer([], dtype=dtypes.int32)
 def test_wrong_dtype(self):
     with self.assertRaisesRegexp(
             TypeError,
             'Cannot create initializer for non-floating point type.'):
         initializers.variance_scaling_initializer(dtype=dtypes.int32)
     initializer = initializers.variance_scaling_initializer()
     with self.assertRaisesRegexp(
             TypeError,
             'Cannot create initializer for non-floating point type.'):
         initializer([], dtype=dtypes.int32)
def generator1(z,
               training=True,
               weight_decay=0.0001,
               batch_norm_decay=0.997,
               batch_norm_epsilon=1e-5,
               batch_norm_scale=True):
    batch_norm_params = {
        'is_training': training,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'updates_collections': ops.GraphKeys.UPDATE_OPS,
    }

    c0 = tf.reshape(z, [batch_size, 1, 1, z_dim])

    with arg_scope(
        [slim.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            activation_fn=nn_ops.relu,
            normalizer_fn=layers.batch_norm,
            normalizer_params=batch_norm_params):

        gen = tf.image.resize_nearest_neighbor(c0, [2, 2])
        gen = slim.conv2d(gen, 1024, [3, 3], [1, 1], scope="convolution1")

        gen = tf.image.resize_nearest_neighbor(gen, [4, 4])
        gen = slim.conv2d(gen, 512, [3, 3], [1, 1], scope="convolution2")

        gen = tf.image.resize_nearest_neighbor(gen, [8, 8])
        gen = slim.conv2d(gen, 256, [3, 3], [1, 1], scope="convolution3")

        gen = tf.image.resize_nearest_neighbor(gen, [16, 16])
        gen = slim.conv2d(gen, 128, [3, 3], [1, 1], scope="convolution4")

        gen = tf.image.resize_nearest_neighbor(gen, [32, 32])
        gen = slim.conv2d(gen, 64, [3, 3], [1, 1], scope="convolution5")

        gen = tf.image.resize_nearest_neighbor(gen, [64, 64])

        # l = [(4096, [3,3], [2,2]), (2048, [3,3], [2,2]), (2048, [3,3], [2,2]),
        #     (2048, [3,3], [2,2]), (1024, [3,3], [2,2]), (512, [3,3], [2,2]), (256, [3,3], [2,2]), (128, [3,3], [1,1]), (3, [3,3], [1,1])]

    gen = slim.conv2d(
        gen,
        3, [3, 3], [1, 1],
        weights_regularizer=regularizers.l2_regularizer(weight_decay),
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=tf.tanh,
        scope="convolutionend")

    return gen
def discriminator2(img,
                   training=True,
                   weight_decay=0.0001,
                   batch_norm_decay=0.997,
                   batch_norm_epsilon=1e-5,
                   batch_norm_scale=True):
    batch_norm_params = {
        'is_training': training,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'updates_collections': ops.GraphKeys.UPDATE_OPS,
    }

    disc = slim.conv2d(
        img,
        64, [4, 4], [2, 2],
        weights_regularizer=regularizers.l2_regularizer(weight_decay),
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=lrelu,
        scope="convolutionstart")
    with arg_scope(
        [slim.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            activation_fn=lrelu,
            normalizer_fn=layers.batch_norm,
            normalizer_params=batch_norm_params):

        disc = slim.stack(disc,
                          slim.conv2d, [(128, [4, 4], [2, 2]),
                                        (256, [4, 4], [2, 2]),
                                        (512, [4, 4], [2, 2]),
                                        (1024, [4, 4], [2, 2])],
                          scope="convolution")

    disc = slim.conv2d(
        disc,
        1024, [1, 1],
        weights_regularizer=regularizers.l2_regularizer(weight_decay),
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=lrelu,
        scope="convolutionend")

    disc = tf.reshape(disc, [batch_size, 4 * 4 * 1024])
    disc = slim.fully_connected(disc, 1, activation_fn=None, scope="logits")

    return disc
Beispiel #6
0
def resnet_arg_scope(bn_is_training,
                     bn_trainable,
                     trainable=True,
                     weight_decay=config.TRAIN.weight_decay_factor,
                     batch_norm_decay=0.99,
                     batch_norm_epsilon=1e-9,
                     batch_norm_scale=True,
                     data_format='NHWC'):
    batch_norm_params = {
        'is_training': bn_is_training,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'trainable': bn_trainable,
        'updates_collections': ops.GraphKeys.UPDATE_OPS,
        'fused': True
    }

    with arg_scope(
        [slim.conv2d, slim.separable_conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            trainable=trainable,
            activation_fn=nn_ops.relu,
            normalizer_fn=slim.batch_norm
            if 'BN' in config.TRAIN.norm else GroupNorm,
            normalizer_params=batch_norm_params
            if 'BN' in config.TRAIN.norm else None,
            data_format=data_format):
        with arg_scope([layers.batch_norm, layers.max_pool2d],
                       data_format=data_format):
            with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:

                return arg_sc
Beispiel #7
0
def resnet_arg_scope(is_training=True,
                     weight_decay=cfg.TRAIN.WEIGHT_DECAY,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
    batch_norm_params = {
        # NOTE 'is_training' here does not work because inside resnet it gets reset:
        # https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187
        'is_training': False,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'trainable': cfg.RESNET.BN_TRAIN,
        'updates_collections': ops.GraphKeys.UPDATE_OPS,
        'fused': True
    }

    with arg_scope(
        [slim.conv2d, slim.separable_conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            trainable=is_training,
            activation_fn=nn_ops.relu,
            #       normalizer_fn=None,
            #       normalizer_params=None):
            normalizer_fn=layers.batch_norm,
            normalizer_params=batch_norm_params):
        with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
            return arg_sc
Beispiel #8
0
def resnet_arg_scope_gn(weight_decay=0.0001,
                        group_norm_num=32,
                        group_norm_dim=None):
    """Defines the default ResNet arg scope.
    Args:
      weight_decay: The weight decay to use for regularizing the model.
    Returns:
      An `arg_scope` to use for the resnet models.
    """
    with arg_scope(
        [conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            activation_fn=nn_ops.relu,
            normalizer_fn=group_norm2d):
        with arg_scope(
            [sep_conv2d],
                normalizer_fn=group_norm2d,
                activation_fn_middle=nn_ops.relu,
                pointwise_weights_regularizer=regularizers.l2_regularizer(
                    weight_decay)):
            with arg_scope([group_norm2d],
                           group_num=group_norm_num,
                           group_size=group_norm_dim,
                           eps=1e-05,
                           affine=True):
                with arg_scope([max_pool2d], padding='SAME') as arg_sc:
                    return arg_sc
Beispiel #9
0
def resnet_arg_scope(is_training=True,
                     weight_decay=0.0001,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
  batch_norm_params = {
      'is_training': False,
      'decay': batch_norm_decay,
      'epsilon': batch_norm_epsilon,
      'scale': batch_norm_scale,
      'trainable': cfg.RESNET.BN_TRAIN,
      'updates_collections': ops.GraphKeys.UPDATE_OPS
  }

  with arg_scope(
      [slim.conv2d],
      weights_regularizer=regularizers.l2_regularizer(weight_decay),
      weights_initializer=initializers.variance_scaling_initializer(),
      trainable=is_training,
      activation_fn=nn_ops.relu,
      normalizer_fn=layers.batch_norm,
      normalizer_params=batch_norm_params):
    with arg_scope([layers.batch_norm], **batch_norm_params):
      # The following implies padding='SAME' for pool1, which makes feature
      # alignment easier for dense prediction tasks. This is also used in
      # https://github.com/facebook/fb.resnet.torch. However the accompanying
      # code of 'Deep Residual Learning for Image Recognition' uses
      # padding='VALID' for pool1. You can switch to that choice by setting
      # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID').
      with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
        return arg_sc
Beispiel #10
0
  def linear_mapping_weightnorm(self, inputs, out_dim,
                                var_scope_name="linear",
                                output_collection=None):
    with tf.variable_scope(var_scope_name):
      # pylint: disable=invalid-name
      input_shape = inputs.get_shape().as_list()  # static shape. may has None
      # use weight normalization (Salimans & Kingma, 2016)  w = g* v/2-norm(v)
      V = tf.get_variable(
          name='V',
          shape=[int(input_shape[-1]), out_dim],
          dtype=tf.float32,
          initializer=initializers.variance_scaling_initializer())
      # V shape is M*N,  V_norm shape is N
      V_norm = tf.norm(V.initialized_value(), axis=0)
      g = tf.get_variable('g', dtype=tf.float32, initializer=V_norm)
      # weightnorm bias is init zero
      b = tf.get_variable(
          name='b',
          shape=[out_dim],
          dtype=tf.float32,
          initializer=tf.zeros_initializer())

      assert len(input_shape) == 3
      inputs = tf.reshape(inputs, [-1, input_shape[-1]])
      inputs = tf.matmul(inputs, V)
      inputs = tf.reshape(inputs, [input_shape[0], -1, out_dim])

      # g/2-norm(v)
      scaler = tf.div(g, tf.norm(V, axis=0))
      # x*v g/2-norm(v) + b
      inputs = tf.reshape(scaler, [1, out_dim]) * inputs + tf.reshape(b, [1, out_dim])

      if self.is_training:
        tf.add_to_collection(output_collection, inputs)
      return inputs
Beispiel #11
0
def fc2_res(phi, name='fc2_res'):
    """
    Converts pretrained (fixed) resnet features phi into movie strip.

    This applies 2 fc then add it to the orig as residuals.

    Args:
        phi (B x T x 2048): Image feature.
        name (str): Scope.

    Returns:
        Phi (B x T x 2048): Hallucinated movie strip.
    """
    with tf.variable_scope(name, reuse=False):
        net = slim.fully_connected(phi, 2048, scope='fc1')
        net = slim.fully_connected(net, 2048, scope='fc2')
        small_xavier = variance_scaling_initializer(factor=.001,
                                                    mode='FAN_AVG',
                                                    uniform=True)
        net_final = slim.fully_connected(net,
                                         2048,
                                         activation_fn=None,
                                         weights_initializer=small_xavier,
                                         scope='fc3')
        new_phi = net_final + phi
    return new_phi
Beispiel #12
0
def resnet_arg_scope(bn_is_training,
                     bn_trainable,
                     trainable=True,
                     weight_decay=cfg.weight_decay,
                     batch_norm_decay=0.99,
                     batch_norm_epsilon=1e-9,
                     batch_norm_scale=True):
    batch_norm_params = {
        'is_training': bn_is_training,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'trainable': bn_trainable,
        'updates_collections': ops.GraphKeys.UPDATE_OPS
    }

    with arg_scope(
        [slim.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            trainable=trainable,
            activation_fn=nn_ops.relu,
            normalizer_fn=layers.batch_norm,
            normalizer_params=batch_norm_params):
        with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
            return arg_sc
Beispiel #13
0
def Encoder_fc3_dropout(x,
                        num_output=85,
                        is_training=True,
                        reuse=False,
                        name="3D_module"):
    """
    3D inference module. 3 MLP layers (last is the output)
    With dropout  on first 2.
    Input:
    - x: N x [|img_feat|, |3D_param|]
    - reuse: bool

    Outputs:
    - 3D params: N x num_output
      if orthogonal: 
           either 85: (3 + 24*3 + 10) or 109 (3 + 24*4 + 10) for factored axis-angle representation
      if perspective:
          86: (f, tx, ty, tz) + 24*3 + 10, or 110 for factored axis-angle.
    - variables: tf variables
    """
    if reuse:
        print('Reuse is on!')
    with tf.compat.v1.variable_scope(name, reuse=reuse) as scope:
        net = tf.contrib.layers.fully_connected(x, 1024, scope='fc1')
        net = tf.contrib.layers.dropout(net, 0.5, is_training=is_training, scope='dropout1')
        net = tf.contrib.layers.fully_connected(net, 1024, scope='fc2')
        net = tf.contrib.layers.dropout(net, 0.5, is_training=is_training, scope='dropout2')
        small_xavier = variance_scaling_initializer(
            factor=.01, mode='FAN_AVG', uniform=True)
        net = tf.contrib.layers.fully_connected(net, num_output,activation_fn=None,
                                                weights_initializer=small_xavier, scope='fc3')

    variables = tf.contrib.framework.get_variables(scope)
    return net, variables
Beispiel #14
0
def resnet_arg_scope(is_training=True,
                     weight_decay=cfg.TRAIN.WEIGHT_DECAY,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
    data_format = 'NCHW' if cfg.RESNET.USE_NCHW else 'NHWC'
    batch_norm_params = {
        # NOTE 'is_training' is set appropriately inside of the resnet if we pass it to it:
        # https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187
        # 'is_training': False,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'trainable': True,
        'data_format': data_format,
        'fused': True
    }

    with arg_scope(
        [slim.conv2d, slim.conv2d_transpose],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            trainable=is_training,
            activation_fn=nn_ops.relu,
            normalizer_fn=layers.batch_norm,
            data_format=data_format,
            normalizer_params=batch_norm_params):
        with arg_scope([slim.max_pool2d, resnet_utils.conv2d_same],
                       data_format=data_format):
            with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
                return arg_sc
Beispiel #15
0
    def conv1d_weightnorm(self,
                          inputs,
                          out_dim,
                          kernel_size,
                          padding="SAME",
                          var_scope_name="conv1d",
                          output_collection=None):
        with tf.variable_scope(var_scope_name):
            # pylint: disable=invalid-name
            in_dim = int(inputs.get_shape()[-1])
            V = tf.get_variable(
                name='V',
                shape=[kernel_size, in_dim, out_dim],
                dtype=tf.float32,
                initializer=initializers.variance_scaling_initializer())
            # V shape is M*N*k,  V_norm shape is k
            V_norm = tf.norm(V.initialized_value(), axis=[0, 1])
            g = tf.get_variable('g', dtype=tf.float32, initializer=V_norm)
            b = tf.get_variable(name='b',
                                shape=[out_dim],
                                dtype=tf.float32,
                                initializer=tf.zeros_initializer())

            # use weight normalization (Salimans & Kingma, 2016)
            W = tf.reshape(g, [1, 1, out_dim]) * tf.nn.l2_normalize(V, [0, 1])
            inputs = tf.nn.conv1d(value=inputs,
                                  filters=W,
                                  stride=1,
                                  padding=padding)
            inputs = tf.nn.bias_add(inputs, b)

            if self.is_training:
                tf.add_to_collection(output_collection, inputs)
            return inputs
Beispiel #16
0
def resnet_arg_scope(is_training=True,
                     weight_decay=cfg.TRAIN.WEIGHT_DECAY,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
  batch_norm_params = {
    # NOTE 'is_training' here does not work because inside resnet it gets reset:
    # https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187
    'is_training': False,
    'decay': batch_norm_decay,
    'epsilon': batch_norm_epsilon,
    'scale': batch_norm_scale,
    'trainable': cfg.RESNET.BN_TRAIN,
    'updates_collections': ops.GraphKeys.UPDATE_OPS
  }

  with arg_scope(
      [slim.conv2d],
      weights_regularizer=regularizers.l2_regularizer(weight_decay),
      weights_initializer=initializers.variance_scaling_initializer(),
      trainable=is_training,
      activation_fn=nn_ops.relu,
      normalizer_fn=layers.batch_norm,
      normalizer_params=batch_norm_params):
    with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
      return arg_sc
Beispiel #17
0
def variance_scaling(factor=2.0, mode='FAN_IN', uniform=False, seed=None,
                     dtype=tf.float32):
    """ Variance Scaling.

    Returns an initializer that generates tensors without scaling variance.

    When initializing a deep network, it is in principle advantageous to keep
    the scale of the input variance constant, so it does not explode or diminish
    by reaching the final layer. This initializer use the following formula:

    ```
    if mode='FAN_IN': # Count only number of input connections.
      n = fan_in
    elif mode='FAN_OUT': # Count only number of output connections.
      n = fan_out
    elif mode='FAN_AVG': # Average number of inputs and output connections.
      n = (fan_in + fan_out)/2.0

      truncated_normal(shape, 0.0, stddev=sqrt(factor / n))
    ```

    To get http://arxiv.org/pdf/1502.01852v1.pdf use (Default):
    - factor=2.0 mode='FAN_IN' uniform=False

    To get http://arxiv.org/abs/1408.5093 use:
    - factor=1.0 mode='FAN_IN' uniform=True

    To get http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf use:
    - factor=1.0 mode='FAN_AVG' uniform=True.

    To get xavier_initializer use either:
    - factor=1.0 mode='FAN_AVG' uniform=True.
    - factor=1.0 mode='FAN_AVG' uniform=False.

    Arguments:
        factor: Float.  A multiplicative factor.
        mode: String.  'FAN_IN', 'FAN_OUT', 'FAN_AVG'.
        uniform: Whether to use uniform or normal distributed random
            initialization.
        seed: A Python integer. Used to create random seeds. See
            `set_random_seed` for behavior.
        dtype: The data type. Only floating point types are supported.

    Returns:
        An initializer that generates tensors with unit variance.

    Raises:
        ValueError: if `dtype` is not a floating point type.
        TypeError: if `mode` is not in ['FAN_IN', 'FAN_OUT', 'FAN_AVG'].
    """
    try:
        from tensorflow.contrib.layers.python.layers.initializers import \
            variance_scaling_initializer
    except ImportError:
        raise NotImplementedError("'variance_scaling_initializer' not "
                                  "supported, please update TensorFlow.")
    return variance_scaling_initializer(factor=factor, mode=mode,
                                        uniform=uniform, seed=seed,
                                        dtype=dtype)
Beispiel #18
0
def variance_scaling(factor=2.0,
                     mode='FAN_IN',
                     uniform=False,
                     seed=None,
                     dtype=tf.float32):
    """ Variance Scaling.

    Returns an initializer that generates tensors without scaling variance.

    When initializing a deep network, it is in principle advantageous to keep
    the scale of the input variance constant, so it does not explode or diminish
    by reaching the final layer. This initializer use the following formula:

    ```
    if mode='FAN_IN': # Count only number of input connections.
      n = fan_in
    elif mode='FAN_OUT': # Count only number of output connections.
      n = fan_out
    elif mode='FAN_AVG': # Average number of inputs and output connections.
      n = (fan_in + fan_out)/2.0

      truncated_normal(shape, 0.0, stddev=sqrt(factor / n))
    ```

    To get http://arxiv.org/pdf/1502.01852v1.pdf use (Default):
    - factor=2.0 mode='FAN_IN' uniform=False

    To get http://arxiv.org/abs/1408.5093 use:
    - factor=1.0 mode='FAN_IN' uniform=True

    To get http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf use:
    - factor=1.0 mode='FAN_AVG' uniform=True.

    To get xavier_initializer use either:
    - factor=1.0 mode='FAN_AVG' uniform=True.
    - factor=1.0 mode='FAN_AVG' uniform=False.

    Arguments:
        factor: Float.  A multiplicative factor.
        mode: String.  'FAN_IN', 'FAN_OUT', 'FAN_AVG'.
        uniform: Whether to use uniform or normal distributed random
            initialization.
        seed: A Python integer. Used to create random seeds. See
            `set_random_seed` for behavior.
        dtype: The data type. Only floating point types are supported.

    Returns:
        An initializer that generates tensors with unit variance.

    Raises:
        ValueError: if `dtype` is not a floating point type.
        TypeError: if `mode` is not in ['FAN_IN', 'FAN_OUT', 'FAN_AVG'].
    """
    return variance_scaling_initializer(factor=factor,
                                        mode=mode,
                                        uniform=uniform,
                                        seed=seed,
                                        dtype=dtype)
Beispiel #19
0
def resnet_arg_scope(
    is_training=True,
    weight_decay=0.0001,
    batch_norm_decay=0.997,
    batch_norm_epsilon=1e-5,
    batch_norm_scale=True
):
    """Defines the default ResNet arg scope.

  TODO(gpapan): The batch-normalization related default values above are
    appropriate for use in conjunction with the reference ResNet models
    released at https://github.com/KaimingHe/deep-residual-networks. When
    training ResNets from scratch, they might need to be tuned.

  Args:
    is_training: Whether or not we are training the parameters in the batch
      normalization layers of the model.
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_decay: The moving average decay when estimating layer activation
      statistics in batch normalization.
    batch_norm_epsilon: Small constant to prevent division by zero when
      normalizing activations by their variance in batch normalization.
    batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the
      activations in the batch normalization layer.

  Returns:
    An `arg_scope` to use for the resnet models.
  """
    batch_norm_params = {
        'is_training': is_training,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'updates_collections': ops.GraphKeys.UPDATE_OPS,
    }

    with arg_scope(
        [layers_lib.conv2d],
        weights_regularizer=regularizers.l2_regularizer(weight_decay),
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=layers.batch_norm,
        normalizer_params=batch_norm_params
    ):
        with arg_scope([layers.batch_norm], **batch_norm_params):
            # The following implies padding='SAME' for pool1, which makes feature
            # alignment easier for dense prediction tasks. This is also used in
            # https://github.com/facebook/fb.resnet.torch. However the accompanying
            # code of 'Deep Residual Learning for Image Recognition' uses
            # padding='VALID' for pool1. You can switch to that choice by setting
            # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID').
            with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
                return arg_sc
Beispiel #20
0
def build_logits(input_net,
                 name="dnn",
                 output_dim=1,
                 l2=0.0,
                 reused=False,
                 is_training=True):
    logits_scope_name = "%s/logits" % name
    with tf.variable_scope(logits_scope_name,
                           values=(input_net, ),
                           reuse=reused) as dnn_logits_scope:

        if l2 > 1e-6:
            # add l2 regular
            l2regular = tf.contrib.layers.l2_regularizer(l2)
            dnn_logits = tf.contrib.layers.fully_connected(
                input_net,
                output_dim,
                activation_fn=None,
                variables_collections=[name],
                weights_initializer=initializers.variance_scaling_initializer(
                ),
                scope=dnn_logits_scope,
                weights_regularizer=l2regular,
                biases_regularizer=l2regular,
                reuse=reused)
        else:
            dnn_logits = tf.contrib.layers.fully_connected(
                input_net,
                output_dim,
                activation_fn=None,
                variables_collections=[name],
                weights_initializer=initializers.variance_scaling_initializer(
                ),
                scope=dnn_logits_scope,
                reuse=reused)
        tf_summary.add_hidden_layer_summary(dnn_logits, logits_scope_name)
        tf_summary.add_net_abs_mean_summary(dnn_logits, logits_scope_name)

    return dnn_logits
Beispiel #21
0
def resnet_arg_scope(is_training=True,
                     weight_decay=0.0001,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
  """Defines the default ResNet arg scope.

  TODO(gpapan): The batch-normalization related default values above are
    appropriate for use in conjunction with the reference ResNet models
    released at https://github.com/KaimingHe/deep-residual-networks. When
    training ResNets from scratch, they might need to be tuned.

  Args:
    is_training: Whether or not we are training the parameters in the batch
      normalization layers of the model.
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_decay: The moving average decay when estimating layer activation
      statistics in batch normalization.
    batch_norm_epsilon: Small constant to prevent division by zero when
      normalizing activations by their variance in batch normalization.
    batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the
      activations in the batch normalization layer.

  Returns:
    An `arg_scope` to use for the resnet models.
  """
  batch_norm_params = {
      'is_training': is_training,
      'decay': batch_norm_decay,
      'epsilon': batch_norm_epsilon,
      'scale': batch_norm_scale,
      'updates_collections': ops.GraphKeys.UPDATE_OPS,
  }

  with arg_scope(
      [layers_lib.conv2d],
      weights_regularizer=regularizers.l2_regularizer(weight_decay),
      weights_initializer=initializers.variance_scaling_initializer(),
      activation_fn=nn_ops.relu,
      normalizer_fn=layers.batch_norm,
      normalizer_params=batch_norm_params):
    with arg_scope([layers.batch_norm], **batch_norm_params):
      # The following implies padding='SAME' for pool1, which makes feature
      # alignment easier for dense prediction tasks. This is also used in
      # https://github.com/facebook/fb.resnet.torch. However the accompanying
      # code of 'Deep Residual Learning for Image Recognition' uses
      # padding='VALID' for pool1. You can switch to that choice by setting
      # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID').
      with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
        return arg_sc
Beispiel #22
0
def inception_v3_arg_scope(weight_decay=0.00004,
                           batch_norm_var_collection='moving_vars',
                           batch_norm_decay=0.9997,
                           batch_norm_epsilon=0.001,
                           updates_collections=ops.GraphKeys.UPDATE_OPS,
                           use_fused_batchnorm=True):
  """Defines the default InceptionV3 arg scope.

  Args:
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_var_collection: The name of the collection for the batch norm
      variables.
    batch_norm_decay: Decay for batch norm moving average
    batch_norm_epsilon: Small float added to variance to avoid division by zero
    updates_collections: Collections for the update ops of the layer
    use_fused_batchnorm: Enable fused batchnorm.

  Returns:
    An `arg_scope` to use for the inception v3 model.
  """
  batch_norm_params = {
      # Decay for the moving averages.
      'decay': batch_norm_decay,
      # epsilon to prevent 0s in variance.
      'epsilon': batch_norm_epsilon,
      # collection containing update_ops.
      'updates_collections': updates_collections,
      # Use fused batch norm if possible.
      'fused': use_fused_batchnorm,
      # collection containing the moving mean and moving variance.
      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }

  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=layers_lib.batch_norm,
        normalizer_params=batch_norm_params) as sc:
      return sc
Beispiel #23
0
def inception_v3_arg_scope(weight_decay=0.00004,
                           batch_norm_var_collection='moving_vars',
                           batch_norm_decay=0.9997,
                           batch_norm_epsilon=0.001,
                           updates_collections=ops.GraphKeys.UPDATE_OPS,
                           use_fused_batchnorm=True):
  """Defines the default InceptionV3 arg scope.

  Args:
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_var_collection: The name of the collection for the batch norm
      variables.
    batch_norm_decay: Decay for batch norm moving average
    batch_norm_epsilon: Small float added to variance to avoid division by zero
    updates_collections: Collections for the update ops of the layer
    use_fused_batchnorm: Enable fused batchnorm.

  Returns:
    An `arg_scope` to use for the inception v3 model.
  """
  batch_norm_params = {
      # Decay for the moving averages.
      'decay': batch_norm_decay,
      # epsilon to prevent 0s in variance.
      'epsilon': batch_norm_epsilon,
      # collection containing update_ops.
      'updates_collections': updates_collections,
      # Use fused batch norm if possible.
      'fused': use_fused_batchnorm,
      # collection containing the moving mean and moving variance.
      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }

  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=layers_lib.batch_norm,
        normalizer_params=batch_norm_params) as sc:
      return sc
Beispiel #24
0
def conv_layer(input,
               output_dep,
               filter_size,
               is_training=True,
               data_format='channels_first'):
    input = tf.layers.conv2d(
        input,
        256,
        1,
        kernel_initializer=initializers.variance_scaling_initializer(
            factor=2.0, mode='FAN_IN', uniform=False),
        data_format=data_format)
    input = batch_norm(input, training=is_training, data_format=data_format)
    input = tf.nn.relu(input)
    return input
Beispiel #25
0
def inception_v1_arg_scope(weight_decay=0.00004,
                           use_batch_norm=True,
                           batch_norm_var_collection='moving_vars'):
  """Defines the default InceptionV1 arg scope.

  Note: Althougth the original paper didn't use batch_norm we found it useful.

  Args:
    weight_decay: The weight decay to use for regularizing the model.
    use_batch_norm: "If `True`, batch_norm is applied after each convolution.
    batch_norm_var_collection: The name of the collection for the batch norm
      variables.

  Returns:
    An `arg_scope` to use for the inception v3 model.
  """
  batch_norm_params = {
      # Decay for the moving averages.
      'decay': 0.9997,
      # epsilon to prevent 0s in variance.
      'epsilon': 0.001,
      # collection containing update_ops.
      'updates_collections': ops.GraphKeys.UPDATE_OPS,
      # collection containing the moving mean and moving variance.
      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }
  if use_batch_norm:
    normalizer_fn = layers_lib.batch_norm
    normalizer_params = batch_norm_params
  else:
    normalizer_fn = None
    normalizer_params = {}
  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=normalizer_fn,
        normalizer_params=normalizer_params) as sc:
      return sc
Beispiel #26
0
def atlas_decode(siggrid,
                 gridn,
                 is_training,
                 scope="",
                 reuse=False,
                 bottleneck_size=1024,
                 weight_decay=1e-4,
                 batch_norm_decay=0.997,
                 batch_norm_epsilon=1e-5,
                 batch_norm_scale=True):
    batch_norm_params = {
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'is_training': is_training,
        'updates_collections': ops.GraphKeys.UPDATE_OPS
    }
    ix = tf.reshape(
        siggrid,
        [tf.shape(siggrid)[0], 1,
         tf.shape(siggrid)[1], siggrid.shape[-1]])
    oy = []
    with arg_scope(
        [slim.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            activation_fn=tf.nn.relu,
            normalizer_fn=slim.batch_norm,
            normalizer_params=batch_norm_params,
            reuse=reuse):
        with arg_scope([slim.batch_norm], **batch_norm_params):
            for i in range(gridn):
                x = slim.conv2d(ix,
                                bottleneck_size, [1, 1],
                                scope=scope + "_grid%d_fc1" % i)
                x = slim.conv2d(x,
                                bottleneck_size // 2, [1, 1],
                                scope=scope + "_grid%d_fc2" % i)
                x = slim.conv2d(x,
                                bottleneck_size // 4, [1, 1],
                                scope=scope + "_grid%d_fc3" % i)
                oy.append(
                    slim.conv2d(x,
                                3, [1, 1],
                                activation_fn=tf.nn.tanh,
                                scope=scope + "_grid%d_fc4" % i))
            y = tf.concat(oy, axis=2)
            return tf.reshape(y, [tf.shape(siggrid)[0], -1, 3])
Beispiel #27
0
def inception_v1_arg_scope(weight_decay=0.00004,
                           use_batch_norm=True,
                           batch_norm_var_collection='moving_vars'):
  """Defines the default InceptionV1 arg scope.

  Note: Althougth the original paper didn't use batch_norm we found it useful.

  Args:
    weight_decay: The weight decay to use for regularizing the model.
    use_batch_norm: "If `True`, batch_norm is applied after each convolution.
    batch_norm_var_collection: The name of the collection for the batch norm
      variables.

  Returns:
    An `arg_scope` to use for the inception v3 model.
  """
  batch_norm_params = {
      # Decay for the moving averages.
      'decay': 0.9997,
      # epsilon to prevent 0s in variance.
      'epsilon': 0.001,
      # collection containing update_ops.
      'updates_collections': ops.GraphKeys.UPDATE_OPS,
      # collection containing the moving mean and moving variance.
      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }
  if use_batch_norm:
    normalizer_fn = layers_lib.batch_norm
    normalizer_params = batch_norm_params
  else:
    normalizer_fn = None
    normalizer_params = {}
  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=normalizer_fn,
        normalizer_params=normalizer_params) as sc:
      return sc
Beispiel #28
0
def kconv(x, k, knn_index, d, scope, is_training, reuse):
    weight_decay = 1e-4
    batch_norm_decay = 0.997
    batch_norm_epsilon = 1e-5
    batch_norm_scale = True
    batch_norm_params = {
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'is_training': is_training,
        'updates_collections': ops.GraphKeys.UPDATE_OPS
    }
    with arg_scope(
        [slim.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            activation_fn=tf.nn.relu,
            normalizer_fn=slim.batch_norm,
            normalizer_params=batch_norm_params,
            reuse=reuse):
        with arg_scope([slim.batch_norm], **batch_norm_params):
            if int(x.shape[-1]) > 256:  # too large to fit in gpu
                x = tf.reshape(
                    x, [tf.shape(x)[0],
                        tf.shape(x)[1], 1,
                        int(x.shape[2])])
                x = slim.conv2d(x, 256, [1, 1], scope=scope + '_reducedim')
            rx = tf.reshape(x,
                            [tf.shape(x)[0],
                             tf.shape(x)[1],
                             int(x.shape[-1])])
            xknn = tf.gather_nd(rx, knn_index, name=scope + '_gather')
            x = tf.concat([
                tf.reshape(
                    x, [tf.shape(x)[0],
                        tf.shape(x)[1], 1,
                        int(x.shape[-1])]), xknn
            ],
                          axis=2)
            x = slim.conv2d(x,
                            d, [1, k + 1],
                            scope=scope + "_kconv",
                            padding='VALID')
    return tf.reshape(x, [tf.shape(x)[0], -1, d])
Beispiel #29
0
def mlp(sig, grid, siggrid, param=None):
    scope = param[0]
    grid_num = param[1]
    is_training = param[2]
    reuse = param[3]
    weight_decay = 1e-4
    batch_norm_decay = 0.997
    batch_norm_epsilon = 1e-5
    batch_norm_scale = True
    batch_norm_params = {
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'is_training': is_training,
        'updates_collections': ops.GraphKeys.UPDATE_OPS
    }
    ix = tf.reshape(
        siggrid,
        [tf.shape(siggrid)[0], 1,
         tf.shape(siggrid)[1], siggrid.shape[-1]])
    oy = []
    with arg_scope(
        [slim.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            activation_fn=tf.nn.relu,
            normalizer_fn=slim.batch_norm,
            normalizer_params=batch_norm_params,
            reuse=reuse):
        with arg_scope([slim.batch_norm], **batch_norm_params):
            for i in range(grid_num):
                x = ix
                for j, size in enumerate(param[-1]):
                    x = slim.conv2d(x,
                                    size, [1, 1],
                                    scope=scope + "_grid%d_fc%d" % (i, j))
                oy.append(
                    slim.conv2d(x,
                                3, [1, 1],
                                activation_fn=tf.nn.tanh,
                                scope=scope + "_grid%d_fc%d" % (i, j + 1)))
            y = tf.concat(oy, axis=2)
            return tf.reshape(y, [tf.shape(siggrid)[0], -1, 3])
def resnet_arg_scope(
        is_training=True, weight_decay=cfg.weight_decay, batch_norm_decay=0.997,
        batch_norm_epsilon=1e-5, batch_norm_scale=True):
    batch_norm_params = {
        'is_training': False, 'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale,
        'trainable': cfg.bn_training,
        'updates_collections': ops.GraphKeys.UPDATE_OPS
    }

    with arg_scope(
            [slim.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            trainable=is_training,
            activation_fn=nn_ops.relu,
            normalizer_fn=layers.batch_norm,
            normalizer_params=batch_norm_params):
        with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
            return arg_sc
Beispiel #31
0
def resnet_arg_scope(is_training=True,
                     weight_decay=0.0001,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=0.00001,
                     batch_norm_scale=True):
  """Defines the default ResNet arg scope.
  
  Args:
    is_training: Whether or not we are training the parameters in the batch
      normalization layers of the model. (deprecated)
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_decay: The moving average decay when estimating layer activation
      statistics in batch normalization.
    batch_norm_epsilon: Small constant to prevent division by zero when
      normalizing activations by their variance in batch normalization.
    batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the
      activations in the batch normalization layer.

  Returns:
    An `arg_scope` to use for the resnet models.
  """
  batch_norm_params = {
      'is_training': is_training,
      'decay': batch_norm_decay,
      'epsilon': batch_norm_epsilon,
      'scale': batch_norm_scale,
      'updates_collections': tf.GraphKeys.UPDATE_OPS
      #'updates_collections': None,
      #'renorm': True
  }

  with arg_scope(
      [layers_lib.conv3d],
      weights_regularizer=regularizers.l2_regularizer(weight_decay),
      weights_initializer=initializers.variance_scaling_initializer(),
      activation_fn=nn_ops.relu,
      normalizer_fn=layers.batch_norm):
    with arg_scope([layers.batch_norm], **batch_norm_params):
      with arg_scope([layers.max_pool3d], padding='SAME') as arg_sc:
        return arg_sc
Beispiel #32
0
def inception_v2_arg_scope(weight_decay=0.00004,
                           batch_norm_var_collection='moving_vars'):
  """Defines the default InceptionV2 arg scope.

  Args:
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_var_collection: The name of the collection for the batch norm
      variables.

  Returns:
    An `arg_scope` to use for the inception v3 model.
  """
  batch_norm_params = {
      # Decay for the moving averages.
      'decay': 0.9997,
      # epsilon to prevent 0s in variance.
      'epsilon': 0.001,
      # collection containing update_ops.
      'updates_collections': ops.GraphKeys.UPDATE_OPS,
      # collection containing the moving mean and moving variance.
      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }

  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=layers_lib.batch_norm,
        normalizer_params=batch_norm_params) as sc:
      return sc
def resnet_arg_scope(weight_decay=0.0001,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
    """Defines the default ResNet arg scope.
  Args:
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_decay: The moving average decay when estimating layer activation
      statistics in batch normalization.
    batch_norm_epsilon: Small constant to prevent division by zero when
      normalizing activations by their variance in batch normalization.
    batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the
      activations in the batch normalization layer.
  Returns:
    An `arg_scope` to use for the resnet models.
  """
    batch_norm_params = {
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'updates_collections': tf.GraphKeys.UPDATE_OPS,
    }

    with arg_scope(
        [layers_lib.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            activation_fn=tf.nn.relu,
            normalizer_fn=layers.batch_norm,
            normalizer_params=batch_norm_params):
        with arg_scope([layers.batch_norm], **batch_norm_params):
            # The following implies padding='SAME' for pool1, which makes feature
            # alignment easier for dense prediction tasks. This is also used in
            # https://github.com/facebook/fb.resnet.torch. However the accompanying
            # code of 'Deep Residual Learning for Image Recognition' uses
            # padding='VALID' for pool1. You can switch to that choice by setting
            # tf.contrib.framework.arg_scope([layers_lib.max_pool2d], padding='VALID').
            with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
                return arg_sc
Beispiel #34
0
def inception_v2_arg_scope(weight_decay=0.00004,
                           batch_norm_var_collection='moving_vars'):
  """Defines the default InceptionV2 arg scope.

  Args:
    weight_decay: The weight decay to use for regularizing the model.
    batch_norm_var_collection: The name of the collection for the batch norm
      variables.

  Returns:
    An `arg_scope` to use for the inception v3 model.
  """
  batch_norm_params = {
      # Decay for the moving averages.
      'decay': 0.9997,
      # epsilon to prevent 0s in variance.
      'epsilon': 0.001,
      # collection containing update_ops.
      'updates_collections': ops.GraphKeys.UPDATE_OPS,
      # collection containing the moving mean and moving variance.
      'variables_collections': {
          'beta': None,
          'gamma': None,
          'moving_mean': [batch_norm_var_collection],
          'moving_variance': [batch_norm_var_collection],
      }
  }

  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=nn_ops.relu,
        normalizer_fn=layers_lib.batch_norm,
        normalizer_params=batch_norm_params) as sc:
      return sc
Beispiel #35
0
def predictron_arg_scope(weight_decay=0.0001,
                         batch_norm_decay=0.997,
                         batch_norm_epsilon=1e-5,
                         batch_norm_scale=True):
  batch_norm_params = {
    'decay': batch_norm_decay,
    'epsilon': batch_norm_epsilon,
    'scale': batch_norm_scale,
    'updates_collections': tf.GraphKeys.UPDATE_OPS,
  }

  # Set weight_decay for weights in Conv and FC layers.
  with arg_scope(
      [layers.conv2d, layers_lib.fully_connected],
      weights_regularizer=regularizers.l2_regularizer(weight_decay)):
    with arg_scope(
        [layers.conv2d],
        weights_initializer=initializers.variance_scaling_initializer(),
        activation_fn=None,
        normalizer_fn=layers_lib.batch_norm,
        normalizer_params=batch_norm_params) as sc:
      return sc
Beispiel #36
0
def Encoder_fc3_dropout(x,
                        num_output=85,
                        is_training=True,
                        reuse=False,
                        name="3D_module"):
    """
    3D inference module. 3 MLP layers (last is the output)
    With dropout  on first 2.
    Input:
    - x: N x [|img_feat|, |3D_param|]
    - reuse: bool

    Outputs:
    - 3D params: N x num_output
      if orthogonal: 
           either 85: (3 + 24*3 + 10) or 109 (3 + 24*4 + 10) for factored axis-angle representation
      if perspective:
          86: (f, tx, ty, tz) + 24*3 + 10, or 110 for factored axis-angle.
    - variables: tf variables
    """
    if reuse:
        print('Reuse is on!')
    with tf.variable_scope(name, reuse=reuse) as scope:
        net = slim.fully_connected(x, 1024, scope='fc1')
        net = slim.dropout(net, 0.5, is_training=is_training, scope='dropout1')
        net = slim.fully_connected(net, 1024, scope='fc2')
        net = slim.dropout(net, 0.5, is_training=is_training, scope='dropout2')
        small_xavier = variance_scaling_initializer(
            factor=.01, mode='FAN_AVG', uniform=True)
        net = slim.fully_connected(
            net,
            num_output,
            activation_fn=None,
            weights_initializer=small_xavier,
            scope='fc3')

    variables = tf.contrib.framework.get_variables(scope)
    return net, variables
Beispiel #37
0
def resnet_arg_scope(weight_decay=0.0001,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True,
                     is_training=True):
    batch_norm_params = {
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'updates_collections': ops.GraphKeys.UPDATE_OPS,
        'is_training': is_training
    }

    with arg_scope(
        [layers_lib.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            activation_fn=nn_ops.relu,
            normalizer_fn=layers.batch_norm,
            normalizer_params=batch_norm_params):
        with arg_scope([layers.batch_norm], **batch_norm_params):
            with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
                return arg_sc
Beispiel #38
0
def model_deconv(inputs,data_format='channels_first',is_training=True):
    model_class = ImagenetModel(
        resnet_size=50, data_format=data_format,num_classes=0,resnet_version=1)
    _, end_points = model_class(inputs, is_training)

    f = [end_points['block_layer4'], end_points['block_layer3'],
         end_points['block_layer2'], end_points['block_layer1']]
    P = [None, None, None, None]
    # attent=[None,None,None,None]
    # import pudb; pudb.set_trace()
    with tf.variable_scope('u_net'):
        def deconv(input,channels,factor=2):
            input=conv_layer(input,channels,1)
            input=conv_layer(input,channels,3)
            return unpool(input,factor,data_format=data_format)

        out4=tf.concat((deconv(f[0],1024),f[1]),1)
        out3=tf.concat((deconv(out4,512),f[2]),1)
        out2=tf.concat((deconv(out3,256),f[3]),1)
        out1=deconv(out2,256,factor=1)

        out1 = tf.layers.conv2d(out1, 1, 3, padding='same', kernel_initializer=initializers.variance_scaling_initializer(
            factor=2.0, mode='FAN_IN', uniform=False), data_format=data_format)
        out1=batch_norm(out1,training=is_training,data_format=data_format)
        out1=tf.nn.relu(out1)

        for i in range(config['n']):
            seg_map=tf.layers.conv2d(out1,1,1,data_format=data_format)
            seg_map = tf.sigmoid(unpool(seg_map, 4,data_format=data_format))
            if i == 0:
                seg_maps = seg_map
            else:
                seg_maps = tf.concat((seg_maps, seg_map), 1 if data_format == 'channels_first' else -1)

    with tf.name_scope('format_change'):
        seg_maps=tf.transpose(seg_maps,[0,2,3,1])    
        return seg_maps, f
Beispiel #39
0
def resnet_pre_arg_scope(#is_training=True,
                     weight_decay=0.0001,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):

  batch_norm_params = {
      #'is_training': is_training,
      'decay': batch_norm_decay,
      'epsilon': batch_norm_epsilon,
      'scale': batch_norm_scale,
     # 'activation_fn': nn_ops.relu,
      'variables_collections': ["batch_norm_params"],
      'updates_collections': tf.GraphKeys.UPDATE_OPS      
  }

  with arg_scope(
      [layers_lib.conv3d],
      weights_regularizer=regularizers.l2_regularizer(weight_decay),
      weights_initializer=initializers.variance_scaling_initializer(),
      activation_fn=None):
    with arg_scope([layers.batch_norm], **batch_norm_params):
      with arg_scope([layers.max_pool3d], padding='SAME') as arg_sc:
        return arg_sc
Beispiel #40
0
def resnet_arg_scope(is_training=False,
                     weight_decay=cfg.TRAIN.WEIGHT_DECAY,
                     batch_norm_decay=0.997,
                     batch_norm_epsilon=1e-5,
                     batch_norm_scale=True):
    batch_norm_params = {
        'is_training': False,
        'decay': batch_norm_decay,
        'epsilon': batch_norm_epsilon,
        'scale': batch_norm_scale,
        'trainable': cfg.RESNET.BN_TRAIN,
        'updates_collections': ops.GraphKeys.UPDATE_OPS
    }

    with arg_scope(
        [slim.conv2d],
            weights_regularizer=regularizers.l2_regularizer(weight_decay),
            weights_initializer=initializers.variance_scaling_initializer(),
            trainable=is_training,
            activation_fn=nn_ops.relu,
            normalizer_fn=layers.batch_norm,
            normalizer_params=batch_norm_params):
        with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
            return arg_sc
Beispiel #41
0
import tensorflow as tf
# from tensorflow.contrib import learn
from tensorflow.contrib.layers import convolution2d
from tensorflow.contrib.framework.python.ops import variables
from tensorflow.contrib.layers.python.layers import utils
from tensorflow.contrib.layers.python.layers import initializers

# method 1: contirb API
net = convolution2d(net,
                    num_outputs=num_ker,
                    kernel_size=[ker_size,1],
                    stride=[stride,1]
                    padding='SAME',
                    activation_fn=None,
                    normalizer_fn=None,
                    weights_initializer=initializers.variance_scaling_initializer(),
                    biases_initializer=init_ops.zeros_initializer,)

# method 2: original conv2d API
weights = variables.model_variable(
    'weights',
    shape=[ker_size, 1, num_ker_in, num_ker],
    dtype=dtype,
    initializer=initializers.variance_scaling_initializer(),
    trainable=True)
biases = variables.model_variable(
    'biases',
    shape=[num_ker],
    dtype=dtype,
    initializer=tf.zeros_initializer,
    trainable=True)