def generator2(s, training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'is_training': training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': ops.GraphKeys.UPDATE_OPS, } with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=lrelu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): s = slim.stack(s, slim.conv2d, [(64, [4,4], [2,2]), (128, [4,4], [2,2])], scope="convolution") # Res with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): res = slim.conv2d(s, 256, [3,3], [1,1], scope="res1_1") res = slim.conv2d(res, 256, [3,3], [1,1], scope="res1_2") gen = s + res res = slim.conv2d(gen, 256, [3,3], [1,1], scope="res2_1") res = slim.conv2d(res, 256, [3,3], [1,1], scope="res2_2") gen = gen + res with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): gen = tf.image.resize_nearest_neighbor(gen, [64,64]) gen = slim.conv2d(gen, 1024, [3,3], [1,1], scope="convolution_2") gen = tf.image.resize_nearest_neighbor(gen, [128,128]) gen = slim.conv2d(gen, 512, [3,3], [1,1], scope="convolution_2_1") # l = [(4096, [3,3], [2,2]), (2048, [3,3], [2,2]), (2048, [3,3], [2,2]), # (2048, [3,3], [2,2]), (1024, [3,3], [2,2]), (512, [3,3], [2,2]), (256, [3,3], [2,2]), (128, [3,3], [1,1]), (3, [3,3], [1,1])] gen = slim.conv2d(gen, 3, [3,3], [1,1], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=tf.tanh, scope="convolutionend") return gen
def test_wrong_dtype(self): with self.assertRaisesRegexp( TypeError, 'Cannot create initializer for non-floating point type.'): initializers.variance_scaling_initializer(dtype=dtypes.int32) initializer = initializers.variance_scaling_initializer() with self.assertRaisesRegexp( TypeError, 'Cannot create initializer for non-floating point type.'): initializer([], dtype=dtypes.int32)
def test_wrong_dtype(self): with self.assertRaisesRegexp( TypeError, 'Cannot create initializer for non-floating point type.'): initializers.variance_scaling_initializer(dtype=dtypes.int32) initializer = initializers.variance_scaling_initializer() with self.assertRaisesRegexp( TypeError, 'Cannot create initializer for non-floating point type.'): initializer([], dtype=dtypes.int32)
def generator1(z, training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'is_training': training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': ops.GraphKeys.UPDATE_OPS, } c0 = tf.reshape(z, [batch_size, 1, 1, z_dim]) with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): gen = tf.image.resize_nearest_neighbor(c0, [2, 2]) gen = slim.conv2d(gen, 1024, [3, 3], [1, 1], scope="convolution1") gen = tf.image.resize_nearest_neighbor(gen, [4, 4]) gen = slim.conv2d(gen, 512, [3, 3], [1, 1], scope="convolution2") gen = tf.image.resize_nearest_neighbor(gen, [8, 8]) gen = slim.conv2d(gen, 256, [3, 3], [1, 1], scope="convolution3") gen = tf.image.resize_nearest_neighbor(gen, [16, 16]) gen = slim.conv2d(gen, 128, [3, 3], [1, 1], scope="convolution4") gen = tf.image.resize_nearest_neighbor(gen, [32, 32]) gen = slim.conv2d(gen, 64, [3, 3], [1, 1], scope="convolution5") gen = tf.image.resize_nearest_neighbor(gen, [64, 64]) # l = [(4096, [3,3], [2,2]), (2048, [3,3], [2,2]), (2048, [3,3], [2,2]), # (2048, [3,3], [2,2]), (1024, [3,3], [2,2]), (512, [3,3], [2,2]), (256, [3,3], [2,2]), (128, [3,3], [1,1]), (3, [3,3], [1,1])] gen = slim.conv2d( gen, 3, [3, 3], [1, 1], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=tf.tanh, scope="convolutionend") return gen
def discriminator2(img, training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'is_training': training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': ops.GraphKeys.UPDATE_OPS, } disc = slim.conv2d( img, 64, [4, 4], [2, 2], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=lrelu, scope="convolutionstart") with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=lrelu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): disc = slim.stack(disc, slim.conv2d, [(128, [4, 4], [2, 2]), (256, [4, 4], [2, 2]), (512, [4, 4], [2, 2]), (1024, [4, 4], [2, 2])], scope="convolution") disc = slim.conv2d( disc, 1024, [1, 1], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=lrelu, scope="convolutionend") disc = tf.reshape(disc, [batch_size, 4 * 4 * 1024]) disc = slim.fully_connected(disc, 1, activation_fn=None, scope="logits") return disc
def resnet_arg_scope(bn_is_training, bn_trainable, trainable=True, weight_decay=config.TRAIN.weight_decay_factor, batch_norm_decay=0.99, batch_norm_epsilon=1e-9, batch_norm_scale=True, data_format='NHWC'): batch_norm_params = { 'is_training': bn_is_training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'trainable': bn_trainable, 'updates_collections': ops.GraphKeys.UPDATE_OPS, 'fused': True } with arg_scope( [slim.conv2d, slim.separable_conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), trainable=trainable, activation_fn=nn_ops.relu, normalizer_fn=slim.batch_norm if 'BN' in config.TRAIN.norm else GroupNorm, normalizer_params=batch_norm_params if 'BN' in config.TRAIN.norm else None, data_format=data_format): with arg_scope([layers.batch_norm, layers.max_pool2d], data_format=data_format): with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc: return arg_sc
def resnet_arg_scope(is_training=True, weight_decay=cfg.TRAIN.WEIGHT_DECAY, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { # NOTE 'is_training' here does not work because inside resnet it gets reset: # https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187 'is_training': False, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'trainable': cfg.RESNET.BN_TRAIN, 'updates_collections': ops.GraphKeys.UPDATE_OPS, 'fused': True } with arg_scope( [slim.conv2d, slim.separable_conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), trainable=is_training, activation_fn=nn_ops.relu, # normalizer_fn=None, # normalizer_params=None): normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc: return arg_sc
def resnet_arg_scope_gn(weight_decay=0.0001, group_norm_num=32, group_norm_dim=None): """Defines the default ResNet arg scope. Args: weight_decay: The weight decay to use for regularizing the model. Returns: An `arg_scope` to use for the resnet models. """ with arg_scope( [conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=group_norm2d): with arg_scope( [sep_conv2d], normalizer_fn=group_norm2d, activation_fn_middle=nn_ops.relu, pointwise_weights_regularizer=regularizers.l2_regularizer( weight_decay)): with arg_scope([group_norm2d], group_num=group_norm_num, group_size=group_norm_dim, eps=1e-05, affine=True): with arg_scope([max_pool2d], padding='SAME') as arg_sc: return arg_sc
def resnet_arg_scope(is_training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'is_training': False, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'trainable': cfg.RESNET.BN_TRAIN, 'updates_collections': ops.GraphKeys.UPDATE_OPS } with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), trainable=is_training, activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params): # The following implies padding='SAME' for pool1, which makes feature # alignment easier for dense prediction tasks. This is also used in # https://github.com/facebook/fb.resnet.torch. However the accompanying # code of 'Deep Residual Learning for Image Recognition' uses # padding='VALID' for pool1. You can switch to that choice by setting # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID'). with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc: return arg_sc
def linear_mapping_weightnorm(self, inputs, out_dim, var_scope_name="linear", output_collection=None): with tf.variable_scope(var_scope_name): # pylint: disable=invalid-name input_shape = inputs.get_shape().as_list() # static shape. may has None # use weight normalization (Salimans & Kingma, 2016) w = g* v/2-norm(v) V = tf.get_variable( name='V', shape=[int(input_shape[-1]), out_dim], dtype=tf.float32, initializer=initializers.variance_scaling_initializer()) # V shape is M*N, V_norm shape is N V_norm = tf.norm(V.initialized_value(), axis=0) g = tf.get_variable('g', dtype=tf.float32, initializer=V_norm) # weightnorm bias is init zero b = tf.get_variable( name='b', shape=[out_dim], dtype=tf.float32, initializer=tf.zeros_initializer()) assert len(input_shape) == 3 inputs = tf.reshape(inputs, [-1, input_shape[-1]]) inputs = tf.matmul(inputs, V) inputs = tf.reshape(inputs, [input_shape[0], -1, out_dim]) # g/2-norm(v) scaler = tf.div(g, tf.norm(V, axis=0)) # x*v g/2-norm(v) + b inputs = tf.reshape(scaler, [1, out_dim]) * inputs + tf.reshape(b, [1, out_dim]) if self.is_training: tf.add_to_collection(output_collection, inputs) return inputs
def fc2_res(phi, name='fc2_res'): """ Converts pretrained (fixed) resnet features phi into movie strip. This applies 2 fc then add it to the orig as residuals. Args: phi (B x T x 2048): Image feature. name (str): Scope. Returns: Phi (B x T x 2048): Hallucinated movie strip. """ with tf.variable_scope(name, reuse=False): net = slim.fully_connected(phi, 2048, scope='fc1') net = slim.fully_connected(net, 2048, scope='fc2') small_xavier = variance_scaling_initializer(factor=.001, mode='FAN_AVG', uniform=True) net_final = slim.fully_connected(net, 2048, activation_fn=None, weights_initializer=small_xavier, scope='fc3') new_phi = net_final + phi return new_phi
def resnet_arg_scope(bn_is_training, bn_trainable, trainable=True, weight_decay=cfg.weight_decay, batch_norm_decay=0.99, batch_norm_epsilon=1e-9, batch_norm_scale=True): batch_norm_params = { 'is_training': bn_is_training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'trainable': bn_trainable, 'updates_collections': ops.GraphKeys.UPDATE_OPS } with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), trainable=trainable, activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc: return arg_sc
def Encoder_fc3_dropout(x, num_output=85, is_training=True, reuse=False, name="3D_module"): """ 3D inference module. 3 MLP layers (last is the output) With dropout on first 2. Input: - x: N x [|img_feat|, |3D_param|] - reuse: bool Outputs: - 3D params: N x num_output if orthogonal: either 85: (3 + 24*3 + 10) or 109 (3 + 24*4 + 10) for factored axis-angle representation if perspective: 86: (f, tx, ty, tz) + 24*3 + 10, or 110 for factored axis-angle. - variables: tf variables """ if reuse: print('Reuse is on!') with tf.compat.v1.variable_scope(name, reuse=reuse) as scope: net = tf.contrib.layers.fully_connected(x, 1024, scope='fc1') net = tf.contrib.layers.dropout(net, 0.5, is_training=is_training, scope='dropout1') net = tf.contrib.layers.fully_connected(net, 1024, scope='fc2') net = tf.contrib.layers.dropout(net, 0.5, is_training=is_training, scope='dropout2') small_xavier = variance_scaling_initializer( factor=.01, mode='FAN_AVG', uniform=True) net = tf.contrib.layers.fully_connected(net, num_output,activation_fn=None, weights_initializer=small_xavier, scope='fc3') variables = tf.contrib.framework.get_variables(scope) return net, variables
def resnet_arg_scope(is_training=True, weight_decay=cfg.TRAIN.WEIGHT_DECAY, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): data_format = 'NCHW' if cfg.RESNET.USE_NCHW else 'NHWC' batch_norm_params = { # NOTE 'is_training' is set appropriately inside of the resnet if we pass it to it: # https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187 # 'is_training': False, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'trainable': True, 'data_format': data_format, 'fused': True } with arg_scope( [slim.conv2d, slim.conv2d_transpose], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), trainable=is_training, activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, data_format=data_format, normalizer_params=batch_norm_params): with arg_scope([slim.max_pool2d, resnet_utils.conv2d_same], data_format=data_format): with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc: return arg_sc
def conv1d_weightnorm(self, inputs, out_dim, kernel_size, padding="SAME", var_scope_name="conv1d", output_collection=None): with tf.variable_scope(var_scope_name): # pylint: disable=invalid-name in_dim = int(inputs.get_shape()[-1]) V = tf.get_variable( name='V', shape=[kernel_size, in_dim, out_dim], dtype=tf.float32, initializer=initializers.variance_scaling_initializer()) # V shape is M*N*k, V_norm shape is k V_norm = tf.norm(V.initialized_value(), axis=[0, 1]) g = tf.get_variable('g', dtype=tf.float32, initializer=V_norm) b = tf.get_variable(name='b', shape=[out_dim], dtype=tf.float32, initializer=tf.zeros_initializer()) # use weight normalization (Salimans & Kingma, 2016) W = tf.reshape(g, [1, 1, out_dim]) * tf.nn.l2_normalize(V, [0, 1]) inputs = tf.nn.conv1d(value=inputs, filters=W, stride=1, padding=padding) inputs = tf.nn.bias_add(inputs, b) if self.is_training: tf.add_to_collection(output_collection, inputs) return inputs
def resnet_arg_scope(is_training=True, weight_decay=cfg.TRAIN.WEIGHT_DECAY, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { # NOTE 'is_training' here does not work because inside resnet it gets reset: # https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187 'is_training': False, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'trainable': cfg.RESNET.BN_TRAIN, 'updates_collections': ops.GraphKeys.UPDATE_OPS } with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), trainable=is_training, activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc: return arg_sc
def variance_scaling(factor=2.0, mode='FAN_IN', uniform=False, seed=None, dtype=tf.float32): """ Variance Scaling. Returns an initializer that generates tensors without scaling variance. When initializing a deep network, it is in principle advantageous to keep the scale of the input variance constant, so it does not explode or diminish by reaching the final layer. This initializer use the following formula: ``` if mode='FAN_IN': # Count only number of input connections. n = fan_in elif mode='FAN_OUT': # Count only number of output connections. n = fan_out elif mode='FAN_AVG': # Average number of inputs and output connections. n = (fan_in + fan_out)/2.0 truncated_normal(shape, 0.0, stddev=sqrt(factor / n)) ``` To get http://arxiv.org/pdf/1502.01852v1.pdf use (Default): - factor=2.0 mode='FAN_IN' uniform=False To get http://arxiv.org/abs/1408.5093 use: - factor=1.0 mode='FAN_IN' uniform=True To get http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf use: - factor=1.0 mode='FAN_AVG' uniform=True. To get xavier_initializer use either: - factor=1.0 mode='FAN_AVG' uniform=True. - factor=1.0 mode='FAN_AVG' uniform=False. Arguments: factor: Float. A multiplicative factor. mode: String. 'FAN_IN', 'FAN_OUT', 'FAN_AVG'. uniform: Whether to use uniform or normal distributed random initialization. seed: A Python integer. Used to create random seeds. See `set_random_seed` for behavior. dtype: The data type. Only floating point types are supported. Returns: An initializer that generates tensors with unit variance. Raises: ValueError: if `dtype` is not a floating point type. TypeError: if `mode` is not in ['FAN_IN', 'FAN_OUT', 'FAN_AVG']. """ try: from tensorflow.contrib.layers.python.layers.initializers import \ variance_scaling_initializer except ImportError: raise NotImplementedError("'variance_scaling_initializer' not " "supported, please update TensorFlow.") return variance_scaling_initializer(factor=factor, mode=mode, uniform=uniform, seed=seed, dtype=dtype)
def variance_scaling(factor=2.0, mode='FAN_IN', uniform=False, seed=None, dtype=tf.float32): """ Variance Scaling. Returns an initializer that generates tensors without scaling variance. When initializing a deep network, it is in principle advantageous to keep the scale of the input variance constant, so it does not explode or diminish by reaching the final layer. This initializer use the following formula: ``` if mode='FAN_IN': # Count only number of input connections. n = fan_in elif mode='FAN_OUT': # Count only number of output connections. n = fan_out elif mode='FAN_AVG': # Average number of inputs and output connections. n = (fan_in + fan_out)/2.0 truncated_normal(shape, 0.0, stddev=sqrt(factor / n)) ``` To get http://arxiv.org/pdf/1502.01852v1.pdf use (Default): - factor=2.0 mode='FAN_IN' uniform=False To get http://arxiv.org/abs/1408.5093 use: - factor=1.0 mode='FAN_IN' uniform=True To get http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf use: - factor=1.0 mode='FAN_AVG' uniform=True. To get xavier_initializer use either: - factor=1.0 mode='FAN_AVG' uniform=True. - factor=1.0 mode='FAN_AVG' uniform=False. Arguments: factor: Float. A multiplicative factor. mode: String. 'FAN_IN', 'FAN_OUT', 'FAN_AVG'. uniform: Whether to use uniform or normal distributed random initialization. seed: A Python integer. Used to create random seeds. See `set_random_seed` for behavior. dtype: The data type. Only floating point types are supported. Returns: An initializer that generates tensors with unit variance. Raises: ValueError: if `dtype` is not a floating point type. TypeError: if `mode` is not in ['FAN_IN', 'FAN_OUT', 'FAN_AVG']. """ return variance_scaling_initializer(factor=factor, mode=mode, uniform=uniform, seed=seed, dtype=dtype)
def resnet_arg_scope( is_training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True ): """Defines the default ResNet arg scope. TODO(gpapan): The batch-normalization related default values above are appropriate for use in conjunction with the reference ResNet models released at https://github.com/KaimingHe/deep-residual-networks. When training ResNets from scratch, they might need to be tuned. Args: is_training: Whether or not we are training the parameters in the batch normalization layers of the model. weight_decay: The weight decay to use for regularizing the model. batch_norm_decay: The moving average decay when estimating layer activation statistics in batch normalization. batch_norm_epsilon: Small constant to prevent division by zero when normalizing activations by their variance in batch normalization. batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the activations in the batch normalization layer. Returns: An `arg_scope` to use for the resnet models. """ batch_norm_params = { 'is_training': is_training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': ops.GraphKeys.UPDATE_OPS, } with arg_scope( [layers_lib.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params ): with arg_scope([layers.batch_norm], **batch_norm_params): # The following implies padding='SAME' for pool1, which makes feature # alignment easier for dense prediction tasks. This is also used in # https://github.com/facebook/fb.resnet.torch. However the accompanying # code of 'Deep Residual Learning for Image Recognition' uses # padding='VALID' for pool1. You can switch to that choice by setting # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID'). with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc: return arg_sc
def build_logits(input_net, name="dnn", output_dim=1, l2=0.0, reused=False, is_training=True): logits_scope_name = "%s/logits" % name with tf.variable_scope(logits_scope_name, values=(input_net, ), reuse=reused) as dnn_logits_scope: if l2 > 1e-6: # add l2 regular l2regular = tf.contrib.layers.l2_regularizer(l2) dnn_logits = tf.contrib.layers.fully_connected( input_net, output_dim, activation_fn=None, variables_collections=[name], weights_initializer=initializers.variance_scaling_initializer( ), scope=dnn_logits_scope, weights_regularizer=l2regular, biases_regularizer=l2regular, reuse=reused) else: dnn_logits = tf.contrib.layers.fully_connected( input_net, output_dim, activation_fn=None, variables_collections=[name], weights_initializer=initializers.variance_scaling_initializer( ), scope=dnn_logits_scope, reuse=reused) tf_summary.add_hidden_layer_summary(dnn_logits, logits_scope_name) tf_summary.add_net_abs_mean_summary(dnn_logits, logits_scope_name) return dnn_logits
def resnet_arg_scope(is_training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): """Defines the default ResNet arg scope. TODO(gpapan): The batch-normalization related default values above are appropriate for use in conjunction with the reference ResNet models released at https://github.com/KaimingHe/deep-residual-networks. When training ResNets from scratch, they might need to be tuned. Args: is_training: Whether or not we are training the parameters in the batch normalization layers of the model. weight_decay: The weight decay to use for regularizing the model. batch_norm_decay: The moving average decay when estimating layer activation statistics in batch normalization. batch_norm_epsilon: Small constant to prevent division by zero when normalizing activations by their variance in batch normalization. batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the activations in the batch normalization layer. Returns: An `arg_scope` to use for the resnet models. """ batch_norm_params = { 'is_training': is_training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': ops.GraphKeys.UPDATE_OPS, } with arg_scope( [layers_lib.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params): # The following implies padding='SAME' for pool1, which makes feature # alignment easier for dense prediction tasks. This is also used in # https://github.com/facebook/fb.resnet.torch. However the accompanying # code of 'Deep Residual Learning for Image Recognition' uses # padding='VALID' for pool1. You can switch to that choice by setting # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID'). with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc: return arg_sc
def inception_v3_arg_scope(weight_decay=0.00004, batch_norm_var_collection='moving_vars', batch_norm_decay=0.9997, batch_norm_epsilon=0.001, updates_collections=ops.GraphKeys.UPDATE_OPS, use_fused_batchnorm=True): """Defines the default InceptionV3 arg scope. Args: weight_decay: The weight decay to use for regularizing the model. batch_norm_var_collection: The name of the collection for the batch norm variables. batch_norm_decay: Decay for batch norm moving average batch_norm_epsilon: Small float added to variance to avoid division by zero updates_collections: Collections for the update ops of the layer use_fused_batchnorm: Enable fused batchnorm. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': batch_norm_decay, # epsilon to prevent 0s in variance. 'epsilon': batch_norm_epsilon, # collection containing update_ops. 'updates_collections': updates_collections, # Use fused batch norm if possible. 'fused': use_fused_batchnorm, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers_lib.batch_norm, normalizer_params=batch_norm_params) as sc: return sc
def inception_v3_arg_scope(weight_decay=0.00004, batch_norm_var_collection='moving_vars', batch_norm_decay=0.9997, batch_norm_epsilon=0.001, updates_collections=ops.GraphKeys.UPDATE_OPS, use_fused_batchnorm=True): """Defines the default InceptionV3 arg scope. Args: weight_decay: The weight decay to use for regularizing the model. batch_norm_var_collection: The name of the collection for the batch norm variables. batch_norm_decay: Decay for batch norm moving average batch_norm_epsilon: Small float added to variance to avoid division by zero updates_collections: Collections for the update ops of the layer use_fused_batchnorm: Enable fused batchnorm. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': batch_norm_decay, # epsilon to prevent 0s in variance. 'epsilon': batch_norm_epsilon, # collection containing update_ops. 'updates_collections': updates_collections, # Use fused batch norm if possible. 'fused': use_fused_batchnorm, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers_lib.batch_norm, normalizer_params=batch_norm_params) as sc: return sc
def conv_layer(input, output_dep, filter_size, is_training=True, data_format='channels_first'): input = tf.layers.conv2d( input, 256, 1, kernel_initializer=initializers.variance_scaling_initializer( factor=2.0, mode='FAN_IN', uniform=False), data_format=data_format) input = batch_norm(input, training=is_training, data_format=data_format) input = tf.nn.relu(input) return input
def inception_v1_arg_scope(weight_decay=0.00004, use_batch_norm=True, batch_norm_var_collection='moving_vars'): """Defines the default InceptionV1 arg scope. Note: Althougth the original paper didn't use batch_norm we found it useful. Args: weight_decay: The weight decay to use for regularizing the model. use_batch_norm: "If `True`, batch_norm is applied after each convolution. batch_norm_var_collection: The name of the collection for the batch norm variables. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': 0.9997, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # collection containing update_ops. 'updates_collections': ops.GraphKeys.UPDATE_OPS, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } if use_batch_norm: normalizer_fn = layers_lib.batch_norm normalizer_params = batch_norm_params else: normalizer_fn = None normalizer_params = {} # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=normalizer_fn, normalizer_params=normalizer_params) as sc: return sc
def atlas_decode(siggrid, gridn, is_training, scope="", reuse=False, bottleneck_size=1024, weight_decay=1e-4, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'is_training': is_training, 'updates_collections': ops.GraphKeys.UPDATE_OPS } ix = tf.reshape( siggrid, [tf.shape(siggrid)[0], 1, tf.shape(siggrid)[1], siggrid.shape[-1]]) oy = [] with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=tf.nn.relu, normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params, reuse=reuse): with arg_scope([slim.batch_norm], **batch_norm_params): for i in range(gridn): x = slim.conv2d(ix, bottleneck_size, [1, 1], scope=scope + "_grid%d_fc1" % i) x = slim.conv2d(x, bottleneck_size // 2, [1, 1], scope=scope + "_grid%d_fc2" % i) x = slim.conv2d(x, bottleneck_size // 4, [1, 1], scope=scope + "_grid%d_fc3" % i) oy.append( slim.conv2d(x, 3, [1, 1], activation_fn=tf.nn.tanh, scope=scope + "_grid%d_fc4" % i)) y = tf.concat(oy, axis=2) return tf.reshape(y, [tf.shape(siggrid)[0], -1, 3])
def inception_v1_arg_scope(weight_decay=0.00004, use_batch_norm=True, batch_norm_var_collection='moving_vars'): """Defines the default InceptionV1 arg scope. Note: Althougth the original paper didn't use batch_norm we found it useful. Args: weight_decay: The weight decay to use for regularizing the model. use_batch_norm: "If `True`, batch_norm is applied after each convolution. batch_norm_var_collection: The name of the collection for the batch norm variables. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': 0.9997, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # collection containing update_ops. 'updates_collections': ops.GraphKeys.UPDATE_OPS, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } if use_batch_norm: normalizer_fn = layers_lib.batch_norm normalizer_params = batch_norm_params else: normalizer_fn = None normalizer_params = {} # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=normalizer_fn, normalizer_params=normalizer_params) as sc: return sc
def kconv(x, k, knn_index, d, scope, is_training, reuse): weight_decay = 1e-4 batch_norm_decay = 0.997 batch_norm_epsilon = 1e-5 batch_norm_scale = True batch_norm_params = { 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'is_training': is_training, 'updates_collections': ops.GraphKeys.UPDATE_OPS } with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=tf.nn.relu, normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params, reuse=reuse): with arg_scope([slim.batch_norm], **batch_norm_params): if int(x.shape[-1]) > 256: # too large to fit in gpu x = tf.reshape( x, [tf.shape(x)[0], tf.shape(x)[1], 1, int(x.shape[2])]) x = slim.conv2d(x, 256, [1, 1], scope=scope + '_reducedim') rx = tf.reshape(x, [tf.shape(x)[0], tf.shape(x)[1], int(x.shape[-1])]) xknn = tf.gather_nd(rx, knn_index, name=scope + '_gather') x = tf.concat([ tf.reshape( x, [tf.shape(x)[0], tf.shape(x)[1], 1, int(x.shape[-1])]), xknn ], axis=2) x = slim.conv2d(x, d, [1, k + 1], scope=scope + "_kconv", padding='VALID') return tf.reshape(x, [tf.shape(x)[0], -1, d])
def mlp(sig, grid, siggrid, param=None): scope = param[0] grid_num = param[1] is_training = param[2] reuse = param[3] weight_decay = 1e-4 batch_norm_decay = 0.997 batch_norm_epsilon = 1e-5 batch_norm_scale = True batch_norm_params = { 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'is_training': is_training, 'updates_collections': ops.GraphKeys.UPDATE_OPS } ix = tf.reshape( siggrid, [tf.shape(siggrid)[0], 1, tf.shape(siggrid)[1], siggrid.shape[-1]]) oy = [] with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=tf.nn.relu, normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params, reuse=reuse): with arg_scope([slim.batch_norm], **batch_norm_params): for i in range(grid_num): x = ix for j, size in enumerate(param[-1]): x = slim.conv2d(x, size, [1, 1], scope=scope + "_grid%d_fc%d" % (i, j)) oy.append( slim.conv2d(x, 3, [1, 1], activation_fn=tf.nn.tanh, scope=scope + "_grid%d_fc%d" % (i, j + 1))) y = tf.concat(oy, axis=2) return tf.reshape(y, [tf.shape(siggrid)[0], -1, 3])
def resnet_arg_scope( is_training=True, weight_decay=cfg.weight_decay, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'is_training': False, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'trainable': cfg.bn_training, 'updates_collections': ops.GraphKeys.UPDATE_OPS } with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), trainable=is_training, activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc: return arg_sc
def resnet_arg_scope(is_training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=0.00001, batch_norm_scale=True): """Defines the default ResNet arg scope. Args: is_training: Whether or not we are training the parameters in the batch normalization layers of the model. (deprecated) weight_decay: The weight decay to use for regularizing the model. batch_norm_decay: The moving average decay when estimating layer activation statistics in batch normalization. batch_norm_epsilon: Small constant to prevent division by zero when normalizing activations by their variance in batch normalization. batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the activations in the batch normalization layer. Returns: An `arg_scope` to use for the resnet models. """ batch_norm_params = { 'is_training': is_training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': tf.GraphKeys.UPDATE_OPS #'updates_collections': None, #'renorm': True } with arg_scope( [layers_lib.conv3d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm): with arg_scope([layers.batch_norm], **batch_norm_params): with arg_scope([layers.max_pool3d], padding='SAME') as arg_sc: return arg_sc
def inception_v2_arg_scope(weight_decay=0.00004, batch_norm_var_collection='moving_vars'): """Defines the default InceptionV2 arg scope. Args: weight_decay: The weight decay to use for regularizing the model. batch_norm_var_collection: The name of the collection for the batch norm variables. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': 0.9997, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # collection containing update_ops. 'updates_collections': ops.GraphKeys.UPDATE_OPS, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers_lib.batch_norm, normalizer_params=batch_norm_params) as sc: return sc
def resnet_arg_scope(weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): """Defines the default ResNet arg scope. Args: weight_decay: The weight decay to use for regularizing the model. batch_norm_decay: The moving average decay when estimating layer activation statistics in batch normalization. batch_norm_epsilon: Small constant to prevent division by zero when normalizing activations by their variance in batch normalization. batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the activations in the batch normalization layer. Returns: An `arg_scope` to use for the resnet models. """ batch_norm_params = { 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': tf.GraphKeys.UPDATE_OPS, } with arg_scope( [layers_lib.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=tf.nn.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params): # The following implies padding='SAME' for pool1, which makes feature # alignment easier for dense prediction tasks. This is also used in # https://github.com/facebook/fb.resnet.torch. However the accompanying # code of 'Deep Residual Learning for Image Recognition' uses # padding='VALID' for pool1. You can switch to that choice by setting # tf.contrib.framework.arg_scope([layers_lib.max_pool2d], padding='VALID'). with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc: return arg_sc
def inception_v2_arg_scope(weight_decay=0.00004, batch_norm_var_collection='moving_vars'): """Defines the default InceptionV2 arg scope. Args: weight_decay: The weight decay to use for regularizing the model. batch_norm_var_collection: The name of the collection for the batch norm variables. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': 0.9997, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # collection containing update_ops. 'updates_collections': ops.GraphKeys.UPDATE_OPS, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers_lib.batch_norm, normalizer_params=batch_norm_params) as sc: return sc
def predictron_arg_scope(weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': tf.GraphKeys.UPDATE_OPS, } # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=None, normalizer_fn=layers_lib.batch_norm, normalizer_params=batch_norm_params) as sc: return sc
def Encoder_fc3_dropout(x, num_output=85, is_training=True, reuse=False, name="3D_module"): """ 3D inference module. 3 MLP layers (last is the output) With dropout on first 2. Input: - x: N x [|img_feat|, |3D_param|] - reuse: bool Outputs: - 3D params: N x num_output if orthogonal: either 85: (3 + 24*3 + 10) or 109 (3 + 24*4 + 10) for factored axis-angle representation if perspective: 86: (f, tx, ty, tz) + 24*3 + 10, or 110 for factored axis-angle. - variables: tf variables """ if reuse: print('Reuse is on!') with tf.variable_scope(name, reuse=reuse) as scope: net = slim.fully_connected(x, 1024, scope='fc1') net = slim.dropout(net, 0.5, is_training=is_training, scope='dropout1') net = slim.fully_connected(net, 1024, scope='fc2') net = slim.dropout(net, 0.5, is_training=is_training, scope='dropout2') small_xavier = variance_scaling_initializer( factor=.01, mode='FAN_AVG', uniform=True) net = slim.fully_connected( net, num_output, activation_fn=None, weights_initializer=small_xavier, scope='fc3') variables = tf.contrib.framework.get_variables(scope) return net, variables
def resnet_arg_scope(weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True, is_training=True): batch_norm_params = { 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': ops.GraphKeys.UPDATE_OPS, 'is_training': is_training } with arg_scope( [layers_lib.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params): with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc: return arg_sc
def model_deconv(inputs,data_format='channels_first',is_training=True): model_class = ImagenetModel( resnet_size=50, data_format=data_format,num_classes=0,resnet_version=1) _, end_points = model_class(inputs, is_training) f = [end_points['block_layer4'], end_points['block_layer3'], end_points['block_layer2'], end_points['block_layer1']] P = [None, None, None, None] # attent=[None,None,None,None] # import pudb; pudb.set_trace() with tf.variable_scope('u_net'): def deconv(input,channels,factor=2): input=conv_layer(input,channels,1) input=conv_layer(input,channels,3) return unpool(input,factor,data_format=data_format) out4=tf.concat((deconv(f[0],1024),f[1]),1) out3=tf.concat((deconv(out4,512),f[2]),1) out2=tf.concat((deconv(out3,256),f[3]),1) out1=deconv(out2,256,factor=1) out1 = tf.layers.conv2d(out1, 1, 3, padding='same', kernel_initializer=initializers.variance_scaling_initializer( factor=2.0, mode='FAN_IN', uniform=False), data_format=data_format) out1=batch_norm(out1,training=is_training,data_format=data_format) out1=tf.nn.relu(out1) for i in range(config['n']): seg_map=tf.layers.conv2d(out1,1,1,data_format=data_format) seg_map = tf.sigmoid(unpool(seg_map, 4,data_format=data_format)) if i == 0: seg_maps = seg_map else: seg_maps = tf.concat((seg_maps, seg_map), 1 if data_format == 'channels_first' else -1) with tf.name_scope('format_change'): seg_maps=tf.transpose(seg_maps,[0,2,3,1]) return seg_maps, f
def resnet_pre_arg_scope(#is_training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { #'is_training': is_training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, # 'activation_fn': nn_ops.relu, 'variables_collections': ["batch_norm_params"], 'updates_collections': tf.GraphKeys.UPDATE_OPS } with arg_scope( [layers_lib.conv3d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=None): with arg_scope([layers.batch_norm], **batch_norm_params): with arg_scope([layers.max_pool3d], padding='SAME') as arg_sc: return arg_sc
def resnet_arg_scope(is_training=False, weight_decay=cfg.TRAIN.WEIGHT_DECAY, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'is_training': False, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'trainable': cfg.RESNET.BN_TRAIN, 'updates_collections': ops.GraphKeys.UPDATE_OPS } with arg_scope( [slim.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), trainable=is_training, activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc: return arg_sc
import tensorflow as tf # from tensorflow.contrib import learn from tensorflow.contrib.layers import convolution2d from tensorflow.contrib.framework.python.ops import variables from tensorflow.contrib.layers.python.layers import utils from tensorflow.contrib.layers.python.layers import initializers # method 1: contirb API net = convolution2d(net, num_outputs=num_ker, kernel_size=[ker_size,1], stride=[stride,1] padding='SAME', activation_fn=None, normalizer_fn=None, weights_initializer=initializers.variance_scaling_initializer(), biases_initializer=init_ops.zeros_initializer,) # method 2: original conv2d API weights = variables.model_variable( 'weights', shape=[ker_size, 1, num_ker_in, num_ker], dtype=dtype, initializer=initializers.variance_scaling_initializer(), trainable=True) biases = variables.model_variable( 'biases', shape=[num_ker], dtype=dtype, initializer=tf.zeros_initializer, trainable=True)