def testVariableCollectionsWithArgScopeNested(self): with self.test_session(): with arg_scope([variables_lib2.variable], collections='A'): a = variables_lib2.variable('a', []) with arg_scope([variables_lib2.variable], collections='B'): b = variables_lib2.variable('b', []) self.assertEquals(a, ops.get_collection('A')[0]) self.assertEquals(b, ops.get_collection('B')[0])
def testVariableCollectionsWithArgScopeNonNested(self): with self.test_session(): with arg_scope([variables_lib2.variable], collections='A'): a = variables_lib2.variable('a', []) with arg_scope([variables_lib2.variable], collections='B'): b = variables_lib2.variable('b', []) variables_lib2.variable('c', []) self.assertListEqual([a], ops.get_collection('A')) self.assertListEqual([b], ops.get_collection('B'))
def alexnet_v2_arg_scope(weight_decay=0.0005): with arg_scope( [layers.conv2d, layers_lib.fully_connected], activation_fn=nn_ops.relu, biases_initializer=init_ops.constant_initializer(0.1), weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope([layers.conv2d], padding='SAME'): with arg_scope([layers_lib.max_pool2d], padding='VALID') as arg_sc: return arg_sc
def testReuseArgScope(self): func1_kwargs = {'a': 1, 'b': None, 'c': [1]} key_op = _key_op(func1) current_scope = {key_op: func1_kwargs.copy()} with self.test_session(): with arg_scope([func1], a=1, b=None, c=[1]) as scope1: pass with arg_scope(scope1) as scope: self.assertDictEqual(scope, current_scope)
def testArgScopeObjectCreatedWithinScopeInheritsArgScope(self): def get_scope_object(): with arg_scope([func1], a=1, b=None, c=[1]) as sc: return sc with arg_scope([func1], b=2, d=10): with arg_scope(get_scope_object()): args, kwargs = func1(0) self.assertTupleEqual(args, (0,)) self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1], 'd': 10})
def testClearArgScope(self): func1_kwargs = {'a': 1, 'b': None, 'c': [1]} key_op = _key_op(func1) func1_scope = {key_op: func1_kwargs.copy()} with self.test_session(): with arg_scope([func1], a=1, b=None, c=[1]) as sc1: self.assertEqual(sc1, func1_scope) with arg_scope({}) as sc2: self.assertEqual(sc2, {}) with arg_scope([]) as current_arg_scope: self.assertEqual(current_arg_scope, func1_scope)
def testCurrentArgScopeNested(self): func1_kwargs = {'a': 1, 'b': None, 'c': [1]} func2_kwargs = {'b': 2, 'd': [2]} key = _key_op current_scope = { key(func1): func1_kwargs.copy(), key(func2): func2_kwargs.copy() } with self.test_session(): with arg_scope([func1], a=1, b=None, c=[1]): with arg_scope([func2], b=2, d=[2]) as scope: self.assertDictEqual(scope, current_scope)
def testNestedArgScopeObjectCreatedOutsideScopeOverridesArgScope(self): def get_scope_object(): with arg_scope([func1], a=1, b=None, c=[1]) as sc: return sc scope_object = get_scope_object() with arg_scope([func1], b=2, d=10): with arg_scope(scope_object): args, kwargs = func1(0) self.assertTupleEqual(args, (0,)) self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1]})
def testNestedArgScope(self): func1_args = (0,) func1_kwargs = {'a': 1, 'b': None, 'c': [1]} with arg_scope([func1], a=1, b=None, c=[1]): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs) func1_kwargs['b'] = 2 with arg_scope([func1], b=2): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs)
def testPartiallySharedArgScope(self): func1_args = (0,) func1_kwargs = {'a': 1, 'b': None, 'c': [1]} func2_args = (1,) func2_kwargs = {'a': 1, 'b': None, 'd': [2]} with arg_scope([func1, func2], a=1, b=None): with arg_scope([func1], c=[1]): with arg_scope([func2], d=[2]): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs) args, kwargs = func2(1) self.assertTupleEqual(args, func2_args) self.assertDictEqual(kwargs, func2_kwargs)
def resnet_arg_scope(is_training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): """Defines the default ResNet arg scope. TODO(gpapan): The batch-normalization related default values above are appropriate for use in conjunction with the reference ResNet models released at https://github.com/KaimingHe/deep-residual-networks. When training ResNets from scratch, they might need to be tuned. Args: is_training: Whether or not we are training the parameters in the batch normalization layers of the model. weight_decay: The weight decay to use for regularizing the model. batch_norm_decay: The moving average decay when estimating layer activation statistics in batch normalization. batch_norm_epsilon: Small constant to prevent division by zero when normalizing activations by their variance in batch normalization. batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the activations in the batch normalization layer. Returns: An `arg_scope` to use for the resnet models. """ batch_norm_params = { 'is_training': is_training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': ops.GraphKeys.UPDATE_OPS, } with arg_scope( [layers_lib.conv2d], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params): with arg_scope([layers.batch_norm], **batch_norm_params): # The following implies padding='SAME' for pool1, which makes feature # alignment easier for dense prediction tasks. This is also used in # https://github.com/facebook/fb.resnet.torch. However the accompanying # code of 'Deep Residual Learning for Image Recognition' uses # padding='VALID' for pool1. You can switch to that choice by setting # tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID'). with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc: return arg_sc
def inception_v3_arg_scope(weight_decay=0.00004, batch_norm_var_collection='moving_vars', batch_norm_decay=0.9997, batch_norm_epsilon=0.001, updates_collections=ops.GraphKeys.UPDATE_OPS, use_fused_batchnorm=True): """Defines the default InceptionV3 arg scope. Args: weight_decay: The weight decay to use for regularizing the model. batch_norm_var_collection: The name of the collection for the batch norm variables. batch_norm_decay: Decay for batch norm moving average batch_norm_epsilon: Small float added to variance to avoid division by zero updates_collections: Collections for the update ops of the layer use_fused_batchnorm: Enable fused batchnorm. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': batch_norm_decay, # epsilon to prevent 0s in variance. 'epsilon': batch_norm_epsilon, # collection containing update_ops. 'updates_collections': updates_collections, # Use fused batch norm if possible. 'fused': use_fused_batchnorm, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers_lib.batch_norm, normalizer_params=batch_norm_params) as sc: return sc
def inception_v1_arg_scope(weight_decay=0.00004, use_batch_norm=True, batch_norm_var_collection='moving_vars'): """Defines the default InceptionV1 arg scope. Note: Althougth the original paper didn't use batch_norm we found it useful. Args: weight_decay: The weight decay to use for regularizing the model. use_batch_norm: "If `True`, batch_norm is applied after each convolution. batch_norm_var_collection: The name of the collection for the batch norm variables. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': 0.9997, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # collection containing update_ops. 'updates_collections': ops.GraphKeys.UPDATE_OPS, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } if use_batch_norm: normalizer_fn = layers_lib.batch_norm normalizer_params = batch_norm_params else: normalizer_fn = None normalizer_params = {} # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=normalizer_fn, normalizer_params=normalizer_params) as sc: return sc
def vgg_a(inputs, num_classes=1000, is_training=True, dropout_keep_prob=0.5, spatial_squeeze=True, scope='vgg_a'): """Oxford Net VGG 11-Layers version A Example. Note: All the fully_connected layers have been transformed to conv2d layers. To use in classification mode, resize input to 224x224. Args: inputs: a tensor of size [batch_size, height, width, channels]. num_classes: number of predicted classes. is_training: whether or not the model is being trained. dropout_keep_prob: the probability that activations are kept in the dropout layers during training. spatial_squeeze: whether or not should squeeze the spatial dimensions of the outputs. Useful to remove unnecessary dimensions for classification. scope: Optional scope for the variables. Returns: the last op containing the log predictions and end_points dict. """ with variable_scope.variable_scope(scope, 'vgg_a', [inputs]) as sc: end_points_collection = sc.original_name_scope + '_end_points' # Collect outputs for conv2d, fully_connected and max_pool2d. with arg_scope( [layers.conv2d, layers_lib.max_pool2d], outputs_collections=end_points_collection): net = layers_lib.repeat( inputs, 1, layers.conv2d, 64, [3, 3], scope='conv1') net = layers_lib.max_pool2d(net, [2, 2], scope='pool1') net = layers_lib.repeat(net, 1, layers.conv2d, 128, [3, 3], scope='conv2') net = layers_lib.max_pool2d(net, [2, 2], scope='pool2') net = layers_lib.repeat(net, 2, layers.conv2d, 256, [3, 3], scope='conv3') net = layers_lib.max_pool2d(net, [2, 2], scope='pool3') net = layers_lib.repeat(net, 2, layers.conv2d, 512, [3, 3], scope='conv4') net = layers_lib.max_pool2d(net, [2, 2], scope='pool4') net = layers_lib.repeat(net, 2, layers.conv2d, 512, [3, 3], scope='conv5') net = layers_lib.max_pool2d(net, [2, 2], scope='pool5') # Use conv2d instead of fully_connected layers. net = layers.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6') net = layers_lib.dropout( net, dropout_keep_prob, is_training=is_training, scope='dropout6') net = layers.conv2d(net, 4096, [1, 1], scope='fc7') net = layers_lib.dropout( net, dropout_keep_prob, is_training=is_training, scope='dropout7') net = layers.conv2d( net, num_classes, [1, 1], activation_fn=None, normalizer_fn=None, scope='fc8') # Convert end_points_collection into a end_point dict. end_points = utils.convert_collection_to_dict(end_points_collection) if spatial_squeeze: net = array_ops.squeeze(net, [1, 2], name='fc8/squeezed') end_points[sc.name + '/fc8'] = net return net, end_points
def testEndPointsV2(self): """Test the end points of a tiny v2 bottleneck network.""" blocks = [ resnet_v2.resnet_v2_block( 'block1', base_depth=1, num_units=2, stride=2), resnet_v2.resnet_v2_block( 'block2', base_depth=2, num_units=2, stride=1), ] inputs = create_test_input(2, 32, 16, 3) with arg_scope(resnet_utils.resnet_arg_scope()): _, end_points = self._resnet_plain(inputs, blocks, scope='tiny') expected = [ 'tiny/block1/unit_1/bottleneck_v2/shortcut', 'tiny/block1/unit_1/bottleneck_v2/conv1', 'tiny/block1/unit_1/bottleneck_v2/conv2', 'tiny/block1/unit_1/bottleneck_v2/conv3', 'tiny/block1/unit_2/bottleneck_v2/conv1', 'tiny/block1/unit_2/bottleneck_v2/conv2', 'tiny/block1/unit_2/bottleneck_v2/conv3', 'tiny/block2/unit_1/bottleneck_v2/shortcut', 'tiny/block2/unit_1/bottleneck_v2/conv1', 'tiny/block2/unit_1/bottleneck_v2/conv2', 'tiny/block2/unit_1/bottleneck_v2/conv3', 'tiny/block2/unit_2/bottleneck_v2/conv1', 'tiny/block2/unit_2/bottleneck_v2/conv2', 'tiny/block2/unit_2/bottleneck_v2/conv3' ] self.assertItemsEqual(expected, end_points)
def testAtrousFullyConvolutionalValues(self): """Verify dense feature extraction with atrous convolution.""" nominal_stride = 32 for output_stride in [4, 8, 16, 32, None]: with arg_scope(resnet_utils.resnet_arg_scope()): with ops.Graph().as_default(): with self.test_session() as sess: random_seed.set_random_seed(0) inputs = create_test_input(2, 81, 81, 3) # Dense feature extraction followed by subsampling. output, _ = self._resnet_small( inputs, None, is_training=False, global_pool=False, output_stride=output_stride) if output_stride is None: factor = 1 else: factor = nominal_stride // output_stride output = resnet_utils.subsample(output, factor) # Make the two networks use the same weights. variable_scope.get_variable_scope().reuse_variables() # Feature extraction at the nominal network rate. expected, _ = self._resnet_small( inputs, None, is_training=False, global_pool=False) sess.run(variables.global_variables_initializer()) self.assertAllClose( output.eval(), expected.eval(), atol=1e-4, rtol=1e-4)
def grad_fn(inputs, variables, outputs, output_grads): """Recompute outputs for gradient computation.""" del outputs # Recompute outputs with framework_ops.control_dependencies(output_grads): if use_data_dep_: inputs = _force_data_dependency(output_grads, inputs) with contrib_framework_ops.arg_scope(cached_arg_scope[0]): with variable_scope.variable_scope(cached_vs[0], reuse=True): outputs = fn(*inputs) if not (isinstance(outputs, list) or isinstance(outputs, tuple)): outputs = [outputs] outputs = list(outputs) grads = gradients_impl.gradients(outputs, inputs + variables, output_grads) if tupleize_grads: if use_data_dep_: grads = _tuple_with_data_dep(grads) else: grads = control_flow_ops.tuple(grads) grad_inputs = grads[:len(inputs)] grad_vars = grads[len(inputs):] return grad_inputs, grad_vars
def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None): """A plain ResNet without extra layers before or after the ResNet blocks.""" with variable_scope.variable_scope(scope, values=[inputs]): with arg_scope([layers.conv2d], outputs_collections='end_points'): net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride) end_points = utils.convert_collection_to_dict('end_points') return net, end_points
def slim_net_original(image, keep_prob): with arg_scope([layers.conv2d, layers.fully_connected], biases_initializer=tf.random_normal_initializer(stddev=0.1)): # conv2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME', # activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, # weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, # biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, scope=None): net = layers.conv2d(image, 32, [5, 5], scope='conv1', weights_regularizer=regularizers.l1_regularizer(0.5)) # max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None) net = layers.max_pool2d(net, 2, scope='pool1') net = layers.conv2d(net, 64, [5, 5], scope='conv2', weights_regularizer=regularizers.l2_regularizer(0.5)) summaries.summarize_tensor(net, tag='conv2') net = layers.max_pool2d(net, 2, scope='pool2') net = layers.flatten(net, scope='flatten1') # fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None, # normalizer_params=None, weights_initializer=initializers.xavier_initializer(), # weights_regularizer=None, biases_initializer=init_ops.zeros_initializer, # biases_regularizer=None, scope=None): net = layers.fully_connected(net, 1024, scope='fc1') # dropout(inputs, keep_prob=0.5, is_training=True, scope=None) net = layers.dropout(net, keep_prob=keep_prob, scope='dropout1') net = layers.fully_connected(net, 10, scope='fc2') return net
def testVariableWithDeviceFunction(self): class DevFn(object): def __init__(self): self.counter = -1 def __call__(self, op): self.counter += 1 return 'cpu:%d' % self.counter with self.test_session(): with arg_scope([variables_lib2.variable], device=DevFn()): a = variables_lib2.variable('a', []) b = variables_lib2.variable('b', []) c = variables_lib2.variable('c', [], device='cpu:12') d = variables_lib2.variable('d', []) with ops.device('cpu:99'): e_init = constant_op.constant(12) e = variables_lib2.variable('e', initializer=e_init) self.assertDeviceEqual(a.device, 'cpu:0') self.assertEqual(a.initial_value.op.colocation_groups(), a.op.colocation_groups()) self.assertDeviceEqual(b.device, 'cpu:1') self.assertEqual(b.initial_value.op.colocation_groups(), b.op.colocation_groups()) self.assertDeviceEqual(c.device, 'cpu:12') self.assertEqual(c.initial_value.op.colocation_groups(), c.op.colocation_groups()) self.assertDeviceEqual(d.device, 'cpu:2') self.assertEqual(d.initial_value.op.colocation_groups(), d.op.colocation_groups()) self.assertDeviceEqual(e.device, 'cpu:3') self.assertDeviceEqual(e.initial_value.device, 'cpu:99')
def vgg_arg_scope(weight_decay=0.0005): """Defines the VGG arg scope. Args: weight_decay: The l2 regularization coefficient. Returns: An arg_scope. """ with arg_scope( [layers.conv2d, layers_lib.fully_connected], activation_fn=nn_ops.relu, weights_regularizer=regularizers.l2_regularizer(weight_decay), biases_initializer=init_ops.zeros_initializer()): with arg_scope([layers.conv2d], padding='SAME') as arg_sc: return arg_sc
def testVariableGPUPlacement(self): with ops.Graph().as_default(): device_fn = variables_lib2.VariableDeviceChooser(device_type='GPU') with arg_scope([variables_lib2.variable], device=device_fn): a = variables_lib2.variable('a', []) b = variables_lib2.variable('b', []) c = variables_lib2.variable('c', [], device='cpu:12') d = variables_lib2.variable('d', []) with ops.device('cpu:99'): e_init = constant_op.constant(12) e = variables_lib2.variable('e', initializer=e_init) # The values below highlight how the VariableDeviceChooser puts initial # values on the same device as the variable job. self.assertDeviceEqual(a.device, '/gpu:0') self.assertEqual(a.initial_value.op.colocation_groups(), a.op.colocation_groups()) self.assertDeviceEqual(b.device, '/gpu:0') self.assertEqual(b.initial_value.op.colocation_groups(), b.op.colocation_groups()) self.assertDeviceEqual(c.device, '/cpu:12') self.assertEqual(c.initial_value.op.colocation_groups(), c.op.colocation_groups()) self.assertDeviceEqual(d.device, '/gpu:0') self.assertEqual(d.initial_value.op.colocation_groups(), d.op.colocation_groups()) self.assertDeviceEqual(e.device, '/gpu:0') self.assertDeviceEqual(e.initial_value.device, '/cpu:99')
def __init__(self, sess, dim=299): self.batch_shape = [16, dim, dim, 3] self._num_classes = 1001 self._scope = 'InceptionV3' self._weights_file = './Weights/inception_v3.ckpt' output_layer = 'Conv2d_1a_3x3' # # network inputs # self.x_tf = tf.placeholder(tf.float32, shape=self.batch_shape) # # network outputs # with slim.arg_scope(inception.inception_v3_arg_scope()): with arg_scope([layers_lib.batch_norm, layers_lib.dropout], is_training=False): # we truncate before these layers, so this is not really necessary... net, endpoints = inception.inception_v3_base(self.x_tf, final_endpoint=output_layer, scope=self._scope) self.output = endpoints[output_layer] # # load weights # saver = tf.train.Saver(slim.get_model_variables(scope=self._scope)) saver.restore(sess, self._weights_file)
def testOverwriteArgScope(self): func1_args = (0,) func1_kwargs = {'a': 1, 'b': 2, 'c': [1]} with arg_scope([func1], a=1, b=None, c=[1]): args, kwargs = func1(0, b=2) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs)
def testEndPointsV2(self): """Test the end points of a tiny v2 bottleneck network.""" bottleneck = resnet_v2.bottleneck blocks = [ resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]), resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 1)]) ] inputs = create_test_input(2, 32, 16, 3) with arg_scope(resnet_utils.resnet_arg_scope()): _, end_points = self._resnet_plain(inputs, blocks, scope='tiny') expected = [ 'tiny/block1/unit_1/bottleneck_v2/shortcut', 'tiny/block1/unit_1/bottleneck_v2/conv1', 'tiny/block1/unit_1/bottleneck_v2/conv2', 'tiny/block1/unit_1/bottleneck_v2/conv3', 'tiny/block1/unit_2/bottleneck_v2/conv1', 'tiny/block1/unit_2/bottleneck_v2/conv2', 'tiny/block1/unit_2/bottleneck_v2/conv3', 'tiny/block2/unit_1/bottleneck_v2/shortcut', 'tiny/block2/unit_1/bottleneck_v2/conv1', 'tiny/block2/unit_1/bottleneck_v2/conv2', 'tiny/block2/unit_1/bottleneck_v2/conv3', 'tiny/block2/unit_2/bottleneck_v2/conv1', 'tiny/block2/unit_2/bottleneck_v2/conv2', 'tiny/block2/unit_2/bottleneck_v2/conv3' ] self.assertItemsEqual(expected, end_points)
def _forward(self, x, gpu): hps = self.hps x = tf.to_float(x) x = tf.clip_by_value((x + 0.5) / 256.0, 0.0, 1.0) - 0.5 # Input images are repeated k times on the input. # This is used for Importance Sampling loss (k is number of samples). data_size = hps.batch_size * hps.k x = repeat(x, hps.k) orig_x = x h_size = hps.h_size with arg_scope([conv2d, deconv2d], init=(self.mode == "init")): layers = [] for i in range(hps.depth): layers.append([]) for j in range(hps.num_blocks): downsample = (i > 0) and (j == 0) layers[-1].append(IAFLayer(hps, self.mode, downsample)) h = conv2d("x_enc", x, h_size, [5, 5], [2, 2]) # -> [16, 16] for i, layer in enumerate(layers): for j, sub_layer in enumerate(layer): with tf.variable_scope("IAF_%d_%d" % (i, j)): h = sub_layer.up(h) # top->down self.h_top = h_top = tf.get_variable("h_top", [h_size], initializer=tf.zeros_initializer) h_top = tf.reshape(h_top, [1, -1, 1, 1]) h = tf.tile(h_top, [data_size, 1, hps.image_size / 2 ** len(layers), hps.image_size / 2 ** len(layers)]) kl_cost = kl_obj = 0.0 for i, layer in reversed(list(enumerate(layers))): for j, sub_layer in reversed(list(enumerate(layer))): with tf.variable_scope("IAF_%d_%d" % (i, j)): h, cur_obj, cur_cost = sub_layer.down(h) kl_obj += cur_obj kl_cost += cur_cost if self.mode == "train" and gpu == hps.num_gpus - 1: tf.scalar_summary("model/kl_obj_%02d_%02d" % (i, j), tf.reduce_mean(cur_obj)) tf.scalar_summary("model/kl_cost_%02d_%02d" % (i, j), tf.reduce_mean(cur_cost)) x = tf.nn.elu(h) x = deconv2d("x_dec", x, 3, [5, 5]) x = tf.clip_by_value(x, -0.5 + 1 / 512., 0.5 - 1 / 512.) log_pxz = discretized_logistic(x, self.dec_log_stdv, sample=orig_x) obj = tf.reduce_sum(kl_obj - log_pxz) if self.mode == "train" and gpu == hps.num_gpus - 1: tf.scalar_summary("model/log_pxz", -tf.reduce_mean(log_pxz)) tf.scalar_summary("model/kl_obj", tf.reduce_mean(kl_obj)) tf.scalar_summary("model/kl_cost", tf.reduce_mean(kl_cost)) loss = tf.reduce_sum(compute_lowerbound(log_pxz, kl_cost, hps.k)) return x, obj, loss
def inception_v1(inputs, num_classes=1000, is_training=True, dropout_keep_prob=0.8, prediction_fn=layers_lib.softmax, spatial_squeeze=True, reuse=None, scope='InceptionV1'): """Defines the Inception V1 architecture. This architecture is defined in: Going deeper with convolutions Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich. http://arxiv.org/pdf/1409.4842v1.pdf. The default image size used to train this network is 224x224. Args: inputs: a tensor of size [batch_size, height, width, channels]. num_classes: number of predicted classes. is_training: whether is training or not. dropout_keep_prob: the percentage of activation values that are retained. prediction_fn: a function to get predictions out of logits. spatial_squeeze: if True, logits is of shape is [B, C], if false logits is of shape [B, 1, 1, C], where B is batch_size and C is number of classes. reuse: whether or not the network and its variables should be reused. To be able to reuse 'scope' must be given. scope: Optional variable_scope. Returns: logits: the pre-softmax activations, a tensor of size [batch_size, num_classes] end_points: a dictionary from components of the network to the corresponding activation. """ # Final pooling and prediction with variable_scope.variable_scope( scope, 'InceptionV1', [inputs, num_classes], reuse=reuse) as scope: with arg_scope( [layers_lib.batch_norm, layers_lib.dropout], is_training=is_training): net, end_points = inception_v1_base(inputs, scope=scope) with variable_scope.variable_scope('Logits'): net = layers_lib.avg_pool2d( net, [7, 7], stride=1, scope='MaxPool_0a_7x7') net = layers_lib.dropout(net, dropout_keep_prob, scope='Dropout_0b') logits = layers.conv2d( net, num_classes, [1, 1], activation_fn=None, normalizer_fn=None, scope='Conv2d_0c_1x1') if spatial_squeeze: logits = array_ops.squeeze(logits, [1, 2], name='SpatialSqueeze') end_points['Logits'] = logits end_points['Predictions'] = prediction_fn(logits, scope='Predictions') return logits, end_points
def testSimpleArgScopeWithTuple(self): func1_args = (0,) func1_kwargs = {'a': 1, 'b': None, 'c': [1]} with self.test_session(): with arg_scope((func1,), a=1, b=None, c=[1]): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs)
def inception_v3_arg_scope(weight_decay=0.00004, stddev=0.1, batch_norm_var_collection='moving_vars', use_fused_batchnorm=True): """Defines the default InceptionV3 arg scope. Args: weight_decay: The weight decay to use for regularizing the model. stddev: The standard deviation of the trunctated normal weight initializer. batch_norm_var_collection: The name of the collection for the batch norm variables. use_fused_batchnorm: Enable fused batchnorm. Returns: An `arg_scope` to use for the inception v3 model. """ batch_norm_params = { # Decay for the moving averages. 'decay': 0.9997, # epsilon to prevent 0s in variance. 'epsilon': 0.001, # collection containing update_ops. 'updates_collections': ops.GraphKeys.UPDATE_OPS, # Use fused batch norm if possible. 'fused': use_fused_batchnorm, # collection containing the moving mean and moving variance. 'variables_collections': { 'beta': None, 'gamma': None, 'moving_mean': [batch_norm_var_collection], 'moving_variance': [batch_norm_var_collection], } } # Set weight_decay for weights in Conv and FC layers. with arg_scope( [layers.conv2d, layers_lib.fully_connected], weights_regularizer=regularizers.l2_regularizer(weight_decay)): with arg_scope( [layers.conv2d], weights_initializer=init_ops.truncated_normal_initializer( stddev=stddev), activation_fn=nn_ops.relu, normalizer_fn=layers_lib.batch_norm, normalizer_params=batch_norm_params) as sc: return sc
def testNonDecorated(self): def my_func(t, a=None): return (t, a) with self.assertRaises(ValueError): with arg_scope([my_func], a=1): pass
def gated_resnet(x, a=None, gh=None, sh=None, nonlinearity=tf.nn.elu, conv=conv2d, dropout_p=0.0, counters={}, **kwargs): name = get_name("gated_resnet", counters) print("construct", name, "...") xs = int_shape(x) num_filters = xs[-1] with arg_scope([conv], **kwargs): c1 = conv(nonlinearity(x), num_filters) if a is not None: # add short-cut connection if auxiliary input 'a' is given c1 += nin(nonlinearity(a), num_filters) c1 = nonlinearity(c1) c1 = tf.nn.dropout(c1, keep_prob=1. - dropout_p) c2 = conv(c1, num_filters * 2) # add projection of h vector if included: conditional generation if sh is not None: c2 += nin(sh, 2*num_filters, nonlinearity=nonlinearity) if gh is not None: # haven't finished this part pass a, b = tf.split(c2, 2, 3) c3 = a * tf.nn.sigmoid(b) return x + c3
def slim_decoder(inputs, batch_norm_decay, weight_decay, is_training, feature_depth=256, output_depth=256): with tf.variable_scope('slim_decoder'): with arg_scope([depthwise_conv2d_layer, conv2d_layer], to_batch_norm=True, batch_norm_decay=batch_norm_decay, is_training=is_training, activation_fn=tf.nn.relu, weights_regularizer=slim.l2_regularizer(weight_decay)): net = depthwise_conv2d_layer(inputs, 3) net = conv2d_layer(net, feature_depth, 1) net = depthwise_conv2d_layer(net, 3) net = conv2d_layer(net, output_depth, 1) return net
def testRootlessFullyConvolutionalEndpointShapes(self): global_pool = False num_classes = 10 inputs = create_test_input(2, 128, 128, 3) with arg_scope(resnet_utils.resnet_arg_scope()): _, end_points = self._resnet_small(inputs, num_classes, global_pool=global_pool, include_root_block=False, scope='resnet') endpoint_to_shape = { 'resnet/block1': [2, 64, 64, 4], 'resnet/block2': [2, 32, 32, 8], 'resnet/block3': [2, 16, 16, 16], 'resnet/block4': [2, 16, 16, 32] } for endpoint in endpoint_to_shape: shape = endpoint_to_shape[endpoint] self.assertListEqual( end_points[endpoint].get_shape().as_list(), shape)
def DeepLabNet(input_batch, is_training, num_classes, output_stride = 16, batch_norm_decay = 0.9997, backbone = 'resnet_v2_101'): #Use channels_first to boost on GPU #Deeplab V3+ with resnet as backbone inputs_size = tf.shape(input_batch)[1:3] with tf.variable_scope('deeplab'): #ResNet as the encoder with tf.variable_scope('encoder'): if backbone == 'resnet_v2_50': base_model = resnet_v2.resnet_v2_50 else: base_model = resnet_v2.resnet_v2_101 # #Implement tensorflow resnetV2 with tf.contrib.slim.arg_scope(resnet_v2.resnet_arg_scope(batch_norm_decay=batch_norm_decay)): logits, end_points = base_model(input_batch, num_classes=None, is_training=is_training, global_pool=False, output_stride=output_stride) #ASPP in the middle layers with tf.variable_scope('aspp'): net = end_points['deeplab/encoder/' + backbone + '/block4'] encoder_output = ASPP(net, output_stride, batch_norm_decay, is_training) # #Decoder with tf.variable_scope('decoder'): with tf.contrib.slim.arg_scope(resnet_v2.resnet_arg_scope(batch_norm_decay = batch_norm_decay)): with arg_scope([layers.batch_norm], is_training=is_training): with tf.variable_scope("low_level_features"): low_level_features = end_points['deeplab/encoder/' + backbone + '/block1/unit_3/bottleneck_v2/conv1'] low_level_features = layers_lib.conv2d(low_level_features, 48, [1,1], stride = 1, scope = 'conv_1x1') low_level_features_size = tf.shape(low_level_features)[1:3] with tf.variable_scope("upsampling_logits"): net = tf.image.resize_bilinear(encoder_output, low_level_features_size, name = 'upsample_1') net = tf.concat([net, low_level_features], axis = 3, name = 'concat') net = layers_lib.conv2d(net, 256, [3,3], stride = 1, scope = 'conv_3x3_1') net = layers_lib.conv2d(net, 256, [3,3], stride = 1, scope = 'conv_3x3_2') net = layers_lib.conv2d(net, num_classes, [1, 1], activation_fn = None, normalizer_fn = None, scope = 'conv_1x1') logits = tf.image.resize_bilinear(net, inputs_size, name = 'upsample_2') # return logits
def _encoder(self, input_images, scope_name="encoder", trainable=True, scope_reuse=False): with arg_scope(resnet_utils.resnet_arg_scope()): output, end_points = resnet_v2.resnet_v2_50( input_images, output_stride=8, global_pool=False, reuse=scope_reuse) #(256, 256, 2048)==>(32, 32, 2048) hidden_state = decoder_layer( output, out_channels=32, stride=1, scope_name='encoder_layer1', trainable=trainable) #(32, 32, 2048)==>(32, 32, 32) print hidden_state.get_shape() tf.summary.histogram(hidden_state.op.name + "/activation", hidden_state) return hidden_state
def testAtrousFullyConvolutionalEndpointShapes(self): global_pool = False num_classes = 10 output_stride = 8 inputs = create_test_input(2, 321, 321, 3) with arg_scope(resnet_utils.resnet_arg_scope()): _, end_points = self._resnet_small( inputs, num_classes, global_pool=global_pool, output_stride=output_stride, scope='resnet') endpoint_to_shape = { 'resnet/block1': [2, 41, 41, 4], 'resnet/block2': [2, 41, 41, 8], 'resnet/block3': [2, 41, 41, 16], 'resnet/block4': [2, 41, 41, 32] } for endpoint in endpoint_to_shape: shape = endpoint_to_shape[endpoint] self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def compute_est_samples(z, params=None, reuse=tf.AUTO_REUSE): with tf.variable_scope("estimator"): with arg_scope([nn.dense], params=params): with tf.variable_scope("decoder", reuse=reuse): h_dec_1 = nn.dense(z, vae_z_dim, 200 * 2, "dense1", nonlinearity=nonlin) h_dec_2 = nn.dense(h_dec_1, 200 * 2, 500 * 2, "dense2", nonlinearity=nonlin) x_mean = nn.dense(h_dec_2, 500 * 2, x_dim, "dense3", nonlinearity=None) x_mean = tf.nn.tanh(x_mean) return x_mean
def actnorm( name, x, scale=1., logdet=None, logscale_factor=3., batch_variance=False, reverse=False, init=False, trainable=True): if arg_scope([get_variable_ddi], trainable=trainable): if not reverse: x = actnorm_center(name + "_center", x, reverse) x = actnorm_scale( name + "_scale", x, scale, logdet, logscale_factor, batch_variance, reverse, init) if logdet is not None: x, logdet = x else: x = actnorm_scale( name + "_scale", x, scale, logdet, logscale_factor, batch_variance, reverse, init) if logdet is not None: x, logdet = x x = actnorm_center(name + "_center", x, reverse) if logdet is not None: return x, logdet return x
def slim_net_original(image, keep_prob): with arg_scope( [layers.conv2d, layers.fully_connected], biases_initializer=tf.random_normal_initializer(stddev=0.1)): # conv2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME', # activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, # weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, # biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, scope=None): net = layers.conv2d( image, 32, [5, 5], scope='conv1', weights_regularizer=regularizers.l1_regularizer(0.5)) # max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None) net = layers.max_pool2d(net, 2, scope='pool1') net = layers.conv2d( net, 64, [5, 5], scope='conv2', weights_regularizer=regularizers.l2_regularizer(0.5)) summaries.summarize_tensor(net, tag='conv2') net = layers.max_pool2d(net, 2, scope='pool2') net = layers.flatten(net, scope='flatten1') # fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None, # normalizer_params=None, weights_initializer=initializers.xavier_initializer(), # weights_regularizer=None, biases_initializer=init_ops.zeros_initializer, # biases_regularizer=None, scope=None): net = layers.fully_connected(net, 1024, scope='fc1') # dropout(inputs, keep_prob=0.5, is_training=True, scope=None) net = layers.dropout(net, keep_prob=keep_prob, scope='dropout1') net = layers.fully_connected(net, 10, scope='fc2') return net
def _grad_fn(output_grads, variables=None): """Recompute outputs for gradient computation.""" variables = variables or [] if original_vars: assert variables, ( "Fn created variables but the variables were not " "passed to the gradient fn.") if set(variables) != original_vars: raise ValueError(_WRONG_VARS_ERR) inputs = [array_ops.identity(x) for x in list(args)] # Recompute outputs with framework_ops.control_dependencies(output_grads): if use_data_dep_: inputs = _force_data_dependency(output_grads, inputs) with contrib_framework_ops.arg_scope(arg_scope): with variable_scope.variable_scope(vs, reuse=True): with backprop.GradientTape() as tape: fn_kwargs = {} if has_is_recompute_kwarg: fn_kwargs["is_recomputing"] = True outputs = fn(*inputs, **fn_kwargs) recompute_vars = set(tape.watched_variables()) if original_vars != recompute_vars: raise ValueError(_WRONG_VARS_ERR) if not isinstance(outputs, (list, tuple)): outputs = [outputs] outputs = list(outputs) grads = gradients_impl.gradients(outputs, inputs + variables, output_grads) if tupleize_grads: if use_data_dep_: grads = _tuple_with_data_dep(grads) else: grads = control_flow_ops.tuple(grads) grad_inputs = grads[:len(inputs)] grad_vars = grads[len(inputs):] return grad_inputs, grad_vars
def _model(self, inputs): kernel_initializer = tf.contrib.layers.xavier_initializer() out = tf.reshape(inputs, (-1, 28, 28, 1)) with arg_scope([conv2d, dense], kernel_initializer=kernel_initializer, activation=tf.nn.relu, norm="None", is_training=self.is_training): out = conv2d(out, 32, 5, strides=1, padding='SAME') out = tf.nn.max_pool(out, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') out = conv2d(out, 64, 5, strides=1, padding='SAME') out = tf.nn.max_pool(out, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') out = tf.layers.Flatten()(out) out = dense(out, 1024) out = dense(out, self.num_classes, activation=None) return out
def separate_conv_layer(self, inputs, num_output_channels, kernel_size, stride, rate, layer_name, full_layer_name): with arg_scope( [slim.conv2d], weights_regularizer=None, weights_initializer=None, trainable=False, activation_fn=None, normalizer_fn=None, normalizer_params=None, biases_initializer=None): #make first layer clean, no BN no biases no activation func K = self._K_by_layer_dict[full_layer_name] layer1_name = LayerName(layer_name.replace('conv', 'convsep')) net = conv2d_same(inputs, K, kernel_size=(kernel_size,1), stride=[stride,1], scope=layer1_name) with slim.arg_scope(resnet_arg_scope(is_training=False)): net = conv2d_same(net, num_output_channels, kernel_size=(1,kernel_size), stride=[1,stride], scope=layer_name) return net
def objective_tower(self, features, init=True): """Objective in terms of bits-per-pixel. """ #features = tf.expand_dims(features, [1, 2]) features = features[:, tf.newaxis, tf.newaxis, :] x = features objective = 0 # The arg_scope call ensures that the actnorm parameters are set such that # the per-channel output activations have zero mean and unit variance # ONLY during the first step. After that the parameters are learned # through optimisation. ops = [ glow_ops.get_variable_ddi, glow_ops.actnorm, glow_ops.get_dropout ] with arg_scope(ops, init=init): encoder = glow_ops.encoder_decoder self.z, encoder_objective, self.eps, _, _ = encoder("flow", x, self.hparams, eps=None, reverse=False) objective += encoder_objective self.z_top_shape = get_shape_list(self.z) prior_dist = self.top_prior() prior_objective = tf.reduce_sum(prior_dist.log_prob(self.z), axis=[1, 2, 3]) #self.z_sample = prior_dist.sample() objective += prior_objective # bits per pixel _, h, w, c = get_shape_list(x) objective = -objective / (np.log(2) * h * w * c) self.z = tf.concat(self.eps + [self.z], axis=-1) return objective
def test_layer(layer, layer_kwargs): shape = [128, 32, 32, 3] x = tf.placeholder(tf.float32, shape, name='image') logdet = tf.zeros_like(x)[:, 0, 0, 0] with arg_scope([get_conv_weight_np], unit_testing=True): with tf.variable_scope('test'): z, logdet = layer('layer', x, logdet, reverse=False, **kwargs) with tf.variable_scope('test', reuse=True): recon, logdet_out = layer('layer', z, logdet, reverse=True, **kwargs) print('layer', layer) sess = tf.Session() sess.run(tf.global_variables_initializer()) x_np = np.random.randn(*shape).astype('float32') z_np, recon_np, logdet_np = sess.run([z, recon, logdet_out], feed_dict={x: x_np}) z_recon_np = sess.run([z], feed_dict={x: recon_np}) def rmse(a, b): return np.sqrt(np.mean(np.power(a - b, 2))) print('RMSE on x:\t', rmse(x_np, recon_np)) print('RMSE on conv(x):\t', rmse(z_np, z_recon_np)) print('log det: \t', rmse(logdet_np, 0)) print('') tf.reset_default_graph()
def grad_fn(*output_grads, **kwargs): """Recompute outputs for gradient computation.""" variables = [] if original_vars: variables = kwargs["variables"] if set(variables) != original_vars: raise ValueError(_WRONG_VARS_ERR) del kwargs inputs = list(args) # Recompute outputs with framework_ops.control_dependencies(output_grads): if use_data_dep_: inputs = _force_data_dependency(output_grads, inputs) with contrib_framework_ops.arg_scope(arg_scope): with variable_scope.variable_scope(vs, reuse=True): with backprop.GradientTape() as tape: fn_kwargs = {} if has_is_recompute_kwarg: fn_kwargs["is_recomputing"] = True outputs = fn(*inputs, **fn_kwargs) recompute_vars = set(tape.watched_variables()) if original_vars != recompute_vars: raise ValueError(_WRONG_VARS_ERR) if not (isinstance(outputs, list) or isinstance(outputs, tuple)): outputs = [outputs] outputs = list(outputs) grads = gradients_impl.gradients(outputs, inputs + variables, output_grads) if tupleize_grads: if use_data_dep_: grads = _tuple_with_data_dep(grads) else: grads = control_flow_ops.tuple(grads) grad_inputs = grads[:len(inputs)] grad_vars = grads[len(inputs):] return grad_inputs, grad_vars
def cnnet(input_patch_img, label, is_training): with arg_scope([layers.conv2d], padding='SAME', normalizer_fn=layers.batch_norm, normalizer_params={"is_training": is_training}): net = tf.image.resize_images(input_patch_img, [256, 256], method=2) net = layers.conv2d(net, 8, [3, 3], stride=[2, 2], scope='convd1') # Nm net = layers.conv2d(net, 32, [7, 7], scope='convd2') #128 128 32 # strides默认为1,激活函数默认为relu net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') # 池化窗口为3,步长为2 net = layers.conv2d(net, 32, [3, 3], scope='convd3') net_m = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') net_a = tf.nn.avg_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') net = tf.concat((net_m, net_a), axis=3) net = layers.conv2d(net, 64, [3, 3], scope='convd4') net_m = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') net_a = tf.nn.avg_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') net = tf.concat((net_m, net_a), axis=3) net = layers.conv2d(net, 128, [3, 3], scope='convd5') net_m = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') net_a = tf.nn.avg_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') net = tf.concat((net_m, net_a), axis=3) net = tf.nn.avg_pool(net, ksize=[1,8,8,1], strides=[1,8,8,1], padding="SAME") # Global average pooling net = tf.reshape(net, [-1,256]) net = layers.fully_connected(net, 1024) output = layers.fully_connected(net, 1, activation_fn=None) score = output # label = label*np.ones(num_patch) label = tf.reshape(label, [-1, 1]) loss = tf.losses.mean_squared_error(label, output) return score, loss # output,loss
def vq_decoder_spec(quant_t, quant_b, ema=None, nr_channel=128, nr_res_block=2, nr_res_channel=64, embedding_dim=64): """ Input: Tensor quant_t of shape (N,H//8,W//8,C) (e.g. (128,32,32,64)) Tensor quant_b of shape (N,H//4,W//4,C) (e.g. (128,64,64,64)) Output: Tensor dec_b of shape (N,H,W,3) (e.g. (128,256,256,3)) """ counters = {} with arg_scope([nn.conv2d, nn.deconv2d], counters=counters, ema=ema): # Bottom decoder quant_t = nn.deconv2d(quant_t, embedding_dim, filter_size=[4, 4], stride=[2, 2]) dec_b = tf.concat([quant_b, quant_t], -1) dec_b = nn.conv2d(dec_b, nr_channel) for rep in range(nr_res_block): dec_b = nn.resnet(dec_b, num_res_channel=nr_res_channel, nonlinearity=tf.nn.elu) dec_b = tf.nn.elu(dec_b) dec_b = nn.deconv2d(dec_b, nr_channel // 2, filter_size=[4, 4], stride=[2, 2]) dec_b = tf.nn.elu(dec_b) dec_b = nn.deconv2d(dec_b, 3, filter_size=[4, 4], stride=[2, 2]) return {'dec_b': dec_b}
def __call__(self, features): """ Define tf graph. """ inputs = features['image'] with tf.variable_scope('encoder') as vsc: with slim.arg_scope(resnet_v2.resnet_arg_scope()): # conv1 with arg_scope([layers_lib.conv2d], activation_fn=None, normalizer_fn=None): net = resnet_utils.conv2d_same(inputs, 16, 5, stride=2, scope='conv1') tf.add_to_collection(vsc.original_name_scope, net) # resnet blocks blocks = [] for i in range(len(self.encoder_params['block_name'])): block = resnet_v2.resnet_v2_block( scope=self.encoder_params['block_name'][i], base_depth=self.encoder_params['base_depth'][i], num_units=self.encoder_params['num_units'][i], stride=self.encoder_params['stride'][i]) blocks.append(block) net, _ = resnet_v2.resnet_v2( net, blocks, is_training=(self.mode == ModeKeys.TRAIN), global_pool=False, output_stride=2, include_root_block=False, scope='resnet') tf.add_to_collection(vsc.original_name_scope, net) return net
def segmentation_discriminator(inputs, batch_norm_decay, weight_decay, is_training): with tf.variable_scope('discriminator', reuse=tf.AUTO_REUSE): with arg_scope([fc_layer, conv2d_layer, global_avg_pooling_layer], to_batch_norm=True, batch_norm_decay=batch_norm_decay, is_training=is_training, activation_fn=tf.nn.leaky_relu, weights_regularizer=slim.l2_regularizer(weight_decay)): net = conv2d_layer(inputs, 32, 5, strides=[1, 2, 2, 1]) # 256 x 256 net = conv2d_layer(net, 64, 5, strides=[1, 2, 2, 1]) # 128 x 128 net = conv2d_layer(net, 128, 3, strides=[1, 2, 2, 1]) # 64 x 64 net = conv2d_layer(net, 256, 3, strides=[1, 2, 2, 1]) # 32 x 32 net = conv2d_layer(net, 512, 3, strides=[1, 2, 2, 1]) # 16 x 16 net = global_avg_pooling_layer(net, upsample=False, keepdims=False) net = fc_layer(net, 2, to_batch_norm=False, activation_fn=tf.nn.sigmoid) return net
def compute_est_ll(x, params=None, reuse=tf.AUTO_REUSE): with tf.variable_scope("estimator"): with arg_scope([nn.dense], params=params): with tf.variable_scope("encoder", reuse=reuse): h_enc_1 = nn.dense(x, 2, 128, "dense1", nonlinearity=nonlin) # h_enc_1 = nn.batch_norm(h_enc_1, "bn1", 128, 2) h_enc_2 = nn.dense(h_enc_1, 128, 128, "dense2", nonlinearity=nonlin) # h_enc_2 = nn.batch_norm(h_enc_2, "bn2", 128, 2) z_mean = nn.dense(h_enc_2, 128, vae_z_dim, "dense3", nonlinearity=None) z_logvar = nn.dense(h_enc_2, 128, vae_z_dim, "dense4", nonlinearity=None) epsilon = tf.random_normal(tf.shape(z_mean), dtype=tf.float32) z = z_mean + tf.exp(0.5 * z_logvar) * epsilon with tf.variable_scope("decoder", reuse=reuse): h_dec_1 = nn.dense(z, vae_z_dim, 128, "dense1", nonlinearity=nonlin) # h_dec_1 = nn.batch_norm(h_dec_1, "bn1", 128, 2) h_dec_2 = nn.dense(h_dec_1, 128, 128, "dense2", nonlinearity=nonlin) # h_dec_2 = nn.batch_norm(h_dec_2, "bn2", 128, 2) x_mean = nn.dense(h_dec_2, 128, x_dim, "dense3", nonlinearity=None) elbo = tf.reduce_mean(tf.reduce_sum(- 0.5 * np.log(2 * np.pi) - 0.5 * np.log(vae_x_var) - tf.square(x - x_mean) / ( 2 * vae_x_var), axis=1) - tf.reduce_sum(- 0.5 * (1 + z_logvar - tf.square(z_mean) - tf.exp(z_logvar)), axis=1)) return elbo, x_mean
def Model(x, is_training, init=False, ema=None): norm_prms = {'is_training': is_training} with arg_scope([conv2d], normalizer_fn=batch_norm, activation_fn=lrelu, normalizer_params=norm_prms): ly_x = tf.reshape(x, [-1, 32, 32, 3]) ly_x = GaussianNoise(ly_x, sigma=flgs.augment_noise_stddev, is_training=is_training) ly_x = conv2d(ly_x, 64, 3) ly_x = conv2d(ly_x, 64, 3) ly_x = conv2d(ly_x, 64, 3) ly_x = max_pool2d(ly_x, kernel_size=2) ly_x = dropout(ly_x, keep_prob=0.5, is_training=is_training) ly_x = conv2d(ly_x, 128, 3) ly_x = conv2d(ly_x, 128, 3) ly_x = conv2d(ly_x, 128, 3) ly_x = max_pool2d(ly_x, kernel_size=2) ly_x = dropout(ly_x, keep_prob=0.5, is_training=is_training) ly_x = conv2d(ly_x, 256, 3, padding='VALID') ly_x_top = conv2d(ly_x, 128, 1) ly_x = tf.reduce_mean(ly_x_top, axis=[1, 2]) class_logits = fully_connected(ly_x, 10, activation_fn=None) return class_logits, ly_x, ly_x_top
def decoder(self, encoder_output, batch_norm_decay, is_training=True): with tf.variable_scope("decoder"): with tf.contrib.slim.arg_scope( resnet_v2.resnet_arg_scope( batch_norm_decay=batch_norm_decay)): with arg_scope([layers.batch_norm], is_training=is_training): with tf.variable_scope("low_level_features"): low_level_features = self.visual_feat_c2 low_level_features = layers_lib.conv2d( low_level_features, 48, [1, 1], stride=1, scope='conv_1x1') low_level_features_size = tf.shape( low_level_features)[1:3] with tf.variable_scope("upsampling_logits"): net = tf.image.resize_bilinear(encoder_output, low_level_features_size, name='upsample_1') net = tf.concat([net, low_level_features], axis=3, name='concat') net = layers_lib.conv2d(net, 256, [3, 3], stride=1, scope='conv_3x3_1') net = layers_lib.conv2d(net, 256, [3, 3], stride=1, scope='conv_3x3_2') net = layers_lib.conv2d(net, 1, [1, 1], activation_fn=None, normalizer_fn=None, scope='conv_1x1') return net
def ar_conv2d(name, x, num_filters, filter_size=(3, 3), stride=(1, 1), pad="SAME", init_scale=1., zerodiagonal=True, **_): h = filter_size[0] w = filter_size[1] n_in = int(x.get_shape()[1]) n_out = num_filters mask = tf.constant(get_conv_ar_mask(h, w, n_in, n_out, zerodiagonal)) with arg_scope([conv2d]): return conv2d(name, x, num_filters, filter_size, stride, pad, init_scale, mask=mask)
def generator(z, training=True, weight_decay=0.0001, batch_norm_decay=0.997, batch_norm_epsilon=1e-5, batch_norm_scale=True): batch_norm_params = { 'is_training': training, 'decay': batch_norm_decay, 'epsilon': batch_norm_epsilon, 'scale': batch_norm_scale, 'updates_collections': ops.GraphKeys.UPDATE_OPS, } gen = slim.fully_connected(gen, 1024) gen = tf.reshape(gen, [batch_size, 1, 1, 1024]) with arg_scope( [slim.conv2d_transpose], weights_regularizer=regularizers.l2_regularizer(weight_decay), weights_initializer=initializers.variance_scaling_initializer(), activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params=batch_norm_params, scope="transposed_convolution"): # Each tuple is (number of channels, kernel size, stride) l = [(1024, [3, 3], [2, 2]), (512, [3, 3], [2, 2]), (256, [3, 3], [2, 2]), (128, [3, 3], [2, 2]), (128, [3, 3], [2, 2]), (64, [3, 3], [2, 2]), (3, [3, 3], [2, 2])] gen = slim.stack(gen, slim.conv2d_transpose, l) gen = tf.tanh(gen) return gen
def actnorm_scale(name, x, scale=1., logdet=None, logscale_factor=3., batch_variance=False, reverse=False, init=False, trainable=True): shape = x.get_shape() with tf.variable_scope(name), arg_scope([get_variable_ddi], trainable=trainable): assert len(shape) == 2 or len(shape) == 4 if len(shape) == 2: x_var = tf.reduce_mean(x ** 2, [0], keepdims=True) logdet_factor = 1 _shape = (1, int_shape(x)[1]) elif len(shape) == 4: x_var = tf.reduce_mean(x ** 2, [0, 1, 2], keepdims=True) logdet_factor = int(shape[1]) * int(shape[2]) _shape = (1, 1, 1, int_shape(x)[3]) if batch_variance: x_var = tf.reduce_mean(x ** 2, keepdims=True) logs = get_variable_ddi("logs", _shape, initial_value=tf.log( scale / (tf.sqrt( x_var) + 1e-6)) / logscale_factor) * logscale_factor if not reverse: x = x * tf.exp(logs) else: x = x * tf.exp(-logs) if logdet != None: dlogdet = tf.reduce_sum(logs) * logdet_factor if reverse: dlogdet *= -1 return x, logdet + dlogdet return x
def testAtrousFullyConvolutionalValues(self): """Verify dense feature extraction with atrous convolution.""" nominal_stride = 32 for output_stride in [4, 8, 16, 32, None]: with arg_scope(resnet_utils.resnet_arg_scope(is_training=False)): with ops.Graph().as_default(): with self.test_session() as sess: random_seed.set_random_seed(0) inputs = create_test_input(2, 81, 81, 3) # Dense feature extraction followed by subsampling. output, _ = self._resnet_small( inputs, None, global_pool=False, output_stride=output_stride) if output_stride is None: factor = 1 else: factor = nominal_stride // output_stride output = resnet_utils.subsample(output, factor) # Make the two networks use the same weights. variable_scope.get_variable_scope().reuse_variables() # Feature extraction at the nominal network rate. expected, _ = self._resnet_small(inputs, None, global_pool=False) sess.run(variables.global_variables_initializer()) self.assertAllClose( output.eval(), expected.eval(), atol=1e-4, rtol=1e-4)
def texture_discriminator_spec(x, nr_channel=64): """ Input: Tensor x of shape (2*N,H,W,4) (e.g. (16,256,256,4)) Output: Tensor x_out of shape (2*N,(H/64)*(W/64)*C") (e.g. (16,4*4*256)) """ counters = {} with arg_scope([nn.snconv2d], filter_size=[5, 5], stride=[2, 2], nonlinearity=tf.nn.leaky_relu, counters=counters): cnum = nr_channel x = nn.snconv2d(x, cnum) x = nn.snconv2d(x, cnum * 2) x = nn.snconv2d(x, cnum * 4) x = nn.snconv2d(x, cnum * 4) x = nn.snconv2d(x, cnum * 4) x = nn.snconv2d(x, cnum * 4) x_out = tf.layers.flatten(x) return x_out
def roi_fc(inputs, boxes, box_idx, scope='vgg_16'): with variable_scope.variable_scope(scope, 'vgg_16', [inputs]) as sc: end_points_collection = sc.original_name_scope + '_end_points' # Collect outputs for conv2d, fully_connected and max_pool2d. with arg_scope( [layers.conv2d, layers_lib.fully_connected, layers_lib.max_pool2d], outputs_collections=end_points_collection): # Use conv2d instead of fully_connected layers. net = roi_pooling(inputs, boxes, box_idx) net = layers.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6') net = layers_lib.dropout(net, 0.5, is_training=True, scope='dropout6') net = layers.conv2d(net, 4096, [1, 1], scope='fc7') net = layers_lib.dropout(net, 0.5, is_training=True, scope='dropout7') return net
def testReuseArgScopeNested(self): func1_kwargs = {'a': 1, 'b': None, 'c': [1]} func2_kwargs = {'b': 2, 'd': [2]} key = _key_op current_scope1 = {key(func1): func1_kwargs.copy()} current_scope2 = { key(func1): func1_kwargs.copy(), key(func2): func2_kwargs.copy() } with self.cached_session(): with arg_scope([func1], a=1, b=None, c=[1]) as scope1: with arg_scope([func2], b=2, d=[2]) as scope2: pass with arg_scope(scope1): with arg_scope([]) as current_arg_scope: self.assertDictEqual(current_arg_scope, current_scope1) with arg_scope(scope2): with arg_scope([]) as current_arg_scope: self.assertDictEqual(current_arg_scope, current_scope2)
def resnet_v2( inputs, blocks, num_classes=None, is_training=True, global_pool=True, output_stride=None, include_root_block=True, centered_stride=False, reuse=None, scope=None): """Generator for v2 (preactivation) ResNet models. This function generates a family of ResNet v2 models. See the resnet_v2_*() methods for specific model instantiations, obtained by selecting different block instantiations that produce ResNets of various depths. Training for image classification on Imagenet is usually done with [224, 224] inputs, resulting in [7, 7] feature maps at the output of the last ResNet block for the ResNets defined in [1] that have nominal stride equal to 32. However, for dense prediction tasks we advise that one uses inputs with spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In this case the feature maps at the ResNet output will have spatial shape [(height - 1) / output_stride + 1, (width - 1) / output_stride + 1] and corners exactly aligned with the input image corners, which greatly facilitates alignment of the features to the image. Using as input [225, 225] images results in [8, 8] feature maps at the output of the last ResNet block. For dense prediction tasks, the ResNet needs to run in fully-convolutional (FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all have nominal stride equal to 32 and a good choice in FCN mode is to use output_stride=16 in order to increase the density of the computed features at small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915. Args: inputs: A tensor of size [batch, height_in, width_in, channels]. blocks: A list of length equal to the number of ResNet blocks. Each element is a resnet_utils.Block object describing the units in the block. num_classes: Number of predicted classes for classification tasks. If None we return the features before the logit layer. is_training: whether batch_norm layers are in training mode. global_pool: If True, we perform global average pooling before computing the logits. Set to True for image classification, False for dense prediction. output_stride: If None, then the output will be computed at the nominal network stride. If output_stride is not None, it specifies the requested ratio of input to output spatial resolution. include_root_block: If True, include the initial convolution followed by max-pooling, if False excludes it. If excluded, `inputs` should be the results of an activation-less convolution. reuse: whether or not the network and its variables should be reused. To be able to reuse 'scope' must be given. scope: Optional variable_scope. Returns: net: A rank-4 tensor of size [batch, height_out, width_out, channels_out]. If global_pool is False, then height_out and width_out are reduced by a factor of output_stride compared to the respective height_in and width_in, else both height_out and width_out equal one. If num_classes is None, then net is the output of the last ResNet block, potentially after global average pooling. If num_classes is not None, net contains the pre-softmax activations. end_points: A dictionary from components of the network to the corresponding activation. Raises: ValueError: If the target output_stride is not valid. """ with variable_scope.variable_scope( scope, 'resnet_v2', [inputs], reuse=reuse) as sc: end_points_collection = sc.original_name_scope + '_end_points' with arg_scope( [layers_lib.conv2d, bottleneck, resnet_utils.stack_blocks_dense], outputs_collections=end_points_collection): with arg_scope([layers.batch_norm], is_training=is_training): net = inputs if include_root_block: if output_stride is not None: if output_stride % 4 != 0: raise ValueError('The output_stride needs to be a multiple of 4.') output_stride /= 4 # We do not include batch normalization or activation functions in # conv1 because the first ResNet unit will perform these. Cf. # Appendix of [2]. with arg_scope([layers_lib.conv2d], activation_fn=None, normalizer_fn=None): net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1') net = resnet_utils.max_pool2d_same( net, 3, stride=2, scope='pool1', centered_stride=centered_stride and output_stride == 4) net = resnet_utils.stack_blocks_dense(net, blocks, output_stride) # This is needed because the pre-activation variant does not have batch # normalization or activation functions in the residual unit output. See # Appendix of [2]. net = slim.batch_norm(net, activation_fn=nn_ops.relu, scope='postnorm') if global_pool: # Global average pooling. net = math_ops.reduce_mean(net, tfu.image_axes(), name='pool5', keepdims=True) if num_classes is not None: net = layers_lib.conv2d( net, num_classes, [1, 1], activation_fn=None, normalizer_fn=None, scope='logits') # Convert end_points_collection into a dictionary of end_points. end_points = utils.convert_collection_to_dict(end_points_collection) if num_classes is not None: end_points['predictions'] = layers.softmax(net, scope='predictions') return net, end_points