def testVariableCollectionsWithArgScopeNested(self): with self.test_session(): with scopes.arg_scope([variables.variable], collections='A'): a = variables.variable('a', []) with scopes.arg_scope([variables.variable], collections='B'): b = variables.variable('b', []) self.assertEquals(a, tf.get_collection('A')[0]) self.assertEquals(b, tf.get_collection('B')[0])
def testReuseArgScope(self): func1_kwargs = {'a': 1, 'b': None, 'c': [1]} key_op = (func1.__module__, func1.__name__) current_scope = {key_op: func1_kwargs.copy()} with self.test_session(): with scopes.arg_scope([func1], a=1, b=None, c=[1]) as scope1: pass with scopes.arg_scope(scope1) as scope: self.assertDictEqual(scope, current_scope)
def testVariableCollectionsWithArgScopeNonNested(self): with self.test_session(): with scopes.arg_scope([variables.variable], collections='A'): a = variables.variable('a', []) with scopes.arg_scope([variables.variable], collections='B'): b = variables.variable('b', []) variables.variable('c', []) self.assertListEqual([a], tf.get_collection('A')) self.assertListEqual([b], tf.get_collection('B'))
def testNestedArgScope(self): func1_args = (0, ) func1_kwargs = {'a': 1, 'b': None, 'c': [1]} with scopes.arg_scope([func1], a=1, b=None, c=[1]): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs) func1_kwargs['b'] = 2 with scopes.arg_scope([func1], b=2): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs)
def testCurrentArgScopeNested(self): func1_kwargs = {'a': 1, 'b': None, 'c': [1]} func2_kwargs = {'b': 2, 'd': [2]} key = lambda f: (f.__module__, f.__name__) current_scope = { key(func1): func1_kwargs.copy(), key(func2): func2_kwargs.copy() } with self.test_session(): with scopes.arg_scope([func1], a=1, b=None, c=[1]): with scopes.arg_scope([func2], b=2, d=[2]) as scope: self.assertDictEqual(scope, current_scope)
def testVariableRestoreWithArgScopeNested(self): with self.test_session(): with scopes.arg_scope([variables.variable], restore=True): a = variables.variable('a', []) with scopes.arg_scope([variables.variable], trainable=False, collections=['A', 'B']): b = variables.variable('b', []) c = variables.variable('c', []) self.assertListEqual([a, b, c], variables.get_variables_to_restore()) self.assertListEqual([a, c], tf.trainable_variables()) self.assertListEqual([b], tf.get_collection('A')) self.assertListEqual([b], tf.get_collection('B'))
def testPartiallySharedArgScope(self): func1_args = (0, ) func1_kwargs = {'a': 1, 'b': None, 'c': [1]} func2_args = (1, ) func2_kwargs = {'a': 1, 'b': None, 'd': [2]} with scopes.arg_scope([func1, func2], a=1, b=None): with scopes.arg_scope([func1], c=[1]), scopes.arg_scope([func2], d=[2]): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs) args, kwargs = func2(1) self.assertTupleEqual(args, func2_args) self.assertDictEqual(kwargs, func2_kwargs)
def testOverwriteArgScope(self): func1_args = (0, ) func1_kwargs = {'a': 1, 'b': 2, 'c': [1]} with scopes.arg_scope([func1], a=1, b=None, c=[1]): args, kwargs = func1(0, b=2) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs)
def testVariableWithDeviceFunction(self): class DevFn(object): def __init__(self): self.counter = -1 def __call__(self, op): self.counter += 1 return 'cpu:%d' % self.counter with self.test_session(): with scopes.arg_scope([variables.variable], device=DevFn()): a = variables.variable('a', []) b = variables.variable('b', []) c = variables.variable('c', [], device='cpu:12') d = variables.variable('d', []) with tf.device('cpu:99'): e_init = tf.constant(12) e = variables.variable('e', initializer=e_init) self.assertDeviceEqual(a.device, 'cpu:0') self.assertDeviceEqual(a.initial_value.device, 'cpu:0') self.assertDeviceEqual(b.device, 'cpu:1') self.assertDeviceEqual(b.initial_value.device, 'cpu:1') self.assertDeviceEqual(c.device, 'cpu:12') self.assertDeviceEqual(c.initial_value.device, 'cpu:12') self.assertDeviceEqual(d.device, 'cpu:2') self.assertDeviceEqual(d.initial_value.device, 'cpu:2') self.assertDeviceEqual(e.device, 'cpu:3') self.assertDeviceEqual(e.initial_value.device, 'cpu:99')
def testSimpleArgScopeWithTuple(self): func1_args = (0, ) func1_kwargs = {'a': 1, 'b': None, 'c': [1]} with self.test_session(): with scopes.arg_scope((func1, ), a=1, b=None, c=[1]): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs)
def testReuseFCWithBatchNorm(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height * width * 3), seed=1) with scopes.arg_scope([ops.fc], batch_norm_params={'decay': 0.9}): net = ops.fc(images, 27, scope='fc1') net = ops.fc(net, 27, scope='fc1', reuse=True) self.assertEquals(len(variables.get_variables()), 4) self.assertEquals(len(variables.get_variables('fc1/BatchNorm')), 3)
def testReuseConvWithBatchNorm(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height, width, 32), seed=1) with scopes.arg_scope([ops.conv2d], batch_norm_params={'decay': 0.9}): net = ops.conv2d(images, 32, [3, 3], scope='Conv') net = ops.conv2d(net, 32, [3, 3], scope='Conv', reuse=True) self.assertEquals(len(variables.get_variables()), 4) self.assertEquals(len(variables.get_variables('Conv/BatchNorm')), 3) self.assertEquals(len(variables.get_variables('Conv_1/BatchNorm')), 0)
def testFCWithBatchNorm(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height * width * 3), seed=1) with scopes.arg_scope([ops.fc], batch_norm_params={}): net = ops.fc(images, 27) net = ops.fc(net, 27) self.assertEquals(len(variables.get_variables()), 8) self.assertEquals(len(variables.get_variables('FC/BatchNorm')), 3) self.assertEquals(len(variables.get_variables('FC_1/BatchNorm')), 3)
def testSharedArgScopeTuple(self): func1_args = (0, ) func1_kwargs = {'a': 1, 'b': None, 'c': [1]} with scopes.arg_scope((func1, func2), a=1, b=None, c=[1]): args, kwargs = func1(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs) args, kwargs = func2(0) self.assertTupleEqual(args, func1_args) self.assertDictEqual(kwargs, func1_kwargs)
def testReplicaDeviceSetter(self): device_fn = tf.train.replica_device_setter(2) with tf.Graph().as_default(): with scopes.arg_scope([variables.global_step], device=device_fn): gs = variables.global_step() gs2 = variables.global_step() self.assertEquals(gs, gs2) self.assertDeviceEqual(gs.device, '/job:ps/task:0') self.assertDeviceEqual(gs.initial_value.device, '/job:ps/task:0') self.assertDeviceEqual(gs2.device, '/job:ps/task:0') self.assertDeviceEqual(gs2.initial_value.device, '/job:ps/task:0')
def testVariableWithVariableDeviceChooser(self): with tf.Graph().as_default(): device_fn = variables.VariableDeviceChooser() with scopes.arg_scope([variables.global_step], device=device_fn): gs = variables.global_step() gs2 = variables.global_step() self.assertEquals(gs, gs2) self.assertDeviceEqual(gs.device, 'cpu:0') self.assertDeviceEqual(gs.initial_value.device, gs.device) self.assertDeviceEqual(gs2.device, 'cpu:0') self.assertDeviceEqual(gs2.initial_value.device, gs2.device)
def testDeviceFn(self): class DevFn(object): def __init__(self): self.counter = -1 def __call__(self, op): self.counter += 1 return '/cpu:%d' % self.counter with tf.Graph().as_default(): with scopes.arg_scope([variables.global_step], device=DevFn()): gs = variables.global_step() gs2 = variables.global_step() self.assertDeviceEqual(gs.device, '/cpu:0') self.assertEquals(gs, gs2) self.assertDeviceEqual(gs2.device, '/cpu:0')
def testVariableGPUPlacement(self): with tf.Graph().as_default(): device_fn = variables.VariableDeviceChooser(placement='gpu:0') with scopes.arg_scope([variables.variable], device=device_fn): a = variables.variable('a', []) b = variables.variable('b', []) c = variables.variable('c', [], device='cpu:12') d = variables.variable('d', []) with tf.device('cpu:99'): e_init = tf.constant(12) e = variables.variable('e', initializer=e_init) # The values below highlight how the VariableDeviceChooser puts initial # values on the same device as the variable job. self.assertDeviceEqual(a.device, '/gpu:0') self.assertDeviceEqual(a.initial_value.device, a.device) self.assertDeviceEqual(b.device, '/gpu:0') self.assertDeviceEqual(b.initial_value.device, b.device) self.assertDeviceEqual(c.device, '/cpu:12') self.assertDeviceEqual(c.initial_value.device, c.device) self.assertDeviceEqual(d.device, '/gpu:0') self.assertDeviceEqual(d.initial_value.device, d.device) self.assertDeviceEqual(e.device, '/gpu:0') self.assertDeviceEqual(e.initial_value.device, '/cpu:99')
def fc(inputs, num_units_out, activation=tf.nn.relu, stddev=0.01, bias=0.0, weight_decay=0, batch_norm_params=None, is_training=True, trainable=True, restore=True, scope=None, reuse=None): """Adds a fully connected layer followed by an optional batch_norm layer. FC creates a variable called 'weights', representing the fully connected weight matrix, that is multiplied by the input. If `batch_norm` is None, a second variable called 'biases' is added to the result of the initial vector-matrix multiplication. Args: inputs: a [B x N] tensor where B is the batch size and N is the number of input units in the layer. num_units_out: the number of output units in the layer. activation: activation function. stddev: the standard deviation for the weights. bias: the initial value of the biases. weight_decay: the weight decay. batch_norm_params: parameters for the batch_norm. If is None don't use it. is_training: whether or not the model is in training mode. trainable: whether or not the variables should be trainable or not. restore: whether or not the variables should be marked for restore. scope: Optional scope for variable_scope. reuse: whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. Returns: the tensor variable representing the result of the series of operations. """ with tf.variable_scope(scope, 'FC', [inputs], reuse=reuse): num_units_in = inputs.get_shape()[1] weights_shape = [num_units_in, num_units_out] weights_initializer = tf.truncated_normal_initializer(stddev=stddev) l2_regularizer = None if weight_decay and weight_decay > 0: l2_regularizer = losses.l2_regularizer(weight_decay) weights = variables.variable('weights', shape=weights_shape, initializer=weights_initializer, regularizer=l2_regularizer, trainable=trainable, restore=restore) if batch_norm_params is not None: outputs = tf.matmul(inputs, weights) with scopes.arg_scope([batch_norm], is_training=is_training, trainable=trainable, restore=restore): outputs = batch_norm(outputs, **batch_norm_params) else: bias_shape = [ num_units_out, ] bias_initializer = tf.constant_initializer(bias) biases = variables.variable('biases', shape=bias_shape, initializer=bias_initializer, trainable=trainable, restore=restore) outputs = tf.nn.xw_plus_b(inputs, weights, biases) if activation: outputs = activation(outputs) return outputs
def conv2d(inputs, num_filters_out, kernel_size, stride=1, padding='SAME', activation=tf.nn.relu, stddev=0.01, bias=0.0, weight_decay=0, batch_norm_params=None, is_training=True, trainable=True, restore=True, scope=None, reuse=None): """Adds a 2D convolution followed by an optional batch_norm layer. conv2d creates a variable called 'weights', representing the convolutional kernel, that is convolved with the input. If `batch_norm_params` is None, a second variable called 'biases' is added to the result of the convolution operation. Args: inputs: a tensor of size [batch_size, height, width, channels]. num_filters_out: the number of output filters. kernel_size: a list of length 2: [kernel_height, kernel_width] of of the filters. Can be an int if both values are the same. stride: a list of length 2: [stride_height, stride_width]. Can be an int if both strides are the same. Note that presently both strides must have the same value. padding: one of 'VALID' or 'SAME'. activation: activation function. stddev: standard deviation of the truncated guassian weight distribution. bias: the initial value of the biases. weight_decay: the weight decay. batch_norm_params: parameters for the batch_norm. If is None don't use it. is_training: whether or not the model is in training mode. trainable: whether or not the variables should be trainable or not. restore: whether or not the variables should be marked for restore. scope: Optional scope for variable_scope. reuse: whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. Returns: a tensor representing the output of the operation. """ with tf.variable_scope(scope, 'Conv', [inputs], reuse=reuse): kernel_h, kernel_w = _two_element_tuple(kernel_size) stride_h, stride_w = _two_element_tuple(stride) num_filters_in = inputs.get_shape()[-1] weights_shape = [kernel_h, kernel_w, num_filters_in, num_filters_out] weights_initializer = tf.truncated_normal_initializer(stddev=stddev) l2_regularizer = None if weight_decay and weight_decay > 0: l2_regularizer = losses.l2_regularizer(weight_decay) weights = variables.variable('weights', shape=weights_shape, initializer=weights_initializer, regularizer=l2_regularizer, trainable=trainable, restore=restore) conv = tf.nn.conv2d(inputs, weights, [1, stride_h, stride_w, 1], padding=padding) if batch_norm_params is not None: with scopes.arg_scope([batch_norm], is_training=is_training, trainable=trainable, restore=restore): outputs = batch_norm(conv, **batch_norm_params) else: bias_shape = [ num_filters_out, ] bias_initializer = tf.constant_initializer(bias) biases = variables.variable('biases', shape=bias_shape, initializer=bias_initializer, trainable=trainable, restore=restore) outputs = tf.nn.bias_add(conv, biases) if activation: outputs = activation(outputs) return outputs
def testDevice(self): with tf.Graph().as_default(): with scopes.arg_scope([variables.global_step], device='/gpu:0'): gs = variables.global_step() self.assertDeviceEqual(gs.device, '/gpu:0')