def testVariableCollectionsWithArgScopeNested(self):
with self.test_session():
with arg_scope([variables_lib2.variable], collections='A'):
a = variables_lib2.variable('a', [])
with arg_scope([variables_lib2.variable], collections='B'):
b = variables_lib2.variable('b', [])
self.assertEquals(a, ops.get_collection('A')[0])
self.assertEquals(b, ops.get_collection('B')[0])
def testVariableCollectionsWithArgScopeNonNested(self):
with self.test_session():
with arg_scope([variables_lib2.variable], collections='A'):
a = variables_lib2.variable('a', [])
with arg_scope([variables_lib2.variable], collections='B'):
b = variables_lib2.variable('b', [])
variables_lib2.variable('c', [])
self.assertListEqual([a], ops.get_collection('A'))
self.assertListEqual([b], ops.get_collection('B'))
def alexnet_v2_arg_scope(weight_decay=0.0005):
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
activation_fn=nn_ops.relu,
biases_initializer=init_ops.constant_initializer(0.1),
weights_regularizer=regularizers.l2_regularizer(weight_decay)):
with arg_scope([layers.conv2d], padding='SAME'):
with arg_scope([layers_lib.max_pool2d], padding='VALID') as arg_sc:
return arg_sc
def testReuseArgScope(self):
func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
key_op = _key_op(func1)
current_scope = {key_op: func1_kwargs.copy()}
with self.test_session():
with arg_scope([func1], a=1, b=None, c=[1]) as scope1:
pass
with arg_scope(scope1) as scope:
self.assertDictEqual(scope, current_scope)
def testArgScopeObjectCreatedWithinScopeInheritsArgScope(self):
def get_scope_object():
with arg_scope([func1], a=1, b=None, c=[1]) as sc:
return sc
with arg_scope([func1], b=2, d=10):
with arg_scope(get_scope_object()):
args, kwargs = func1(0)
self.assertTupleEqual(args, (0,))
self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1], 'd': 10})
def testClearArgScope(self):
func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
key_op = _key_op(func1)
func1_scope = {key_op: func1_kwargs.copy()}
with self.test_session():
with arg_scope([func1], a=1, b=None, c=[1]) as sc1:
self.assertEqual(sc1, func1_scope)
with arg_scope({}) as sc2:
self.assertEqual(sc2, {})
with arg_scope([]) as current_arg_scope:
self.assertEqual(current_arg_scope, func1_scope)
def testCurrentArgScopeNested(self):
func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
func2_kwargs = {'b': 2, 'd': [2]}
key = _key_op
current_scope = {
key(func1): func1_kwargs.copy(),
key(func2): func2_kwargs.copy()
}
with self.test_session():
with arg_scope([func1], a=1, b=None, c=[1]):
with arg_scope([func2], b=2, d=[2]) as scope:
self.assertDictEqual(scope, current_scope)
def testNestedArgScopeObjectCreatedOutsideScopeOverridesArgScope(self):
def get_scope_object():
with arg_scope([func1], a=1, b=None, c=[1]) as sc:
return sc
scope_object = get_scope_object()
with arg_scope([func1], b=2, d=10):
with arg_scope(scope_object):
args, kwargs = func1(0)
self.assertTupleEqual(args, (0,))
self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1]})
def testNestedArgScope(self):
func1_args = (0,)
func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
with arg_scope([func1], a=1, b=None, c=[1]):
args, kwargs = func1(0)
self.assertTupleEqual(args, func1_args)
self.assertDictEqual(kwargs, func1_kwargs)
func1_kwargs['b'] = 2
with arg_scope([func1], b=2):
args, kwargs = func1(0)
self.assertTupleEqual(args, func1_args)
self.assertDictEqual(kwargs, func1_kwargs)
def testPartiallySharedArgScope(self):
func1_args = (0,)
func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
func2_args = (1,)
func2_kwargs = {'a': 1, 'b': None, 'd': [2]}
with arg_scope([func1, func2], a=1, b=None):
with arg_scope([func1], c=[1]):
with arg_scope([func2], d=[2]):
args, kwargs = func1(0)
self.assertTupleEqual(args, func1_args)
self.assertDictEqual(kwargs, func1_kwargs)
args, kwargs = func2(1)
self.assertTupleEqual(args, func2_args)
self.assertDictEqual(kwargs, func2_kwargs)
def resnet_arg_scope(is_training=True,
weight_decay=0.0001,
batch_norm_decay=0.997,
batch_norm_epsilon=1e-5,
batch_norm_scale=True):
"""Defines the default ResNet arg scope.
TODO(gpapan): The batch-normalization related default values above are
appropriate for use in conjunction with the reference ResNet models
released at https://github.com/KaimingHe/deep-residual-networks. When
training ResNets from scratch, they might need to be tuned.
Args:
is_training: Whether or not we are training the parameters in the batch
normalization layers of the model.
weight_decay: The weight decay to use for regularizing the model.
batch_norm_decay: The moving average decay when estimating layer activation
statistics in batch normalization.
batch_norm_epsilon: Small constant to prevent division by zero when
normalizing activations by their variance in batch normalization.
batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the
activations in the batch normalization layer.
Returns:
An `arg_scope` to use for the resnet models.
"""
batch_norm_params = {
'is_training': is_training,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'updates_collections': ops.GraphKeys.UPDATE_OPS,
}
with arg_scope(
[layers_lib.conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params):
with arg_scope([layers.batch_norm], **batch_norm_params):
# The following implies padding='SAME' for pool1, which makes feature
# alignment easier for dense prediction tasks. This is also used in
# https://github.com/facebook/fb.resnet.torch. However the accompanying
# code of 'Deep Residual Learning for Image Recognition' uses
# padding='VALID' for pool1. You can switch to that choice by setting
# tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID').
with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
return arg_sc
def inception_v3_arg_scope(weight_decay=0.00004,
batch_norm_var_collection='moving_vars',
batch_norm_decay=0.9997,
batch_norm_epsilon=0.001,
updates_collections=ops.GraphKeys.UPDATE_OPS,
use_fused_batchnorm=True):
"""Defines the default InceptionV3 arg scope.
Args:
weight_decay: The weight decay to use for regularizing the model.
batch_norm_var_collection: The name of the collection for the batch norm
variables.
batch_norm_decay: Decay for batch norm moving average
batch_norm_epsilon: Small float added to variance to avoid division by zero
updates_collections: Collections for the update ops of the layer
use_fused_batchnorm: Enable fused batchnorm.
Returns:
An `arg_scope` to use for the inception v3 model.
"""
batch_norm_params = {
# Decay for the moving averages.
'decay': batch_norm_decay,
# epsilon to prevent 0s in variance.
'epsilon': batch_norm_epsilon,
# collection containing update_ops.
'updates_collections': updates_collections,
# Use fused batch norm if possible.
'fused': use_fused_batchnorm,
# collection containing the moving mean and moving variance.
'variables_collections': {
'beta': None,
'gamma': None,
'moving_mean': [batch_norm_var_collection],
'moving_variance': [batch_norm_var_collection],
}
}
# Set weight_decay for weights in Conv and FC layers.
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
weights_regularizer=regularizers.l2_regularizer(weight_decay)):
with arg_scope(
[layers.conv2d],
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=nn_ops.relu,
normalizer_fn=layers_lib.batch_norm,
normalizer_params=batch_norm_params) as sc:
return sc
def inception_v1_arg_scope(weight_decay=0.00004,
use_batch_norm=True,
batch_norm_var_collection='moving_vars'):
"""Defines the default InceptionV1 arg scope.
Note: Althougth the original paper didn't use batch_norm we found it useful.
Args:
weight_decay: The weight decay to use for regularizing the model.
use_batch_norm: "If `True`, batch_norm is applied after each convolution.
batch_norm_var_collection: The name of the collection for the batch norm
variables.
Returns:
An `arg_scope` to use for the inception v3 model.
"""
batch_norm_params = {
# Decay for the moving averages.
'decay': 0.9997,
# epsilon to prevent 0s in variance.
'epsilon': 0.001,
# collection containing update_ops.
'updates_collections': ops.GraphKeys.UPDATE_OPS,
# collection containing the moving mean and moving variance.
'variables_collections': {
'beta': None,
'gamma': None,
'moving_mean': [batch_norm_var_collection],
'moving_variance': [batch_norm_var_collection],
}
}
if use_batch_norm:
normalizer_fn = layers_lib.batch_norm
normalizer_params = batch_norm_params
else:
normalizer_fn = None
normalizer_params = {}
# Set weight_decay for weights in Conv and FC layers.
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
weights_regularizer=regularizers.l2_regularizer(weight_decay)):
with arg_scope(
[layers.conv2d],
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=nn_ops.relu,
normalizer_fn=normalizer_fn,
normalizer_params=normalizer_params) as sc:
return sc
def vgg_a(inputs,
num_classes=1000,
is_training=True,
dropout_keep_prob=0.5,
spatial_squeeze=True,
scope='vgg_a'):
"""Oxford Net VGG 11-Layers version A Example.
Note: All the fully_connected layers have been transformed to conv2d layers.
To use in classification mode, resize input to 224x224.
Args:
inputs: a tensor of size [batch_size, height, width, channels].
num_classes: number of predicted classes.
is_training: whether or not the model is being trained.
dropout_keep_prob: the probability that activations are kept in the dropout
layers during training.
spatial_squeeze: whether or not should squeeze the spatial dimensions of the
outputs. Useful to remove unnecessary dimensions for classification.
scope: Optional scope for the variables.
Returns:
the last op containing the log predictions and end_points dict.
"""
with variable_scope.variable_scope(scope, 'vgg_a', [inputs]) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
# Collect outputs for conv2d, fully_connected and max_pool2d.
with arg_scope(
[layers.conv2d, layers_lib.max_pool2d],
outputs_collections=end_points_collection):
net = layers_lib.repeat(
inputs, 1, layers.conv2d, 64, [3, 3], scope='conv1')
net = layers_lib.max_pool2d(net, [2, 2], scope='pool1')
net = layers_lib.repeat(net, 1, layers.conv2d, 128, [3, 3], scope='conv2')
net = layers_lib.max_pool2d(net, [2, 2], scope='pool2')
net = layers_lib.repeat(net, 2, layers.conv2d, 256, [3, 3], scope='conv3')
net = layers_lib.max_pool2d(net, [2, 2], scope='pool3')
net = layers_lib.repeat(net, 2, layers.conv2d, 512, [3, 3], scope='conv4')
net = layers_lib.max_pool2d(net, [2, 2], scope='pool4')
net = layers_lib.repeat(net, 2, layers.conv2d, 512, [3, 3], scope='conv5')
net = layers_lib.max_pool2d(net, [2, 2], scope='pool5')
# Use conv2d instead of fully_connected layers.
net = layers.conv2d(net, 4096, [7, 7], padding='VALID', scope='fc6')
net = layers_lib.dropout(
net, dropout_keep_prob, is_training=is_training, scope='dropout6')
net = layers.conv2d(net, 4096, [1, 1], scope='fc7')
net = layers_lib.dropout(
net, dropout_keep_prob, is_training=is_training, scope='dropout7')
net = layers.conv2d(
net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='fc8')
# Convert end_points_collection into a end_point dict.
end_points = utils.convert_collection_to_dict(end_points_collection)
if spatial_squeeze:
net = array_ops.squeeze(net, [1, 2], name='fc8/squeezed')
end_points[sc.name + '/fc8'] = net
return net, end_points
def testEndPointsV2(self):
"""Test the end points of a tiny v2 bottleneck network."""
blocks = [
resnet_v2.resnet_v2_block(
'block1', base_depth=1, num_units=2, stride=2),
resnet_v2.resnet_v2_block(
'block2', base_depth=2, num_units=2, stride=1),
]
inputs = create_test_input(2, 32, 16, 3)
with arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
expected = [
'tiny/block1/unit_1/bottleneck_v2/shortcut',
'tiny/block1/unit_1/bottleneck_v2/conv1',
'tiny/block1/unit_1/bottleneck_v2/conv2',
'tiny/block1/unit_1/bottleneck_v2/conv3',
'tiny/block1/unit_2/bottleneck_v2/conv1',
'tiny/block1/unit_2/bottleneck_v2/conv2',
'tiny/block1/unit_2/bottleneck_v2/conv3',
'tiny/block2/unit_1/bottleneck_v2/shortcut',
'tiny/block2/unit_1/bottleneck_v2/conv1',
'tiny/block2/unit_1/bottleneck_v2/conv2',
'tiny/block2/unit_1/bottleneck_v2/conv3',
'tiny/block2/unit_2/bottleneck_v2/conv1',
'tiny/block2/unit_2/bottleneck_v2/conv2',
'tiny/block2/unit_2/bottleneck_v2/conv3'
]
self.assertItemsEqual(expected, end_points)
def testAtrousFullyConvolutionalValues(self):
"""Verify dense feature extraction with atrous convolution."""
nominal_stride = 32
for output_stride in [4, 8, 16, 32, None]:
with arg_scope(resnet_utils.resnet_arg_scope()):
with ops.Graph().as_default():
with self.test_session() as sess:
random_seed.set_random_seed(0)
inputs = create_test_input(2, 81, 81, 3)
# Dense feature extraction followed by subsampling.
output, _ = self._resnet_small(
inputs,
None,
is_training=False,
global_pool=False,
output_stride=output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
variable_scope.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected, _ = self._resnet_small(
inputs, None, is_training=False, global_pool=False)
sess.run(variables.global_variables_initializer())
self.assertAllClose(
output.eval(), expected.eval(), atol=1e-4, rtol=1e-4)
def grad_fn(inputs, variables, outputs, output_grads):
"""Recompute outputs for gradient computation."""
del outputs
# Recompute outputs
with framework_ops.control_dependencies(output_grads):
if use_data_dep_:
inputs = _force_data_dependency(output_grads, inputs)
with contrib_framework_ops.arg_scope(cached_arg_scope[0]):
with variable_scope.variable_scope(cached_vs[0], reuse=True):
outputs = fn(*inputs)
if not (isinstance(outputs, list) or isinstance(outputs, tuple)):
outputs = [outputs]
outputs = list(outputs)
grads = gradients_impl.gradients(outputs, inputs + variables, output_grads)
if tupleize_grads:
if use_data_dep_:
grads = _tuple_with_data_dep(grads)
else:
grads = control_flow_ops.tuple(grads)
grad_inputs = grads[:len(inputs)]
grad_vars = grads[len(inputs):]
return grad_inputs, grad_vars
def _resnet_plain(self, inputs, blocks, output_stride=None, scope=None):
"""A plain ResNet without extra layers before or after the ResNet blocks."""
with variable_scope.variable_scope(scope, values=[inputs]):
with arg_scope([layers.conv2d], outputs_collections='end_points'):
net = resnet_utils.stack_blocks_dense(inputs, blocks, output_stride)
end_points = utils.convert_collection_to_dict('end_points')
return net, end_points
def slim_net_original(image, keep_prob):
with arg_scope([layers.conv2d, layers.fully_connected], biases_initializer=tf.random_normal_initializer(stddev=0.1)):
# conv2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME',
# activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None,
# weights_initializer=initializers.xavier_initializer(), weights_regularizer=None,
# biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, scope=None):
net = layers.conv2d(image, 32, [5, 5], scope='conv1', weights_regularizer=regularizers.l1_regularizer(0.5))
# max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None)
net = layers.max_pool2d(net, 2, scope='pool1')
net = layers.conv2d(net, 64, [5, 5], scope='conv2', weights_regularizer=regularizers.l2_regularizer(0.5))
summaries.summarize_tensor(net, tag='conv2')
net = layers.max_pool2d(net, 2, scope='pool2')
net = layers.flatten(net, scope='flatten1')
# fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None,
# normalizer_params=None, weights_initializer=initializers.xavier_initializer(),
# weights_regularizer=None, biases_initializer=init_ops.zeros_initializer,
# biases_regularizer=None, scope=None):
net = layers.fully_connected(net, 1024, scope='fc1')
# dropout(inputs, keep_prob=0.5, is_training=True, scope=None)
net = layers.dropout(net, keep_prob=keep_prob, scope='dropout1')
net = layers.fully_connected(net, 10, scope='fc2')
return net
def testVariableWithDeviceFunction(self):
class DevFn(object):
def __init__(self):
self.counter = -1
def __call__(self, op):
self.counter += 1
return 'cpu:%d' % self.counter
with self.test_session():
with arg_scope([variables_lib2.variable], device=DevFn()):
a = variables_lib2.variable('a', [])
b = variables_lib2.variable('b', [])
c = variables_lib2.variable('c', [], device='cpu:12')
d = variables_lib2.variable('d', [])
with ops.device('cpu:99'):
e_init = constant_op.constant(12)
e = variables_lib2.variable('e', initializer=e_init)
self.assertDeviceEqual(a.device, 'cpu:0')
self.assertEqual(a.initial_value.op.colocation_groups(),
a.op.colocation_groups())
self.assertDeviceEqual(b.device, 'cpu:1')
self.assertEqual(b.initial_value.op.colocation_groups(),
b.op.colocation_groups())
self.assertDeviceEqual(c.device, 'cpu:12')
self.assertEqual(c.initial_value.op.colocation_groups(),
c.op.colocation_groups())
self.assertDeviceEqual(d.device, 'cpu:2')
self.assertEqual(d.initial_value.op.colocation_groups(),
d.op.colocation_groups())
self.assertDeviceEqual(e.device, 'cpu:3')
self.assertDeviceEqual(e.initial_value.device, 'cpu:99')
def vgg_arg_scope(weight_decay=0.0005):
"""Defines the VGG arg scope.
Args:
weight_decay: The l2 regularization coefficient.
Returns:
An arg_scope.
"""
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
activation_fn=nn_ops.relu,
weights_regularizer=regularizers.l2_regularizer(weight_decay),
biases_initializer=init_ops.zeros_initializer()):
with arg_scope([layers.conv2d], padding='SAME') as arg_sc:
return arg_sc
def testVariableGPUPlacement(self):
with ops.Graph().as_default():
device_fn = variables_lib2.VariableDeviceChooser(device_type='GPU')
with arg_scope([variables_lib2.variable], device=device_fn):
a = variables_lib2.variable('a', [])
b = variables_lib2.variable('b', [])
c = variables_lib2.variable('c', [], device='cpu:12')
d = variables_lib2.variable('d', [])
with ops.device('cpu:99'):
e_init = constant_op.constant(12)
e = variables_lib2.variable('e', initializer=e_init)
# The values below highlight how the VariableDeviceChooser puts initial
# values on the same device as the variable job.
self.assertDeviceEqual(a.device, '/gpu:0')
self.assertEqual(a.initial_value.op.colocation_groups(),
a.op.colocation_groups())
self.assertDeviceEqual(b.device, '/gpu:0')
self.assertEqual(b.initial_value.op.colocation_groups(),
b.op.colocation_groups())
self.assertDeviceEqual(c.device, '/cpu:12')
self.assertEqual(c.initial_value.op.colocation_groups(),
c.op.colocation_groups())
self.assertDeviceEqual(d.device, '/gpu:0')
self.assertEqual(d.initial_value.op.colocation_groups(),
d.op.colocation_groups())
self.assertDeviceEqual(e.device, '/gpu:0')
self.assertDeviceEqual(e.initial_value.device, '/cpu:99')
def __init__(self, sess, dim=299):
self.batch_shape = [16, dim, dim, 3]
self._num_classes = 1001
self._scope = 'InceptionV3'
self._weights_file = './Weights/inception_v3.ckpt'
output_layer = 'Conv2d_1a_3x3'
#
# network inputs
#
self.x_tf = tf.placeholder(tf.float32, shape=self.batch_shape)
#
# network outputs
#
with slim.arg_scope(inception.inception_v3_arg_scope()):
with arg_scope([layers_lib.batch_norm, layers_lib.dropout], is_training=False): # we truncate before these layers, so this is not really necessary...
net, endpoints = inception.inception_v3_base(self.x_tf, final_endpoint=output_layer, scope=self._scope)
self.output = endpoints[output_layer]
#
# load weights
#
saver = tf.train.Saver(slim.get_model_variables(scope=self._scope))
saver.restore(sess, self._weights_file)
def testOverwriteArgScope(self):
func1_args = (0,)
func1_kwargs = {'a': 1, 'b': 2, 'c': [1]}
with arg_scope([func1], a=1, b=None, c=[1]):
args, kwargs = func1(0, b=2)
self.assertTupleEqual(args, func1_args)
self.assertDictEqual(kwargs, func1_kwargs)
def testEndPointsV2(self):
"""Test the end points of a tiny v2 bottleneck network."""
bottleneck = resnet_v2.bottleneck
blocks = [
resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 1)])
]
inputs = create_test_input(2, 32, 16, 3)
with arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
expected = [
'tiny/block1/unit_1/bottleneck_v2/shortcut',
'tiny/block1/unit_1/bottleneck_v2/conv1',
'tiny/block1/unit_1/bottleneck_v2/conv2',
'tiny/block1/unit_1/bottleneck_v2/conv3',
'tiny/block1/unit_2/bottleneck_v2/conv1',
'tiny/block1/unit_2/bottleneck_v2/conv2',
'tiny/block1/unit_2/bottleneck_v2/conv3',
'tiny/block2/unit_1/bottleneck_v2/shortcut',
'tiny/block2/unit_1/bottleneck_v2/conv1',
'tiny/block2/unit_1/bottleneck_v2/conv2',
'tiny/block2/unit_1/bottleneck_v2/conv3',
'tiny/block2/unit_2/bottleneck_v2/conv1',
'tiny/block2/unit_2/bottleneck_v2/conv2',
'tiny/block2/unit_2/bottleneck_v2/conv3'
]
self.assertItemsEqual(expected, end_points)
def _forward(self, x, gpu):
hps = self.hps
x = tf.to_float(x)
x = tf.clip_by_value((x + 0.5) / 256.0, 0.0, 1.0) - 0.5
# Input images are repeated k times on the input.
# This is used for Importance Sampling loss (k is number of samples).
data_size = hps.batch_size * hps.k
x = repeat(x, hps.k)
orig_x = x
h_size = hps.h_size
with arg_scope([conv2d, deconv2d], init=(self.mode == "init")):
layers = []
for i in range(hps.depth):
layers.append([])
for j in range(hps.num_blocks):
downsample = (i > 0) and (j == 0)
layers[-1].append(IAFLayer(hps, self.mode, downsample))
h = conv2d("x_enc", x, h_size, [5, 5], [2, 2]) # -> [16, 16]
for i, layer in enumerate(layers):
for j, sub_layer in enumerate(layer):
with tf.variable_scope("IAF_%d_%d" % (i, j)):
h = sub_layer.up(h)
# top->down
self.h_top = h_top = tf.get_variable("h_top", [h_size], initializer=tf.zeros_initializer)
h_top = tf.reshape(h_top, [1, -1, 1, 1])
h = tf.tile(h_top, [data_size, 1, hps.image_size / 2 ** len(layers), hps.image_size / 2 ** len(layers)])
kl_cost = kl_obj = 0.0
for i, layer in reversed(list(enumerate(layers))):
for j, sub_layer in reversed(list(enumerate(layer))):
with tf.variable_scope("IAF_%d_%d" % (i, j)):
h, cur_obj, cur_cost = sub_layer.down(h)
kl_obj += cur_obj
kl_cost += cur_cost
if self.mode == "train" and gpu == hps.num_gpus - 1:
tf.scalar_summary("model/kl_obj_%02d_%02d" % (i, j), tf.reduce_mean(cur_obj))
tf.scalar_summary("model/kl_cost_%02d_%02d" % (i, j), tf.reduce_mean(cur_cost))
x = tf.nn.elu(h)
x = deconv2d("x_dec", x, 3, [5, 5])
x = tf.clip_by_value(x, -0.5 + 1 / 512., 0.5 - 1 / 512.)
log_pxz = discretized_logistic(x, self.dec_log_stdv, sample=orig_x)
obj = tf.reduce_sum(kl_obj - log_pxz)
if self.mode == "train" and gpu == hps.num_gpus - 1:
tf.scalar_summary("model/log_pxz", -tf.reduce_mean(log_pxz))
tf.scalar_summary("model/kl_obj", tf.reduce_mean(kl_obj))
tf.scalar_summary("model/kl_cost", tf.reduce_mean(kl_cost))
loss = tf.reduce_sum(compute_lowerbound(log_pxz, kl_cost, hps.k))
return x, obj, loss
def inception_v1(inputs,
num_classes=1000,
is_training=True,
dropout_keep_prob=0.8,
prediction_fn=layers_lib.softmax,
spatial_squeeze=True,
reuse=None,
scope='InceptionV1'):
"""Defines the Inception V1 architecture.
This architecture is defined in:
Going deeper with convolutions
Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed,
Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich.
http://arxiv.org/pdf/1409.4842v1.pdf.
The default image size used to train this network is 224x224.
Args:
inputs: a tensor of size [batch_size, height, width, channels].
num_classes: number of predicted classes.
is_training: whether is training or not.
dropout_keep_prob: the percentage of activation values that are retained.
prediction_fn: a function to get predictions out of logits.
spatial_squeeze: if True, logits is of shape is [B, C], if false logits is
of shape [B, 1, 1, C], where B is batch_size and C is number of classes.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
Returns:
logits: the pre-softmax activations, a tensor of size
[batch_size, num_classes]
end_points: a dictionary from components of the network to the corresponding
activation.
"""
# Final pooling and prediction
with variable_scope.variable_scope(
scope, 'InceptionV1', [inputs, num_classes], reuse=reuse) as scope:
with arg_scope(
[layers_lib.batch_norm, layers_lib.dropout], is_training=is_training):
net, end_points = inception_v1_base(inputs, scope=scope)
with variable_scope.variable_scope('Logits'):
net = layers_lib.avg_pool2d(
net, [7, 7], stride=1, scope='MaxPool_0a_7x7')
net = layers_lib.dropout(net, dropout_keep_prob, scope='Dropout_0b')
logits = layers.conv2d(
net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='Conv2d_0c_1x1')
if spatial_squeeze:
logits = array_ops.squeeze(logits, [1, 2], name='SpatialSqueeze')
end_points['Logits'] = logits
end_points['Predictions'] = prediction_fn(logits, scope='Predictions')
return logits, end_points
def testSimpleArgScopeWithTuple(self):
func1_args = (0,)
func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
with self.test_session():
with arg_scope((func1,), a=1, b=None, c=[1]):
args, kwargs = func1(0)
self.assertTupleEqual(args, func1_args)
self.assertDictEqual(kwargs, func1_kwargs)
def inception_v3_arg_scope(weight_decay=0.00004,
stddev=0.1,
batch_norm_var_collection='moving_vars',
use_fused_batchnorm=True):
"""Defines the default InceptionV3 arg scope.
Args:
weight_decay: The weight decay to use for regularizing the model.
stddev: The standard deviation of the trunctated normal weight initializer.
batch_norm_var_collection: The name of the collection for the batch norm
variables.
use_fused_batchnorm: Enable fused batchnorm.
Returns:
An `arg_scope` to use for the inception v3 model.
"""
batch_norm_params = {
# Decay for the moving averages.
'decay': 0.9997,
# epsilon to prevent 0s in variance.
'epsilon': 0.001,
# collection containing update_ops.
'updates_collections': ops.GraphKeys.UPDATE_OPS,
# Use fused batch norm if possible.
'fused': use_fused_batchnorm,
# collection containing the moving mean and moving variance.
'variables_collections': {
'beta': None,
'gamma': None,
'moving_mean': [batch_norm_var_collection],
'moving_variance': [batch_norm_var_collection],
}
}
# Set weight_decay for weights in Conv and FC layers.
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
weights_regularizer=regularizers.l2_regularizer(weight_decay)):
with arg_scope(
[layers.conv2d],
weights_initializer=init_ops.truncated_normal_initializer(
stddev=stddev),
activation_fn=nn_ops.relu,
normalizer_fn=layers_lib.batch_norm,
normalizer_params=batch_norm_params) as sc:
return sc
def testNonDecorated(self):
def my_func(t, a=None):
return (t, a)
with self.assertRaises(ValueError):
with arg_scope([my_func], a=1):
pass
def gated_resnet(x, a=None, gh=None, sh=None, nonlinearity=tf.nn.elu, conv=conv2d, dropout_p=0.0, counters={}, **kwargs):
name = get_name("gated_resnet", counters)
print("construct", name, "...")
xs = int_shape(x)
num_filters = xs[-1]
with arg_scope([conv], **kwargs):
c1 = conv(nonlinearity(x), num_filters)
if a is not None: # add short-cut connection if auxiliary input 'a' is given
c1 += nin(nonlinearity(a), num_filters)
c1 = nonlinearity(c1)
c1 = tf.nn.dropout(c1, keep_prob=1. - dropout_p)
c2 = conv(c1, num_filters * 2)
# add projection of h vector if included: conditional generation
if sh is not None:
c2 += nin(sh, 2*num_filters, nonlinearity=nonlinearity)
if gh is not None: # haven't finished this part
pass
a, b = tf.split(c2, 2, 3)
c3 = a * tf.nn.sigmoid(b)
return x + c3
def slim_decoder(inputs,
batch_norm_decay,
weight_decay,
is_training,
feature_depth=256,
output_depth=256):
with tf.variable_scope('slim_decoder'):
with arg_scope([depthwise_conv2d_layer, conv2d_layer],
to_batch_norm=True,
batch_norm_decay=batch_norm_decay,
is_training=is_training,
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(weight_decay)):
net = depthwise_conv2d_layer(inputs, 3)
net = conv2d_layer(net, feature_depth, 1)
net = depthwise_conv2d_layer(net, 3)
net = conv2d_layer(net, output_depth, 1)
return net
def testRootlessFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 128, 128, 3)
with arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs,
num_classes,
global_pool=global_pool,
include_root_block=False,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 64, 64, 4],
'resnet/block2': [2, 32, 32, 8],
'resnet/block3': [2, 16, 16, 16],
'resnet/block4': [2, 16, 16, 32]
}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(
end_points[endpoint].get_shape().as_list(), shape)
def DeepLabNet(input_batch, is_training, num_classes, output_stride = 16, batch_norm_decay = 0.9997, backbone = 'resnet_v2_101'):
#Use channels_first to boost on GPU
#Deeplab V3+ with resnet as backbone
inputs_size = tf.shape(input_batch)[1:3]
with tf.variable_scope('deeplab'):
#ResNet as the encoder
with tf.variable_scope('encoder'):
if backbone == 'resnet_v2_50':
base_model = resnet_v2.resnet_v2_50
else:
base_model = resnet_v2.resnet_v2_101
#
#Implement tensorflow resnetV2
with tf.contrib.slim.arg_scope(resnet_v2.resnet_arg_scope(batch_norm_decay=batch_norm_decay)):
logits, end_points = base_model(input_batch,
num_classes=None,
is_training=is_training,
global_pool=False,
output_stride=output_stride)
#ASPP in the middle layers
with tf.variable_scope('aspp'):
net = end_points['deeplab/encoder/' + backbone + '/block4']
encoder_output = ASPP(net, output_stride, batch_norm_decay, is_training)
#
#Decoder
with tf.variable_scope('decoder'):
with tf.contrib.slim.arg_scope(resnet_v2.resnet_arg_scope(batch_norm_decay = batch_norm_decay)):
with arg_scope([layers.batch_norm], is_training=is_training):
with tf.variable_scope("low_level_features"):
low_level_features = end_points['deeplab/encoder/' + backbone + '/block1/unit_3/bottleneck_v2/conv1']
low_level_features = layers_lib.conv2d(low_level_features, 48, [1,1], stride = 1, scope = 'conv_1x1')
low_level_features_size = tf.shape(low_level_features)[1:3]
with tf.variable_scope("upsampling_logits"):
net = tf.image.resize_bilinear(encoder_output, low_level_features_size, name = 'upsample_1')
net = tf.concat([net, low_level_features], axis = 3, name = 'concat')
net = layers_lib.conv2d(net, 256, [3,3], stride = 1, scope = 'conv_3x3_1')
net = layers_lib.conv2d(net, 256, [3,3], stride = 1, scope = 'conv_3x3_2')
net = layers_lib.conv2d(net, num_classes, [1, 1], activation_fn = None, normalizer_fn = None, scope = 'conv_1x1')
logits = tf.image.resize_bilinear(net, inputs_size, name = 'upsample_2')
#
return logits
def _encoder(self,
input_images,
scope_name="encoder",
trainable=True,
scope_reuse=False):
with arg_scope(resnet_utils.resnet_arg_scope()):
output, end_points = resnet_v2.resnet_v2_50(
input_images,
output_stride=8,
global_pool=False,
reuse=scope_reuse) #(256, 256, 2048)==>(32, 32, 2048)
hidden_state = decoder_layer(
output,
out_channels=32,
stride=1,
scope_name='encoder_layer1',
trainable=trainable) #(32, 32, 2048)==>(32, 32, 32)
print hidden_state.get_shape()
tf.summary.histogram(hidden_state.op.name + "/activation",
hidden_state)
return hidden_state
def testAtrousFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
output_stride = 8
inputs = create_test_input(2, 321, 321, 3)
with arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(
inputs,
num_classes,
global_pool=global_pool,
output_stride=output_stride,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 41, 41, 4],
'resnet/block2': [2, 41, 41, 8],
'resnet/block3': [2, 41, 41, 16],
'resnet/block4': [2, 41, 41, 32]
}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def compute_est_samples(z, params=None, reuse=tf.AUTO_REUSE):
with tf.variable_scope("estimator"):
with arg_scope([nn.dense], params=params):
with tf.variable_scope("decoder", reuse=reuse):
h_dec_1 = nn.dense(z,
vae_z_dim,
200 * 2,
"dense1",
nonlinearity=nonlin)
h_dec_2 = nn.dense(h_dec_1,
200 * 2,
500 * 2,
"dense2",
nonlinearity=nonlin)
x_mean = nn.dense(h_dec_2,
500 * 2,
x_dim,
"dense3",
nonlinearity=None)
x_mean = tf.nn.tanh(x_mean)
return x_mean
def actnorm(
name,
x,
scale=1.,
logdet=None,
logscale_factor=3.,
batch_variance=False,
reverse=False,
init=False,
trainable=True):
if arg_scope([get_variable_ddi], trainable=trainable):
if not reverse:
x = actnorm_center(name + "_center", x, reverse)
x = actnorm_scale(
name + "_scale",
x,
scale,
logdet,
logscale_factor,
batch_variance,
reverse,
init)
if logdet is not None:
x, logdet = x
else:
x = actnorm_scale(
name + "_scale",
x,
scale,
logdet,
logscale_factor,
batch_variance,
reverse,
init)
if logdet is not None:
x, logdet = x
x = actnorm_center(name + "_center", x, reverse)
if logdet is not None:
return x, logdet
return x
def slim_net_original(image, keep_prob):
with arg_scope(
[layers.conv2d, layers.fully_connected],
biases_initializer=tf.random_normal_initializer(stddev=0.1)):
# conv2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME',
# activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None,
# weights_initializer=initializers.xavier_initializer(), weights_regularizer=None,
# biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, scope=None):
net = layers.conv2d(
image,
32, [5, 5],
scope='conv1',
weights_regularizer=regularizers.l1_regularizer(0.5))
# max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None)
net = layers.max_pool2d(net, 2, scope='pool1')
net = layers.conv2d(
net,
64, [5, 5],
scope='conv2',
weights_regularizer=regularizers.l2_regularizer(0.5))
summaries.summarize_tensor(net, tag='conv2')
net = layers.max_pool2d(net, 2, scope='pool2')
net = layers.flatten(net, scope='flatten1')
# fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None,
# normalizer_params=None, weights_initializer=initializers.xavier_initializer(),
# weights_regularizer=None, biases_initializer=init_ops.zeros_initializer,
# biases_regularizer=None, scope=None):
net = layers.fully_connected(net, 1024, scope='fc1')
# dropout(inputs, keep_prob=0.5, is_training=True, scope=None)
net = layers.dropout(net, keep_prob=keep_prob, scope='dropout1')
net = layers.fully_connected(net, 10, scope='fc2')
return net
def _grad_fn(output_grads, variables=None):
"""Recompute outputs for gradient computation."""
variables = variables or []
if original_vars:
assert variables, (
"Fn created variables but the variables were not "
"passed to the gradient fn.")
if set(variables) != original_vars:
raise ValueError(_WRONG_VARS_ERR)
inputs = [array_ops.identity(x) for x in list(args)]
# Recompute outputs
with framework_ops.control_dependencies(output_grads):
if use_data_dep_:
inputs = _force_data_dependency(output_grads, inputs)
with contrib_framework_ops.arg_scope(arg_scope):
with variable_scope.variable_scope(vs, reuse=True):
with backprop.GradientTape() as tape:
fn_kwargs = {}
if has_is_recompute_kwarg:
fn_kwargs["is_recomputing"] = True
outputs = fn(*inputs, **fn_kwargs)
recompute_vars = set(tape.watched_variables())
if original_vars != recompute_vars:
raise ValueError(_WRONG_VARS_ERR)
if not isinstance(outputs, (list, tuple)):
outputs = [outputs]
outputs = list(outputs)
grads = gradients_impl.gradients(outputs, inputs + variables,
output_grads)
if tupleize_grads:
if use_data_dep_:
grads = _tuple_with_data_dep(grads)
else:
grads = control_flow_ops.tuple(grads)
grad_inputs = grads[:len(inputs)]
grad_vars = grads[len(inputs):]
return grad_inputs, grad_vars
def _model(self, inputs):
kernel_initializer = tf.contrib.layers.xavier_initializer()
out = tf.reshape(inputs, (-1, 28, 28, 1))
with arg_scope([conv2d, dense],
kernel_initializer=kernel_initializer,
activation=tf.nn.relu,
norm="None",
is_training=self.is_training):
out = conv2d(out, 32, 5, strides=1, padding='SAME')
out = tf.nn.max_pool(out,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
out = conv2d(out, 64, 5, strides=1, padding='SAME')
out = tf.nn.max_pool(out,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
out = tf.layers.Flatten()(out)
out = dense(out, 1024)
out = dense(out, self.num_classes, activation=None)
return out
def separate_conv_layer(self, inputs, num_output_channels, kernel_size, stride, rate,
layer_name, full_layer_name):
with arg_scope(
[slim.conv2d],
weights_regularizer=None,
weights_initializer=None,
trainable=False,
activation_fn=None,
normalizer_fn=None,
normalizer_params=None,
biases_initializer=None): #make first layer clean, no BN no biases no activation func
K = self._K_by_layer_dict[full_layer_name]
layer1_name = LayerName(layer_name.replace('conv', 'convsep'))
net = conv2d_same(inputs, K, kernel_size=(kernel_size,1), stride=[stride,1],
scope=layer1_name)
with slim.arg_scope(resnet_arg_scope(is_training=False)):
net = conv2d_same(net, num_output_channels, kernel_size=(1,kernel_size),
stride=[1,stride], scope=layer_name)
return net
def objective_tower(self, features, init=True):
"""Objective in terms of bits-per-pixel.
"""
#features = tf.expand_dims(features, [1, 2])
features = features[:, tf.newaxis, tf.newaxis, :]
x = features
objective = 0
# The arg_scope call ensures that the actnorm parameters are set such that
# the per-channel output activations have zero mean and unit variance
# ONLY during the first step. After that the parameters are learned
# through optimisation.
ops = [
glow_ops.get_variable_ddi, glow_ops.actnorm, glow_ops.get_dropout
]
with arg_scope(ops, init=init):
encoder = glow_ops.encoder_decoder
self.z, encoder_objective, self.eps, _, _ = encoder("flow",
x,
self.hparams,
eps=None,
reverse=False)
objective += encoder_objective
self.z_top_shape = get_shape_list(self.z)
prior_dist = self.top_prior()
prior_objective = tf.reduce_sum(prior_dist.log_prob(self.z),
axis=[1, 2, 3])
#self.z_sample = prior_dist.sample()
objective += prior_objective
# bits per pixel
_, h, w, c = get_shape_list(x)
objective = -objective / (np.log(2) * h * w * c)
self.z = tf.concat(self.eps + [self.z], axis=-1)
return objective
def test_layer(layer, layer_kwargs):
shape = [128, 32, 32, 3]
x = tf.placeholder(tf.float32, shape, name='image')
logdet = tf.zeros_like(x)[:, 0, 0, 0]
with arg_scope([get_conv_weight_np], unit_testing=True):
with tf.variable_scope('test'):
z, logdet = layer('layer', x, logdet, reverse=False, **kwargs)
with tf.variable_scope('test', reuse=True):
recon, logdet_out = layer('layer',
z,
logdet,
reverse=True,
**kwargs)
print('layer', layer)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
x_np = np.random.randn(*shape).astype('float32')
z_np, recon_np, logdet_np = sess.run([z, recon, logdet_out],
feed_dict={x: x_np})
z_recon_np = sess.run([z], feed_dict={x: recon_np})
def rmse(a, b):
return np.sqrt(np.mean(np.power(a - b, 2)))
print('RMSE on x:\t', rmse(x_np, recon_np))
print('RMSE on conv(x):\t', rmse(z_np, z_recon_np))
print('log det: \t', rmse(logdet_np, 0))
print('')
tf.reset_default_graph()
def grad_fn(*output_grads, **kwargs):
"""Recompute outputs for gradient computation."""
variables = []
if original_vars:
variables = kwargs["variables"]
if set(variables) != original_vars:
raise ValueError(_WRONG_VARS_ERR)
del kwargs
inputs = list(args)
# Recompute outputs
with framework_ops.control_dependencies(output_grads):
if use_data_dep_:
inputs = _force_data_dependency(output_grads, inputs)
with contrib_framework_ops.arg_scope(arg_scope):
with variable_scope.variable_scope(vs, reuse=True):
with backprop.GradientTape() as tape:
fn_kwargs = {}
if has_is_recompute_kwarg:
fn_kwargs["is_recomputing"] = True
outputs = fn(*inputs, **fn_kwargs)
recompute_vars = set(tape.watched_variables())
if original_vars != recompute_vars:
raise ValueError(_WRONG_VARS_ERR)
if not (isinstance(outputs, list) or isinstance(outputs, tuple)):
outputs = [outputs]
outputs = list(outputs)
grads = gradients_impl.gradients(outputs, inputs + variables,
output_grads)
if tupleize_grads:
if use_data_dep_:
grads = _tuple_with_data_dep(grads)
else:
grads = control_flow_ops.tuple(grads)
grad_inputs = grads[:len(inputs)]
grad_vars = grads[len(inputs):]
return grad_inputs, grad_vars
def cnnet(input_patch_img, label, is_training):
with arg_scope([layers.conv2d], padding='SAME',
normalizer_fn=layers.batch_norm, normalizer_params={"is_training": is_training}):
net = tf.image.resize_images(input_patch_img, [256, 256], method=2)
net = layers.conv2d(net, 8, [3, 3], stride=[2, 2], scope='convd1') # Nm
net = layers.conv2d(net, 32, [7, 7], scope='convd2') #128 128 32 # strides默认为1，激活函数默认为relu
net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') # 池化窗口为3，步长为2
net = layers.conv2d(net, 32, [3, 3], scope='convd3')
net_m = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
net_a = tf.nn.avg_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
net = tf.concat((net_m, net_a), axis=3)
net = layers.conv2d(net, 64, [3, 3], scope='convd4')
net_m = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
net_a = tf.nn.avg_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
net = tf.concat((net_m, net_a), axis=3)
net = layers.conv2d(net, 128, [3, 3], scope='convd5')
net_m = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
net_a = tf.nn.avg_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
net = tf.concat((net_m, net_a), axis=3)
net = tf.nn.avg_pool(net, ksize=[1,8,8,1], strides=[1,8,8,1], padding="SAME") # Global average pooling
net = tf.reshape(net, [-1,256])
net = layers.fully_connected(net, 1024)
output = layers.fully_connected(net, 1, activation_fn=None)
score = output
# label = label*np.ones(num_patch)
label = tf.reshape(label, [-1, 1])
loss = tf.losses.mean_squared_error(label, output)
return score, loss # output,loss
def vq_decoder_spec(quant_t,
quant_b,
ema=None,
nr_channel=128,
nr_res_block=2,
nr_res_channel=64,
embedding_dim=64):
"""
Input:
Tensor quant_t of shape (N,H//8,W//8,C) (e.g. (128,32,32,64))
Tensor quant_b of shape (N,H//4,W//4,C) (e.g. (128,64,64,64))
Output:
Tensor dec_b of shape (N,H,W,3) (e.g. (128,256,256,3))
"""
counters = {}
with arg_scope([nn.conv2d, nn.deconv2d], counters=counters, ema=ema):
# Bottom decoder
quant_t = nn.deconv2d(quant_t,
embedding_dim,
filter_size=[4, 4],
stride=[2, 2])
dec_b = tf.concat([quant_b, quant_t], -1)
dec_b = nn.conv2d(dec_b, nr_channel)
for rep in range(nr_res_block):
dec_b = nn.resnet(dec_b,
num_res_channel=nr_res_channel,
nonlinearity=tf.nn.elu)
dec_b = tf.nn.elu(dec_b)
dec_b = nn.deconv2d(dec_b,
nr_channel // 2,
filter_size=[4, 4],
stride=[2, 2])
dec_b = tf.nn.elu(dec_b)
dec_b = nn.deconv2d(dec_b, 3, filter_size=[4, 4], stride=[2, 2])
return {'dec_b': dec_b}
def __call__(self, features):
""" Define tf graph.
"""
inputs = features['image']
with tf.variable_scope('encoder') as vsc:
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
# conv1
with arg_scope([layers_lib.conv2d],
activation_fn=None,
normalizer_fn=None):
net = resnet_utils.conv2d_same(inputs,
16,
5,
stride=2,
scope='conv1')
tf.add_to_collection(vsc.original_name_scope, net)
# resnet blocks
blocks = []
for i in range(len(self.encoder_params['block_name'])):
block = resnet_v2.resnet_v2_block(
scope=self.encoder_params['block_name'][i],
base_depth=self.encoder_params['base_depth'][i],
num_units=self.encoder_params['num_units'][i],
stride=self.encoder_params['stride'][i])
blocks.append(block)
net, _ = resnet_v2.resnet_v2(
net,
blocks,
is_training=(self.mode == ModeKeys.TRAIN),
global_pool=False,
output_stride=2,
include_root_block=False,
scope='resnet')
tf.add_to_collection(vsc.original_name_scope, net)
return net
def segmentation_discriminator(inputs, batch_norm_decay, weight_decay,
is_training):
with tf.variable_scope('discriminator', reuse=tf.AUTO_REUSE):
with arg_scope([fc_layer, conv2d_layer, global_avg_pooling_layer],
to_batch_norm=True,
batch_norm_decay=batch_norm_decay,
is_training=is_training,
activation_fn=tf.nn.leaky_relu,
weights_regularizer=slim.l2_regularizer(weight_decay)):
net = conv2d_layer(inputs, 32, 5, strides=[1, 2, 2,
1]) # 256 x 256
net = conv2d_layer(net, 64, 5, strides=[1, 2, 2, 1]) # 128 x 128
net = conv2d_layer(net, 128, 3, strides=[1, 2, 2, 1]) # 64 x 64
net = conv2d_layer(net, 256, 3, strides=[1, 2, 2, 1]) # 32 x 32
net = conv2d_layer(net, 512, 3, strides=[1, 2, 2, 1]) # 16 x 16
net = global_avg_pooling_layer(net, upsample=False, keepdims=False)
net = fc_layer(net,
2,
to_batch_norm=False,
activation_fn=tf.nn.sigmoid)
return net
def compute_est_ll(x, params=None, reuse=tf.AUTO_REUSE):
with tf.variable_scope("estimator"):
with arg_scope([nn.dense], params=params):
with tf.variable_scope("encoder", reuse=reuse):
h_enc_1 = nn.dense(x, 2, 128, "dense1", nonlinearity=nonlin)
# h_enc_1 = nn.batch_norm(h_enc_1, "bn1", 128, 2)
h_enc_2 = nn.dense(h_enc_1, 128, 128, "dense2", nonlinearity=nonlin)
# h_enc_2 = nn.batch_norm(h_enc_2, "bn2", 128, 2)
z_mean = nn.dense(h_enc_2, 128, vae_z_dim, "dense3", nonlinearity=None)
z_logvar = nn.dense(h_enc_2, 128, vae_z_dim, "dense4", nonlinearity=None)
epsilon = tf.random_normal(tf.shape(z_mean), dtype=tf.float32)
z = z_mean + tf.exp(0.5 * z_logvar) * epsilon
with tf.variable_scope("decoder", reuse=reuse):
h_dec_1 = nn.dense(z, vae_z_dim, 128, "dense1", nonlinearity=nonlin)
# h_dec_1 = nn.batch_norm(h_dec_1, "bn1", 128, 2)
h_dec_2 = nn.dense(h_dec_1, 128, 128, "dense2", nonlinearity=nonlin)
# h_dec_2 = nn.batch_norm(h_dec_2, "bn2", 128, 2)
x_mean = nn.dense(h_dec_2, 128, x_dim, "dense3", nonlinearity=None)
elbo = tf.reduce_mean(tf.reduce_sum(- 0.5 * np.log(2 * np.pi) - 0.5 * np.log(vae_x_var) - tf.square(x - x_mean) / (
2 * vae_x_var), axis=1) - tf.reduce_sum(- 0.5 * (1 + z_logvar - tf.square(z_mean) - tf.exp(z_logvar)), axis=1))
return elbo, x_mean
def Model(x, is_training, init=False, ema=None):
norm_prms = {'is_training': is_training}
with arg_scope([conv2d], normalizer_fn=batch_norm,
activation_fn=lrelu,
normalizer_params=norm_prms):
ly_x = tf.reshape(x, [-1, 32, 32, 3])
ly_x = GaussianNoise(ly_x, sigma=flgs.augment_noise_stddev,
is_training=is_training)
ly_x = conv2d(ly_x, 64, 3)
ly_x = conv2d(ly_x, 64, 3)
ly_x = conv2d(ly_x, 64, 3)
ly_x = max_pool2d(ly_x, kernel_size=2)
ly_x = dropout(ly_x, keep_prob=0.5, is_training=is_training)
ly_x = conv2d(ly_x, 128, 3)
ly_x = conv2d(ly_x, 128, 3)
ly_x = conv2d(ly_x, 128, 3)
ly_x = max_pool2d(ly_x, kernel_size=2)
ly_x = dropout(ly_x, keep_prob=0.5, is_training=is_training)
ly_x = conv2d(ly_x, 256, 3, padding='VALID')
ly_x_top = conv2d(ly_x, 128, 1)
ly_x = tf.reduce_mean(ly_x_top, axis=[1, 2])
class_logits = fully_connected(ly_x, 10, activation_fn=None)
return class_logits, ly_x, ly_x_top
def decoder(self, encoder_output, batch_norm_decay, is_training=True):
with tf.variable_scope("decoder"):
with tf.contrib.slim.arg_scope(
resnet_v2.resnet_arg_scope(
batch_norm_decay=batch_norm_decay)):
with arg_scope([layers.batch_norm], is_training=is_training):
with tf.variable_scope("low_level_features"):
low_level_features = self.visual_feat_c2
low_level_features = layers_lib.conv2d(
low_level_features,
48, [1, 1],
stride=1,
scope='conv_1x1')
low_level_features_size = tf.shape(
low_level_features)[1:3]
with tf.variable_scope("upsampling_logits"):
net = tf.image.resize_bilinear(encoder_output,
low_level_features_size,
name='upsample_1')
net = tf.concat([net, low_level_features],
axis=3,
name='concat')
net = layers_lib.conv2d(net,
256, [3, 3],
stride=1,
scope='conv_3x3_1')
net = layers_lib.conv2d(net,
256, [3, 3],
stride=1,
scope='conv_3x3_2')
net = layers_lib.conv2d(net,
1, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='conv_1x1')
return net
def ar_conv2d(name,
x,
num_filters,
filter_size=(3, 3),
stride=(1, 1),
pad="SAME",
init_scale=1.,
zerodiagonal=True,
**_):
h = filter_size[0]
w = filter_size[1]
n_in = int(x.get_shape()[1])
n_out = num_filters
mask = tf.constant(get_conv_ar_mask(h, w, n_in, n_out, zerodiagonal))
with arg_scope([conv2d]):
return conv2d(name,
x,
num_filters,
filter_size,
stride,
pad,
init_scale,
mask=mask)
def generator(z,
training=True,
weight_decay=0.0001,
batch_norm_decay=0.997,
batch_norm_epsilon=1e-5,
batch_norm_scale=True):
batch_norm_params = {
'is_training': training,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'updates_collections': ops.GraphKeys.UPDATE_OPS,
}
gen = slim.fully_connected(gen, 1024)
gen = tf.reshape(gen, [batch_size, 1, 1, 1024])
with arg_scope(
[slim.conv2d_transpose],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params,
scope="transposed_convolution"):
# Each tuple is (number of channels, kernel size, stride)
l = [(1024, [3, 3], [2, 2]), (512, [3, 3], [2, 2]),
(256, [3, 3], [2, 2]), (128, [3, 3], [2, 2]), (128, [3,
3], [2, 2]),
(64, [3, 3], [2, 2]), (3, [3, 3], [2, 2])]
gen = slim.stack(gen, slim.conv2d_transpose, l)
gen = tf.tanh(gen)
return gen
def actnorm_scale(name, x, scale=1., logdet=None, logscale_factor=3.,
batch_variance=False, reverse=False, init=False,
trainable=True):
shape = x.get_shape()
with tf.variable_scope(name), arg_scope([get_variable_ddi],
trainable=trainable):
assert len(shape) == 2 or len(shape) == 4
if len(shape) == 2:
x_var = tf.reduce_mean(x ** 2, [0], keepdims=True)
logdet_factor = 1
_shape = (1, int_shape(x)[1])
elif len(shape) == 4:
x_var = tf.reduce_mean(x ** 2, [0, 1, 2], keepdims=True)
logdet_factor = int(shape[1]) * int(shape[2])
_shape = (1, 1, 1, int_shape(x)[3])
if batch_variance:
x_var = tf.reduce_mean(x ** 2, keepdims=True)
logs = get_variable_ddi("logs", _shape, initial_value=tf.log(
scale / (tf.sqrt(
x_var) + 1e-6)) / logscale_factor) * logscale_factor
if not reverse:
x = x * tf.exp(logs)
else:
x = x * tf.exp(-logs)
if logdet != None:
dlogdet = tf.reduce_sum(logs) * logdet_factor
if reverse:
dlogdet *= -1
return x, logdet + dlogdet
return x
def testAtrousFullyConvolutionalValues(self):
"""Verify dense feature extraction with atrous convolution."""
nominal_stride = 32
for output_stride in [4, 8, 16, 32, None]:
with arg_scope(resnet_utils.resnet_arg_scope(is_training=False)):
with ops.Graph().as_default():
with self.test_session() as sess:
random_seed.set_random_seed(0)
inputs = create_test_input(2, 81, 81, 3)
# Dense feature extraction followed by subsampling.
output, _ = self._resnet_small(
inputs, None, global_pool=False, output_stride=output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
variable_scope.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected, _ = self._resnet_small(inputs, None, global_pool=False)
sess.run(variables.global_variables_initializer())
self.assertAllClose(
output.eval(), expected.eval(), atol=1e-4, rtol=1e-4)
def texture_discriminator_spec(x, nr_channel=64):
"""
Input:
Tensor x of shape (2*N,H,W,4) (e.g. (16,256,256,4))
Output:
Tensor x_out of shape (2*N,(H/64)*(W/64)*C") (e.g. (16,4*4*256))
"""
counters = {}
with arg_scope([nn.snconv2d],
filter_size=[5, 5],
stride=[2, 2],
nonlinearity=tf.nn.leaky_relu,
counters=counters):
cnum = nr_channel
x = nn.snconv2d(x, cnum)
x = nn.snconv2d(x, cnum * 2)
x = nn.snconv2d(x, cnum * 4)
x = nn.snconv2d(x, cnum * 4)
x = nn.snconv2d(x, cnum * 4)
x = nn.snconv2d(x, cnum * 4)
x_out = tf.layers.flatten(x)
return x_out
def roi_fc(inputs, boxes, box_idx, scope='vgg_16'):
with variable_scope.variable_scope(scope, 'vgg_16', [inputs]) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
# Collect outputs for conv2d, fully_connected and max_pool2d.
with arg_scope(
[layers.conv2d, layers_lib.fully_connected, layers_lib.max_pool2d],
outputs_collections=end_points_collection):
# Use conv2d instead of fully_connected layers.
net = roi_pooling(inputs, boxes, box_idx)
net = layers.conv2d(net,
4096, [7, 7],
padding='VALID',
scope='fc6')
net = layers_lib.dropout(net,
0.5,
is_training=True,
scope='dropout6')
net = layers.conv2d(net, 4096, [1, 1], scope='fc7')
net = layers_lib.dropout(net,
0.5,
is_training=True,
scope='dropout7')
return net
def testReuseArgScopeNested(self):
func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
func2_kwargs = {'b': 2, 'd': [2]}
key = _key_op
current_scope1 = {key(func1): func1_kwargs.copy()}
current_scope2 = {
key(func1): func1_kwargs.copy(),
key(func2): func2_kwargs.copy()
}
with self.cached_session():
with arg_scope([func1], a=1, b=None, c=[1]) as scope1:
with arg_scope([func2], b=2, d=[2]) as scope2:
pass
with arg_scope(scope1):
with arg_scope([]) as current_arg_scope:
self.assertDictEqual(current_arg_scope, current_scope1)
with arg_scope(scope2):
with arg_scope([]) as current_arg_scope:
self.assertDictEqual(current_arg_scope, current_scope2)
def resnet_v2(
inputs, blocks, num_classes=None, is_training=True, global_pool=True, output_stride=None,
include_root_block=True, centered_stride=False, reuse=None, scope=None):
"""Generator for v2 (preactivation) ResNet models.
This function generates a family of ResNet v2 models. See the resnet_v2_*()
methods for specific model instantiations, obtained by selecting different
block instantiations that produce ResNets of various depths.
Training for image classification on Imagenet is usually done with [224, 224]
inputs, resulting in [7, 7] feature maps at the output of the last ResNet
block for the ResNets defined in [1] that have nominal stride equal to 32.
However, for dense prediction tasks we advise that one uses inputs with
spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In
this case the feature maps at the ResNet output will have spatial shape
[(height - 1) / output_stride + 1, (width - 1) / output_stride + 1]
and corners exactly aligned with the input image corners, which greatly
facilitates alignment of the features to the image. Using as input [225, 225]
images results in [8, 8] feature maps at the output of the last ResNet block.
For dense prediction tasks, the ResNet needs to run in fully-convolutional
(FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all
have nominal stride equal to 32 and a good choice in FCN mode is to use
output_stride=16 in order to increase the density of the computed features at
small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915.
Args:
inputs: A tensor of size [batch, height_in, width_in, channels].
blocks: A list of length equal to the number of ResNet blocks. Each element
is a resnet_utils.Block object describing the units in the block.
num_classes: Number of predicted classes for classification tasks. If None
we return the features before the logit layer.
is_training: whether batch_norm layers are in training mode.
global_pool: If True, we perform global average pooling before computing the
logits. Set to True for image classification, False for dense prediction.
output_stride: If None, then the output will be computed at the nominal
network stride. If output_stride is not None, it specifies the requested
ratio of input to output spatial resolution.
include_root_block: If True, include the initial convolution followed by
max-pooling, if False excludes it. If excluded, `inputs` should be the
results of an activation-less convolution.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
Returns:
net: A rank-4 tensor of size [batch, height_out, width_out, channels_out].
If global_pool is False, then height_out and width_out are reduced by a
factor of output_stride compared to the respective height_in and width_in,
else both height_out and width_out equal one. If num_classes is None, then
net is the output of the last ResNet block, potentially after global
average pooling. If num_classes is not None, net contains the pre-softmax
activations.
end_points: A dictionary from components of the network to the corresponding
activation.
Raises:
ValueError: If the target output_stride is not valid.
"""
with variable_scope.variable_scope(
scope, 'resnet_v2', [inputs], reuse=reuse) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
with arg_scope(
[layers_lib.conv2d, bottleneck, resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with arg_scope([layers.batch_norm], is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError('The output_stride needs to be a multiple of 4.')
output_stride /= 4
# We do not include batch normalization or activation functions in
# conv1 because the first ResNet unit will perform these. Cf.
# Appendix of [2].
with arg_scope([layers_lib.conv2d], activation_fn=None, normalizer_fn=None):
net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1')
net = resnet_utils.max_pool2d_same(
net, 3, stride=2, scope='pool1',
centered_stride=centered_stride and output_stride == 4)
net = resnet_utils.stack_blocks_dense(net, blocks, output_stride)
# This is needed because the pre-activation variant does not have batch
# normalization or activation functions in the residual unit output. See
# Appendix of [2].
net = slim.batch_norm(net, activation_fn=nn_ops.relu, scope='postnorm')
if global_pool:
# Global average pooling.
net = math_ops.reduce_mean(net, tfu.image_axes(), name='pool5', keepdims=True)
if num_classes is not None:
net = layers_lib.conv2d(
net, num_classes, [1, 1], activation_fn=None, normalizer_fn=None,
scope='logits')
# Convert end_points_collection into a dictionary of end_points.
end_points = utils.convert_collection_to_dict(end_points_collection)
if num_classes is not None:
end_points['predictions'] = layers.softmax(net, scope='predictions')
return net, end_points
