def _AssertIdempotent(self, graph):
# Ensure that calling the rewrite again doesn't change the graph.
graph_def_before = str(graph.as_graph_def())
with graph.as_default():
# Ensuring that calling the rewrite again doesn't add more nodes.
fold_batch_norms.FoldBatchNorms(graph, is_training=True)
quantize.Quantize(graph, True)
graph_def_after = str(graph.as_graph_def())
self.assertEqual(graph_def_before, graph_def_after)
def _create_graph(input_graph=None,
is_training=True,
weight_bits=8,
activation_bits=8,
symmetric=False,
quant_delay=None,
freeze_bn_delay=None,
scope=None):
"""Rewrites an input_graph in place for simulated quantization.
The graph has fake quantization ops inserted to simulate the error
introduced by quantization. Since the graph is transformed in place,
the expected behavior of previously held references to nodes and tensors may
change.
Args:
input_graph: The tf.Graph to be transformed, if None then defaults to the
default graph.
is_training: Whether quantizing training or eval graph.
weight_bits: Number of bits to use for quantizing weights.
activation_bits: Number of bits to use for quantizing activations.
symmetric: If true, use symmetric quantization limits instead of training
the minimum and maximum of each quantization range separately.
quant_delay: Number of steps after which weights and activations are
quantized during training.
freeze_bn_delay: Number of steps after which moving mean and variance are
frozen and used instead of batch statistics during training.
freeze_bn_delay should be greater than quant_delay and should correspond
to the number of steps when training has almost converged
scope: The scope to be transformed. If it's not None, only the ops which
are in this scope will be transformed.
Raises:
ValueError: If elements contains an element that isn't a tf.Tensor or
tf.Operation.
"""
if input_graph is None:
input_graph = ops.get_default_graph()
# Add check to see if graph has training ops, if so provide error message and
# exit
_check_for_training_ops(input_graph)
with input_graph.as_default():
fold_batch_norms.FoldBatchNorms(
input_graph,
freeze_batch_norm_delay=freeze_bn_delay,
is_training=is_training)
quantize.Quantize(input_graph,
is_training,
quant_delay=quant_delay,
weight_bits=weight_bits,
activation_bits=activation_bits,
symmetric=symmetric,
scope=scope)
def _TestQuantize_AtrousConvWithBatchNorm(
self, activation, activation_op_name, with_bypass, delay,
fused_batch_norm, use_resource, scope):
"""Tests quantization: inputs -> atrous conv with batch norm -> Activation.
Args:
activation: Callable that returns an Operation, a factory method for the
Activation.
activation_op_name: String, name of the Activation operation.
with_bypass: Bool, when true there is an extra connection added from
inputs to just before Activation.
delay: Int (optional), delay in number of steps until quantization starts.
fused_batch_norm: Bool, when true use FusedBatchNorm.
use_resource: Bool, when true uses resource variables.
scope: String, specifies top level scope for the graph
"""
graph = ops.Graph()
with graph.as_default():
variable_scope.get_variable_scope().set_use_resource(use_resource)
batch_size, height, width, depth = 5, 128, 128, 3
inputs = array_ops.zeros((batch_size, height, width, depth))
dilation_rate = 2
conv_scope = self._GetConvScope(scope, with_bypass)
scope = '' if scope is None else scope
delim = '/' if scope else ''
node = separable_conv2d(
inputs,
None, [3, 3],
rate=dilation_rate,
depth_multiplier=1.0,
padding='SAME',
weights_initializer=self._WeightInit(0.09),
activation_fn=None,
normalizer_fn=batch_norm,
normalizer_params=self._BatchNormParams(fused_batch_norm),
scope=conv_scope)
# Manually add a bypass (optional) and an activation.
if with_bypass:
node = math_ops.add(inputs, node, name=scope + delim + 'AddV2')
node = activation(node, name=scope + delim + activation_op_name)
update_barrier = control_flow_ops.no_op(name='update_barrier')
with ops.control_dependencies([update_barrier]):
array_ops.identity(node, name='control_dependency')
fold_batch_norms.FoldBatchNorms(graph, is_training=True)
quantize.Quantize(graph, True, quant_delay=delay)
self._AssertCorrectQuantizedGraphWithBatchNorm(
graph, scope, 'DepthwiseConv2dNative', activation_op_name,
with_bypass, delay, use_resource)
def _TestBatchNormForcedUpdates(self, activation, activation_op_name,
fused_batch_norm, use_resource):
"""post_activation bypass quantization should happen with forced updates."""
graph = ops.Graph()
with graph.as_default():
variable_scope.get_variable_scope().set_use_resource(use_resource)
batch_size, height, width, depth = 5, 128, 128, 3
input1 = array_ops.zeros((batch_size, height, width, depth))
input2 = array_ops.zeros((batch_size, height / 2, width / 2, 32))
# Setting updates_collections to None forces updates adding an extra
# identity operation following batch norms.
bn_params = self._BatchNormParams(
fused=fused_batch_norm, force_updates=True)
conv = conv2d(
input1,
32, [5, 5],
stride=2,
padding='SAME',
weights_initializer=self._WeightInit(0.09),
activation_fn=activation,
normalizer_fn=batch_norm,
normalizer_params=bn_params,
scope='test/test')
bypass_tensor = math_ops.add(conv, input2, name='test/add')
# The output of the post_activation bypass will be another layer.
_ = conv2d(
bypass_tensor,
32, [5, 5],
stride=2,
padding='SAME',
weights_initializer=self._WeightInit(0.09),
normalizer_fn=batch_norm,
normalizer_params=bn_params,
activation_fn=activation,
scope='test/unused')
fold_batch_norms.FoldBatchNorms(graph, is_training=True)
quantize.Quantize(graph, is_training=True)
# Ensure that the bypass node is preceded by and followed by a
# FakeQuantWithMinMaxVar operation, since the output of the Add isn't an
# activation.
self.assertTrue('FakeQuantWithMinMaxVars' in
[c.type for c in bypass_tensor.consumers()])
self.assertTrue('FakeQuantWithMinMaxVars' in
[i.op.type for i in bypass_tensor.op.inputs])
with open('/tmp/bn_quant_test.pbtxt', 'w') as f:
f.write(str(graph.as_graph_def()))
def _TestFoldConv2d(self, relu, relu_op_name, with_bypass, has_scaling,
fused_batch_norm, freeze_batch_norm_delay,
insert_identity_node):
"""Tests folding cases: inputs -> Conv2d with batch norm -> Relu*.
Args:
relu: Callable that returns an Operation, a factory method for the Relu*.
relu_op_name: String, name of the Relu* operation.
with_bypass: Bool, when true there is an extra connection added from
inputs to just before Relu*.
has_scaling: Bool, when true the batch norm has scaling.
fused_batch_norm: Bool, when true the batch norm is fused.
freeze_batch_norm_delay: None or the number of steps after which training
switches to using frozen mean and variance
insert_identity_node: Bool, insert identity node between conv and batch
norm
"""
g = ops.Graph()
with g.as_default():
batch_size, height, width = 5, 128, 128
inputs = array_ops.zeros((batch_size, height, width, 3))
out_depth = 3 if with_bypass else 32
stride = 1 if with_bypass else 2
activation_fn = None if with_bypass else relu
name = 'test/test2' if with_bypass else 'test'
if insert_identity_node:
with g.name_scope(name):
node = conv2d(inputs,
out_depth, [5, 5],
stride=stride,
padding='SAME',
weights_initializer=self._WeightInit(0.09),
activation_fn=None,
normalizer_fn=None,
biases_initializer=None)
conv_out = array_ops.identity(node, name='conv_out')
node = batch_norm(conv_out,
center=True,
scale=has_scaling,
decay=1.0 - 0.003,
fused=fused_batch_norm)
if activation_fn is not None:
node = activation_fn(node)
conv_name = name + '/Conv'
else:
node = conv2d(inputs,
out_depth, [5, 5],
stride=stride,
padding='SAME',
weights_initializer=self._WeightInit(0.09),
activation_fn=activation_fn,
normalizer_fn=batch_norm,
normalizer_params=self._BatchNormParams(
scale=has_scaling, fused=fused_batch_norm),
scope=name)
conv_name = name
if with_bypass:
node = math_ops.add(inputs, node, name='test/AddV2')
relu(node, name='test/' + relu_op_name)
fold_batch_norms.FoldBatchNorms(
g,
is_training=True,
freeze_batch_norm_delay=freeze_batch_norm_delay)
folded_mul = g.get_operation_by_name(conv_name + '/mul_fold')
self.assertEqual(folded_mul.type, 'Mul')
self._AssertInputOpsAre(folded_mul, [
conv_name + '/correction_mult',
self._BatchNormMultiplierName(conv_name, has_scaling,
fused_batch_norm)
])
self._AssertOutputGoesToOps(folded_mul, g,
[conv_name + '/Conv2D_Fold'])
folded_conv = g.get_operation_by_name(conv_name + '/Conv2D_Fold')
self.assertEqual(folded_conv.type, 'Conv2D')
self._AssertInputOpsAre(folded_conv,
[conv_name + '/mul_fold', inputs.op.name])
self._AssertOutputGoesToOps(folded_conv, g,
[conv_name + '/post_conv_mul'])
folded_add = g.get_operation_by_name(conv_name + '/add_fold')
self.assertEqual(folded_add.type, 'Add')
self._AssertInputOpsAre(folded_add, [
conv_name + '/correction_add',
self._BathNormBiasName(conv_name, fused_batch_norm)
])
output_op_names = [
'test/AddV2' if with_bypass else 'test/' + relu_op_name
]
self._AssertOutputGoesToOps(folded_add, g, output_op_names)
if freeze_batch_norm_delay is not None:
self._AssertMovingAveragesAreFrozen(g, name)
for op in g.get_operations():
self.assertFalse('//' in op.name,
'Double slash in op %s' % op.name)
def _TestCompareFoldAndUnfolded(self,
relu,
relu_op_name,
with_bypass,
has_scaling,
fused_batch_norm,
freeze_batch_norm_delay,
insert_identity_node=False):
"""Tests that running folded and unfolded BN returns the same results.
Args:
relu: Callable that returns an Operation, a factory method for the Relu*.
relu_op_name: String, name of the Relu* operation.
with_bypass: Bool, when true there is an extra connection added from
inputs to just before Relu*.
has_scaling: Bool, when true the batch norm has scaling.
fused_batch_norm: Bool, when true the batch norm is fused.
freeze_batch_norm_delay: None or the number of steps after which training
switches to using frozen mean and variance
insert_identity_node: Bool, insert identity node between conv and batch
norm
"""
random_seed.set_random_seed(1234)
unfolded_g = ops.Graph()
with unfolded_g.as_default():
batch_size, height, width = 5, 128, 128
inputs = random_ops.random_uniform((batch_size, height, width, 3),
dtype=dtypes.float32,
seed=1234)
out_depth = 3 if with_bypass else 32
stride = 1 if with_bypass else 2
activation_fn = None if with_bypass else relu
scope = 'test/test2' if with_bypass else 'test'
node = conv2d(inputs,
out_depth, [5, 5],
stride=stride,
padding='SAME',
weights_initializer=self._WeightInit(0.09),
activation_fn=activation_fn,
normalizer_fn=batch_norm,
normalizer_params=self._BatchNormParams(
scale=has_scaling, fused=fused_batch_norm),
scope=scope)
if with_bypass:
node = math_ops.add(inputs, node, name='test/AddV2')
relu_node = relu(node, name='test/' + relu_op_name)
folded_g = self._CopyGraph(unfolded_g)
with folded_g.as_default():
fold_batch_norms.FoldBatchNorms(
folded_g,
is_training=True,
freeze_batch_norm_delay=freeze_batch_norm_delay)
with session.Session(graph=unfolded_g) as sess:
sess.run(variables.global_variables_initializer())
grad_node = gradients.gradients(relu_node, inputs)
results = sess.run([relu_node, grad_node])
unfolded_forward, unfolded_backward = results[0], results[1]
with session.Session(graph=folded_g) as sess:
sess.run(variables.global_variables_initializer())
relu_node = folded_g.get_tensor_by_name(relu_node.name)
inputs = folded_g.get_tensor_by_name(inputs.name)
grad_node = gradients.gradients(relu_node, inputs)
results = sess.run([relu_node, grad_node])
folded_forward, folded_backward = results[0], results[1]
# Check that the folded and unfolded results match.
self.assertAllClose(unfolded_forward, folded_forward, atol=1e-3)
self.assertAllClose(unfolded_backward, folded_backward, atol=1e-3)
def testMultipleLayerConv2d(self,
relu=nn_ops.relu,
relu_op_name='Relu',
has_scaling=True,
fused_batch_norm=False,
freeze_batch_norm_delay=None,
insert_identity_node=False):
"""Tests folding cases for a network with multiple layers.
Args:
relu: Callable that returns an Operation, a factory method for the Relu*.
relu_op_name: String, name of the Relu* operation.
has_scaling: Bool, when true the batch norm has scaling.
fused_batch_norm: Bool, when true the batch norm is fused.
freeze_batch_norm_delay: None or the number of steps after which training
switches to using frozen mean and variance
insert_identity_node: Bool, insert identity node between conv and batch
norm
"""
g = ops.Graph()
with g.as_default():
batch_size, height, width = 5, 128, 128
inputs = array_ops.zeros((batch_size, height, width, 3))
out_depth = 3
stride = 1
activation_fn = relu
scope = 'topnet/testnet'
with variable_scope.variable_scope(scope, [inputs]):
layer1 = conv2d(inputs,
out_depth, [5, 5],
stride=stride,
padding='SAME',
weights_initializer=self._WeightInit(0.09),
activation_fn=None,
normalizer_fn=None,
scope='testnet/layer1')
# Add bn and relu with different scope
layer1 = batch_norm(layer1,
scale=has_scaling,
fused=fused_batch_norm,
scope='layer1')
layer1 = activation_fn(layer1)
layer2 = conv2d(layer1,
2 * out_depth, [5, 5],
stride=stride,
padding='SAME',
weights_initializer=self._WeightInit(0.09),
activation_fn=activation_fn,
normalizer_fn=batch_norm,
normalizer_params=self._BatchNormParams(
scale=has_scaling, fused=fused_batch_norm),
scope='testnet/layer2')
# Add bn and relu with different scope
layer2 = batch_norm(layer2,
scale=has_scaling,
fused=fused_batch_norm,
scope='layer2')
_ = activation_fn(layer2)
scope = 'topnet/testnet/testnet/layer2'
fold_batch_norms.FoldBatchNorms(
g,
is_training=True,
freeze_batch_norm_delay=freeze_batch_norm_delay)
folded_mul = g.get_operation_by_name(scope + '/mul_fold')
self.assertEqual(folded_mul.type, 'Mul')
self._AssertInputOpsAre(folded_mul, [
scope + '/correction_mult',
self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm)
])
self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold'])
folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold')
self.assertEqual(folded_conv.type, 'Conv2D')
# Remove :0 at end of name for tensor prior to comparison
self._AssertInputOpsAre(folded_conv,
[scope + '/mul_fold', layer1.name[:-2]])
self._AssertOutputGoesToOps(folded_conv, g, [scope + '/post_conv_mul'])
folded_add = g.get_operation_by_name(scope + '/add_fold')
self.assertEqual(folded_add.type, 'Add')
self._AssertInputOpsAre(folded_add, [
scope + '/correction_add',
self._BathNormBiasName(scope, fused_batch_norm)
])
output_op_names = [scope + '/' + relu_op_name]
self._AssertOutputGoesToOps(folded_add, g, output_op_names)
if freeze_batch_norm_delay is not None:
self._AssertMovingAveragesAreFrozen(g, scope)
for op in g.get_operations():
self.assertFalse('//' in op.name,
'Double slash in op %s' % op.name)