def _AssertIdempotent(self, graph): # Ensure that calling the rewrite again doesn't change the graph. graph_def_before = str(graph.as_graph_def()) with graph.as_default(): # Ensuring that calling the rewrite again doesn't add more nodes. fold_batch_norms.FoldBatchNorms(graph, is_training=True) quantize.Quantize(graph, True) graph_def_after = str(graph.as_graph_def()) self.assertEqual(graph_def_before, graph_def_after)
def _create_graph(input_graph=None, is_training=True, weight_bits=8, activation_bits=8, symmetric=False, quant_delay=None, freeze_bn_delay=None, scope=None): """Rewrites an input_graph in place for simulated quantization. The graph has fake quantization ops inserted to simulate the error introduced by quantization. Since the graph is transformed in place, the expected behavior of previously held references to nodes and tensors may change. Args: input_graph: The tf.Graph to be transformed, if None then defaults to the default graph. is_training: Whether quantizing training or eval graph. weight_bits: Number of bits to use for quantizing weights. activation_bits: Number of bits to use for quantizing activations. symmetric: If true, use symmetric quantization limits instead of training the minimum and maximum of each quantization range separately. quant_delay: Number of steps after which weights and activations are quantized during training. freeze_bn_delay: Number of steps after which moving mean and variance are frozen and used instead of batch statistics during training. freeze_bn_delay should be greater than quant_delay and should correspond to the number of steps when training has almost converged scope: The scope to be transformed. If it's not None, only the ops which are in this scope will be transformed. Raises: ValueError: If elements contains an element that isn't a tf.Tensor or tf.Operation. """ if input_graph is None: input_graph = ops.get_default_graph() # Add check to see if graph has training ops, if so provide error message and # exit _check_for_training_ops(input_graph) with input_graph.as_default(): fold_batch_norms.FoldBatchNorms( input_graph, freeze_batch_norm_delay=freeze_bn_delay, is_training=is_training) quantize.Quantize(input_graph, is_training, quant_delay=quant_delay, weight_bits=weight_bits, activation_bits=activation_bits, symmetric=symmetric, scope=scope)
def _TestQuantize_AtrousConvWithBatchNorm( self, activation, activation_op_name, with_bypass, delay, fused_batch_norm, use_resource, scope): """Tests quantization: inputs -> atrous conv with batch norm -> Activation. Args: activation: Callable that returns an Operation, a factory method for the Activation. activation_op_name: String, name of the Activation operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. fused_batch_norm: Bool, when true use FusedBatchNorm. use_resource: Bool, when true uses resource variables. scope: String, specifies top level scope for the graph """ graph = ops.Graph() with graph.as_default(): variable_scope.get_variable_scope().set_use_resource(use_resource) batch_size, height, width, depth = 5, 128, 128, 3 inputs = array_ops.zeros((batch_size, height, width, depth)) dilation_rate = 2 conv_scope = self._GetConvScope(scope, with_bypass) scope = '' if scope is None else scope delim = '/' if scope else '' node = separable_conv2d( inputs, None, [3, 3], rate=dilation_rate, depth_multiplier=1.0, padding='SAME', weights_initializer=self._WeightInit(0.09), activation_fn=None, normalizer_fn=batch_norm, normalizer_params=self._BatchNormParams(fused_batch_norm), scope=conv_scope) # Manually add a bypass (optional) and an activation. if with_bypass: node = math_ops.add(inputs, node, name=scope + delim + 'AddV2') node = activation(node, name=scope + delim + activation_op_name) update_barrier = control_flow_ops.no_op(name='update_barrier') with ops.control_dependencies([update_barrier]): array_ops.identity(node, name='control_dependency') fold_batch_norms.FoldBatchNorms(graph, is_training=True) quantize.Quantize(graph, True, quant_delay=delay) self._AssertCorrectQuantizedGraphWithBatchNorm( graph, scope, 'DepthwiseConv2dNative', activation_op_name, with_bypass, delay, use_resource)
def _TestBatchNormForcedUpdates(self, activation, activation_op_name, fused_batch_norm, use_resource): """post_activation bypass quantization should happen with forced updates.""" graph = ops.Graph() with graph.as_default(): variable_scope.get_variable_scope().set_use_resource(use_resource) batch_size, height, width, depth = 5, 128, 128, 3 input1 = array_ops.zeros((batch_size, height, width, depth)) input2 = array_ops.zeros((batch_size, height / 2, width / 2, 32)) # Setting updates_collections to None forces updates adding an extra # identity operation following batch norms. bn_params = self._BatchNormParams( fused=fused_batch_norm, force_updates=True) conv = conv2d( input1, 32, [5, 5], stride=2, padding='SAME', weights_initializer=self._WeightInit(0.09), activation_fn=activation, normalizer_fn=batch_norm, normalizer_params=bn_params, scope='test/test') bypass_tensor = math_ops.add(conv, input2, name='test/add') # The output of the post_activation bypass will be another layer. _ = conv2d( bypass_tensor, 32, [5, 5], stride=2, padding='SAME', weights_initializer=self._WeightInit(0.09), normalizer_fn=batch_norm, normalizer_params=bn_params, activation_fn=activation, scope='test/unused') fold_batch_norms.FoldBatchNorms(graph, is_training=True) quantize.Quantize(graph, is_training=True) # Ensure that the bypass node is preceded by and followed by a # FakeQuantWithMinMaxVar operation, since the output of the Add isn't an # activation. self.assertTrue('FakeQuantWithMinMaxVars' in [c.type for c in bypass_tensor.consumers()]) self.assertTrue('FakeQuantWithMinMaxVars' in [i.op.type for i in bypass_tensor.op.inputs]) with open('/tmp/bn_quant_test.pbtxt', 'w') as f: f.write(str(graph.as_graph_def()))
def _TestFoldConv2d(self, relu, relu_op_name, with_bypass, has_scaling, fused_batch_norm, freeze_batch_norm_delay, insert_identity_node): """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. Args: relu: Callable that returns an Operation, a factory method for the Relu*. relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. has_scaling: Bool, when true the batch norm has scaling. fused_batch_norm: Bool, when true the batch norm is fused. freeze_batch_norm_delay: None or the number of steps after which training switches to using frozen mean and variance insert_identity_node: Bool, insert identity node between conv and batch norm """ g = ops.Graph() with g.as_default(): batch_size, height, width = 5, 128, 128 inputs = array_ops.zeros((batch_size, height, width, 3)) out_depth = 3 if with_bypass else 32 stride = 1 if with_bypass else 2 activation_fn = None if with_bypass else relu name = 'test/test2' if with_bypass else 'test' if insert_identity_node: with g.name_scope(name): node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', weights_initializer=self._WeightInit(0.09), activation_fn=None, normalizer_fn=None, biases_initializer=None) conv_out = array_ops.identity(node, name='conv_out') node = batch_norm(conv_out, center=True, scale=has_scaling, decay=1.0 - 0.003, fused=fused_batch_norm) if activation_fn is not None: node = activation_fn(node) conv_name = name + '/Conv' else: node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', weights_initializer=self._WeightInit(0.09), activation_fn=activation_fn, normalizer_fn=batch_norm, normalizer_params=self._BatchNormParams( scale=has_scaling, fused=fused_batch_norm), scope=name) conv_name = name if with_bypass: node = math_ops.add(inputs, node, name='test/AddV2') relu(node, name='test/' + relu_op_name) fold_batch_norms.FoldBatchNorms( g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) folded_mul = g.get_operation_by_name(conv_name + '/mul_fold') self.assertEqual(folded_mul.type, 'Mul') self._AssertInputOpsAre(folded_mul, [ conv_name + '/correction_mult', self._BatchNormMultiplierName(conv_name, has_scaling, fused_batch_norm) ]) self._AssertOutputGoesToOps(folded_mul, g, [conv_name + '/Conv2D_Fold']) folded_conv = g.get_operation_by_name(conv_name + '/Conv2D_Fold') self.assertEqual(folded_conv.type, 'Conv2D') self._AssertInputOpsAre(folded_conv, [conv_name + '/mul_fold', inputs.op.name]) self._AssertOutputGoesToOps(folded_conv, g, [conv_name + '/post_conv_mul']) folded_add = g.get_operation_by_name(conv_name + '/add_fold') self.assertEqual(folded_add.type, 'Add') self._AssertInputOpsAre(folded_add, [ conv_name + '/correction_add', self._BathNormBiasName(conv_name, fused_batch_norm) ]) output_op_names = [ 'test/AddV2' if with_bypass else 'test/' + relu_op_name ] self._AssertOutputGoesToOps(folded_add, g, output_op_names) if freeze_batch_norm_delay is not None: self._AssertMovingAveragesAreFrozen(g, name) for op in g.get_operations(): self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name)
def _TestCompareFoldAndUnfolded(self, relu, relu_op_name, with_bypass, has_scaling, fused_batch_norm, freeze_batch_norm_delay, insert_identity_node=False): """Tests that running folded and unfolded BN returns the same results. Args: relu: Callable that returns an Operation, a factory method for the Relu*. relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. has_scaling: Bool, when true the batch norm has scaling. fused_batch_norm: Bool, when true the batch norm is fused. freeze_batch_norm_delay: None or the number of steps after which training switches to using frozen mean and variance insert_identity_node: Bool, insert identity node between conv and batch norm """ random_seed.set_random_seed(1234) unfolded_g = ops.Graph() with unfolded_g.as_default(): batch_size, height, width = 5, 128, 128 inputs = random_ops.random_uniform((batch_size, height, width, 3), dtype=dtypes.float32, seed=1234) out_depth = 3 if with_bypass else 32 stride = 1 if with_bypass else 2 activation_fn = None if with_bypass else relu scope = 'test/test2' if with_bypass else 'test' node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', weights_initializer=self._WeightInit(0.09), activation_fn=activation_fn, normalizer_fn=batch_norm, normalizer_params=self._BatchNormParams( scale=has_scaling, fused=fused_batch_norm), scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/AddV2') relu_node = relu(node, name='test/' + relu_op_name) folded_g = self._CopyGraph(unfolded_g) with folded_g.as_default(): fold_batch_norms.FoldBatchNorms( folded_g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) with session.Session(graph=unfolded_g) as sess: sess.run(variables.global_variables_initializer()) grad_node = gradients.gradients(relu_node, inputs) results = sess.run([relu_node, grad_node]) unfolded_forward, unfolded_backward = results[0], results[1] with session.Session(graph=folded_g) as sess: sess.run(variables.global_variables_initializer()) relu_node = folded_g.get_tensor_by_name(relu_node.name) inputs = folded_g.get_tensor_by_name(inputs.name) grad_node = gradients.gradients(relu_node, inputs) results = sess.run([relu_node, grad_node]) folded_forward, folded_backward = results[0], results[1] # Check that the folded and unfolded results match. self.assertAllClose(unfolded_forward, folded_forward, atol=1e-3) self.assertAllClose(unfolded_backward, folded_backward, atol=1e-3)
def testMultipleLayerConv2d(self, relu=nn_ops.relu, relu_op_name='Relu', has_scaling=True, fused_batch_norm=False, freeze_batch_norm_delay=None, insert_identity_node=False): """Tests folding cases for a network with multiple layers. Args: relu: Callable that returns an Operation, a factory method for the Relu*. relu_op_name: String, name of the Relu* operation. has_scaling: Bool, when true the batch norm has scaling. fused_batch_norm: Bool, when true the batch norm is fused. freeze_batch_norm_delay: None or the number of steps after which training switches to using frozen mean and variance insert_identity_node: Bool, insert identity node between conv and batch norm """ g = ops.Graph() with g.as_default(): batch_size, height, width = 5, 128, 128 inputs = array_ops.zeros((batch_size, height, width, 3)) out_depth = 3 stride = 1 activation_fn = relu scope = 'topnet/testnet' with variable_scope.variable_scope(scope, [inputs]): layer1 = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', weights_initializer=self._WeightInit(0.09), activation_fn=None, normalizer_fn=None, scope='testnet/layer1') # Add bn and relu with different scope layer1 = batch_norm(layer1, scale=has_scaling, fused=fused_batch_norm, scope='layer1') layer1 = activation_fn(layer1) layer2 = conv2d(layer1, 2 * out_depth, [5, 5], stride=stride, padding='SAME', weights_initializer=self._WeightInit(0.09), activation_fn=activation_fn, normalizer_fn=batch_norm, normalizer_params=self._BatchNormParams( scale=has_scaling, fused=fused_batch_norm), scope='testnet/layer2') # Add bn and relu with different scope layer2 = batch_norm(layer2, scale=has_scaling, fused=fused_batch_norm, scope='layer2') _ = activation_fn(layer2) scope = 'topnet/testnet/testnet/layer2' fold_batch_norms.FoldBatchNorms( g, is_training=True, freeze_batch_norm_delay=freeze_batch_norm_delay) folded_mul = g.get_operation_by_name(scope + '/mul_fold') self.assertEqual(folded_mul.type, 'Mul') self._AssertInputOpsAre(folded_mul, [ scope + '/correction_mult', self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) ]) self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold']) folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold') self.assertEqual(folded_conv.type, 'Conv2D') # Remove :0 at end of name for tensor prior to comparison self._AssertInputOpsAre(folded_conv, [scope + '/mul_fold', layer1.name[:-2]]) self._AssertOutputGoesToOps(folded_conv, g, [scope + '/post_conv_mul']) folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') self._AssertInputOpsAre(folded_add, [ scope + '/correction_add', self._BathNormBiasName(scope, fused_batch_norm) ]) output_op_names = [scope + '/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) if freeze_batch_norm_delay is not None: self._AssertMovingAveragesAreFrozen(g, scope) for op in g.get_operations(): self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name)