def test_layer_direct_compare_metrics(self, quantized_io): def _corr(float_values, quant_values, scale, zero_point): dequant_values = (quant_values.astype(np.int32) - zero_point) * scale return np.corrcoef(float_values.flatten(), dequant_values.flatten())[0, 1] if quantized_io: debug_model = QuantizationDebuggerTest.debug_model_int8 else: debug_model = QuantizationDebuggerTest.debug_model_float options = debugger.QuantizationDebugOptions( layer_direct_compare_metrics={'corr': _corr}) quant_debugger = debugger.QuantizationDebugger( quant_debug_model_content=debug_model, debug_dataset=_calibration_gen, debug_options=options) quant_debugger.run() expected_metrics = { 'corr': 0.99999, } self.assertLen(quant_debugger.layer_statistics, 1) actual_metrics = next(iter(quant_debugger.layer_statistics.values())) for key, value in expected_metrics.items(): self.assertAlmostEqual(value, actual_metrics[key], places=4)
def test_quantization_debugger_layer_metrics(self, quantized_io): if quantized_io: debug_model = QuantizationDebuggerTest.debug_model_int8 else: debug_model = QuantizationDebuggerTest.debug_model_float options = debugger.QuantizationDebugOptions( layer_debug_metrics={ 'l1_norm': lambda diffs: np.mean(np.abs(diffs)) }) quant_debugger = debugger.QuantizationDebugger( quant_debug_model_content=debug_model, debug_dataset=_calibration_gen, debug_options=options) quant_debugger.run() expected_metrics = { 'num_elements': 9, 'stddev': 0.03850026, 'mean_error': 0.01673192, 'max_abs_error': 0.10039272, 'mean_squared_error': 0.0027558778, 'l1_norm': 0.023704167, } self.assertLen(quant_debugger.layer_statistics, 1) actual_metrics = next(iter(quant_debugger.layer_statistics.values())) self.assertCountEqual(expected_metrics.keys(), actual_metrics.keys()) for key, value in expected_metrics.items(): self.assertAlmostEqual(value, actual_metrics[key], places=5) buffer = io.StringIO() quant_debugger.layer_statistics_dump(buffer) reader = csv.DictReader(buffer.getvalue().split()) actual_values = next(iter(reader)) expected_values = expected_metrics.copy() expected_values.update({ 'op_name': 'CONV_2D', 'tensor_idx': 7 if quantized_io else 8, 'scales': [0.15686275], 'zero_points': [-128], 'tensor_name': r'Identity[1-9]?$' }) for key, value in expected_values.items(): if isinstance(value, str): self.assertIsNotNone( re.match(value, actual_values[key]), 'String is different from expected string. Please fix test code if' " it's being affected by graph manipulation changes.") elif isinstance(value, list): self.assertAlmostEqual(value[0], float(actual_values[key][1:-1]), places=5) else: self.assertAlmostEqual(value, float(actual_values[key]), places=5)
def test_denylisted_nodes_from_option_constructor(self, quantized_io): options = debugger.QuantizationDebugOptions( layer_debug_metrics={ 'l1_norm': lambda diffs: np.mean(np.abs(diffs)) }, fully_quantize=quantized_io, denylisted_nodes=['Identity']) # TODO(b/195084873): Count the number of NumericVerify op. with self.assertRaisesRegex( ValueError, 'Please check if the quantized model is in debug mode'): _ = debugger.QuantizationDebugger(converter=_quantize_converter( self.tf_model_root, self.tf_model, _calibration_gen), debug_dataset=_calibration_gen, debug_options=options)
def test_init_duplicate_keys_raises_ValueError(self): with self.assertRaises(ValueError): debugger.QuantizationDebugOptions(layer_debug_metrics={ 'a': _dummy_fn, 'b': _dummy_fn }, model_debug_metrics={ 'c': _dummy_fn, 'd': _dummy_fn }, layer_direct_compare_metrics={ 'a': _dummy_fn, 'e': _dummy_fn }) with self.assertRaises(ValueError): debugger.QuantizationDebugOptions(layer_debug_metrics={ 'a': _dummy_fn, 'b': _dummy_fn }, layer_direct_compare_metrics={ 'a': _dummy_fn, 'e': _dummy_fn })
def test_denylisted_ops(self, quantized_io): options = debugger.QuantizationDebugOptions( layer_debug_metrics={ 'l1_norm': lambda diffs: np.mean(np.abs(diffs)) }) options.fully_quantize = quantized_io quant_debugger = debugger.QuantizationDebugger( converter=_quantize_converter(self.tf_model, _calibration_gen), debug_dataset=_calibration_gen, debug_options=options) options.denylisted_ops = ['CONV_2D'] with self.assertRaisesRegex( ValueError, 'Please check if the quantized model is in debug mode'): quant_debugger.options = options
def test_denylisted_ops_from_option_setter(self, quantized_io): options = debugger.QuantizationDebugOptions( layer_debug_metrics={ 'l1_norm': lambda diffs: np.mean(np.abs(diffs)) }, fully_quantize=quantized_io) quant_debugger = debugger.QuantizationDebugger( converter=_quantize_converter(self.tf_model_root, self.tf_model, _calibration_gen), debug_dataset=_calibration_gen, debug_options=options) options.denylisted_ops = ['CONV_2D'] # TODO(b/195084873): The exception is expected to check whether selective # quantization was done properly, since after the selective quantization # the model will have no quantized layers thus have no NumericVerify ops, # resulted in this exception. Marked with a bug to fix this in more # straightforward way. with self.assertRaisesRegex( ValueError, 'Please check if the quantized model is in debug mode'): quant_debugger.options = options
def test_model_metrics(self, quantized_io): if quantized_io: debug_model = QuantizationDebuggerTest.debug_model_int8 else: debug_model = QuantizationDebuggerTest.debug_model_float options = debugger.QuantizationDebugOptions( model_debug_metrics={ 'stdev': lambda x, y: np.std(x[0] - y[0]) }) quant_debugger = debugger.QuantizationDebugger( quant_debug_model_content=debug_model, float_model_content=QuantizationDebuggerTest.float_model, debug_dataset=_calibration_gen, debug_options=options) quant_debugger.run() expected_metrics = {'stdev': 0.050998904} actual_metrics = quant_debugger.model_statistics self.assertCountEqual(expected_metrics.keys(), actual_metrics.keys()) for key, value in expected_metrics.items(): self.assertAlmostEqual(value, actual_metrics[key], places=5)
def _quant_single_layer(quant_node_name: str, node_list: List[str], converter: tf.lite.TFLiteConverter) -> bytes: """Build a single layer quantized model. Args: quant_node_name: Name of a op to quantize. node_list: List of every op names. converter: TFLiteConverter to quantize a model. Returns: TFLite model with a single layer quantized. """ for i, node_name in enumerate(node_list): if node_name == quant_node_name: node_list = node_list[:i] + node_list[i + 1:] break quant_debug_option = debugger.QuantizationDebugOptions() quant_debug_option.denylisted_nodes = node_list quant_debugger = debugger.QuantizationDebugger( converter=converter, debug_options=quant_debug_option) return quant_debugger.get_nondebug_quantized_model()