def test_layer_direct_compare_metrics(self, quantized_io):
def _corr(float_values, quant_values, scale, zero_point):
dequant_values = (quant_values.astype(np.int32) -
zero_point) * scale
return np.corrcoef(float_values.flatten(),
dequant_values.flatten())[0, 1]
if quantized_io:
debug_model = QuantizationDebuggerTest.debug_model_int8
else:
debug_model = QuantizationDebuggerTest.debug_model_float
options = debugger.QuantizationDebugOptions(
layer_direct_compare_metrics={'corr': _corr})
quant_debugger = debugger.QuantizationDebugger(
quant_debug_model_content=debug_model,
debug_dataset=_calibration_gen,
debug_options=options)
quant_debugger.run()
expected_metrics = {
'corr': 0.99999,
}
self.assertLen(quant_debugger.layer_statistics, 1)
actual_metrics = next(iter(quant_debugger.layer_statistics.values()))
for key, value in expected_metrics.items():
self.assertAlmostEqual(value, actual_metrics[key], places=4)
def test_quantization_debugger_layer_metrics(self, quantized_io):
if quantized_io:
debug_model = QuantizationDebuggerTest.debug_model_int8
else:
debug_model = QuantizationDebuggerTest.debug_model_float
options = debugger.QuantizationDebugOptions(
layer_debug_metrics={
'l1_norm': lambda diffs: np.mean(np.abs(diffs))
})
quant_debugger = debugger.QuantizationDebugger(
quant_debug_model_content=debug_model,
debug_dataset=_calibration_gen,
debug_options=options)
quant_debugger.run()
expected_metrics = {
'num_elements': 9,
'stddev': 0.03850026,
'mean_error': 0.01673192,
'max_abs_error': 0.10039272,
'mean_squared_error': 0.0027558778,
'l1_norm': 0.023704167,
}
self.assertLen(quant_debugger.layer_statistics, 1)
actual_metrics = next(iter(quant_debugger.layer_statistics.values()))
self.assertCountEqual(expected_metrics.keys(), actual_metrics.keys())
for key, value in expected_metrics.items():
self.assertAlmostEqual(value, actual_metrics[key], places=5)
buffer = io.StringIO()
quant_debugger.layer_statistics_dump(buffer)
reader = csv.DictReader(buffer.getvalue().split())
actual_values = next(iter(reader))
expected_values = expected_metrics.copy()
expected_values.update({
'op_name': 'CONV_2D',
'tensor_idx': 7 if quantized_io else 8,
'scales': [0.15686275],
'zero_points': [-128],
'tensor_name': r'Identity[1-9]?$'
})
for key, value in expected_values.items():
if isinstance(value, str):
self.assertIsNotNone(
re.match(value, actual_values[key]),
'String is different from expected string. Please fix test code if'
" it's being affected by graph manipulation changes.")
elif isinstance(value, list):
self.assertAlmostEqual(value[0],
float(actual_values[key][1:-1]),
places=5)
else:
self.assertAlmostEqual(value,
float(actual_values[key]),
places=5)
def test_denylisted_nodes_from_option_constructor(self, quantized_io):
options = debugger.QuantizationDebugOptions(
layer_debug_metrics={
'l1_norm': lambda diffs: np.mean(np.abs(diffs))
},
fully_quantize=quantized_io,
denylisted_nodes=['Identity'])
# TODO(b/195084873): Count the number of NumericVerify op.
with self.assertRaisesRegex(
ValueError,
'Please check if the quantized model is in debug mode'):
_ = debugger.QuantizationDebugger(converter=_quantize_converter(
self.tf_model_root, self.tf_model, _calibration_gen),
debug_dataset=_calibration_gen,
debug_options=options)
def test_init_duplicate_keys_raises_ValueError(self):
with self.assertRaises(ValueError):
debugger.QuantizationDebugOptions(layer_debug_metrics={
'a': _dummy_fn,
'b': _dummy_fn
},
model_debug_metrics={
'c': _dummy_fn,
'd': _dummy_fn
},
layer_direct_compare_metrics={
'a': _dummy_fn,
'e': _dummy_fn
})
with self.assertRaises(ValueError):
debugger.QuantizationDebugOptions(layer_debug_metrics={
'a': _dummy_fn,
'b': _dummy_fn
},
layer_direct_compare_metrics={
'a': _dummy_fn,
'e': _dummy_fn
})
def test_denylisted_ops(self, quantized_io):
options = debugger.QuantizationDebugOptions(
layer_debug_metrics={
'l1_norm': lambda diffs: np.mean(np.abs(diffs))
})
options.fully_quantize = quantized_io
quant_debugger = debugger.QuantizationDebugger(
converter=_quantize_converter(self.tf_model, _calibration_gen),
debug_dataset=_calibration_gen,
debug_options=options)
options.denylisted_ops = ['CONV_2D']
with self.assertRaisesRegex(
ValueError,
'Please check if the quantized model is in debug mode'):
quant_debugger.options = options
def test_denylisted_ops_from_option_setter(self, quantized_io):
options = debugger.QuantizationDebugOptions(
layer_debug_metrics={
'l1_norm': lambda diffs: np.mean(np.abs(diffs))
},
fully_quantize=quantized_io)
quant_debugger = debugger.QuantizationDebugger(
converter=_quantize_converter(self.tf_model_root, self.tf_model,
_calibration_gen),
debug_dataset=_calibration_gen,
debug_options=options)
options.denylisted_ops = ['CONV_2D']
# TODO(b/195084873): The exception is expected to check whether selective
# quantization was done properly, since after the selective quantization
# the model will have no quantized layers thus have no NumericVerify ops,
# resulted in this exception. Marked with a bug to fix this in more
# straightforward way.
with self.assertRaisesRegex(
ValueError,
'Please check if the quantized model is in debug mode'):
quant_debugger.options = options
def test_model_metrics(self, quantized_io):
if quantized_io:
debug_model = QuantizationDebuggerTest.debug_model_int8
else:
debug_model = QuantizationDebuggerTest.debug_model_float
options = debugger.QuantizationDebugOptions(
model_debug_metrics={
'stdev': lambda x, y: np.std(x[0] - y[0])
})
quant_debugger = debugger.QuantizationDebugger(
quant_debug_model_content=debug_model,
float_model_content=QuantizationDebuggerTest.float_model,
debug_dataset=_calibration_gen,
debug_options=options)
quant_debugger.run()
expected_metrics = {'stdev': 0.050998904}
actual_metrics = quant_debugger.model_statistics
self.assertCountEqual(expected_metrics.keys(), actual_metrics.keys())
for key, value in expected_metrics.items():
self.assertAlmostEqual(value, actual_metrics[key], places=5)
def _quant_single_layer(quant_node_name: str, node_list: List[str],
converter: tf.lite.TFLiteConverter) -> bytes:
"""Build a single layer quantized model.
Args:
quant_node_name: Name of a op to quantize.
node_list: List of every op names.
converter: TFLiteConverter to quantize a model.
Returns:
TFLite model with a single layer quantized.
"""
for i, node_name in enumerate(node_list):
if node_name == quant_node_name:
node_list = node_list[:i] + node_list[i + 1:]
break
quant_debug_option = debugger.QuantizationDebugOptions()
quant_debug_option.denylisted_nodes = node_list
quant_debugger = debugger.QuantizationDebugger(
converter=converter, debug_options=quant_debug_option)
return quant_debugger.get_nondebug_quantized_model()
