def _graph_def_from_concrete_fn(self, cfs):
if len(cfs) != 1:
raise NotImplementedError(
"Only a single concrete function is supported.")
if _get_version(_tf.__version__) >= _StrictVersion("2.2.0"):
frozen_fn = _convert_variables_to_constants_v2(
cfs[0], lower_control_flow=False, aggressive_inlining=True)
else:
frozen_fn = _convert_variables_to_constants_v2(
cfs[0], lower_control_flow=False)
graph_def = frozen_fn.graph.as_graph_def(add_shapes=True)
# run a Grappler's constant folding pass.
fn_inputs = [
t for t in frozen_fn.inputs if t.dtype != _dtypes.resource
]
grappler_optimizers_list = self._get_grappler_optimizers_list()
graph_def = _run_graph_optimizations(
graph_def,
fn_inputs,
frozen_fn.outputs,
config=_get_grappler_config(grappler_optimizers_list),
graph=frozen_fn.graph,
)
return graph_def
def _grappler_config(self, target_ops):
is_only_flex_enabled = set([OpsSet.SELECT_TF_OPS]) == target_ops
if is_only_flex_enabled:
# The layout optimizer turns NHCW to NCHW. This provides performance
# optimizations when Flex mode is enabled. However, this is not compatible
# with builtin ops.
return _get_grappler_config(["layout"])
return None
def _grappler_config(self):
is_only_flex_enabled = set([OpsSet.SELECT_TF_OPS]) == set(self._target_ops)
optimizers = ["constfold"]
if is_only_flex_enabled:
# The layout optimizer turns NHCW to NCHW. This provides performance
# optimizations when Flex mode is enabled. However, this is not compatible
# with builtin ops.
optimizers.append("layout")
return _get_grappler_config(optimizers)
def _grappler_config(self, optimizers=None):
"""Creates a tf.compat.v1.ConfigProto for configuring Grappler.
Args:
optimizers: List of strings that represents the list of optimizers.
Returns:
tf.ConfigProto.
"""
if not optimizers:
optimizers = []
optimizers.append("constfold")
is_only_flex_enabled = (
set([OpsSet.SELECT_TF_OPS]) == set(self.target_spec.supported_ops))
if is_only_flex_enabled:
# The layout optimizer turns NHCW to NCHW. This provides performance
# optimizations when Flex mode is enabled. However, this is not compatible
# with builtin ops.
optimizers.append("layout")
return _get_grappler_config(optimizers)
def _graph_def_from_concrete_fn(cfs):
if len(cfs) != 1:
raise NotImplementedError(
"Only a single concrete function is supported.")
frozen_fn = _convert_variables_to_constants_v2(
cfs[0], lower_control_flow=False)
graph_def = frozen_fn.graph.as_graph_def(add_shapes=True)
# run a Grappler's constant folding pass.
fn_inputs = [
t for t in frozen_fn.inputs if t.dtype != _dtypes.resource
]
graph_def = _run_graph_optimizations(
graph_def,
fn_inputs,
frozen_fn.outputs,
config=_get_grappler_config(["constfold", "dependency"]),
graph=frozen_fn.graph,
)
return graph_def
def convert(self):
"""Converts a TensorFlow GraphDef based on instance variables.
Returns:
The converted data in serialized format. Either a TFLite Flatbuffer or a
Graphviz graph depending on value in `output_format`.
Raises:
ValueError:
Input shape is not specified.
None value for dimension in input_tensor.
"""
# Checks dimensions in input tensor.
if self._has_valid_tensors():
for tensor in self._input_tensors:
shape = tensor.shape
if not shape:
raise ValueError("Provide an input shape for input array "
"'{0}'.".format(_get_tensor_name(tensor)))
# Note that shape_list might be empty for scalar shapes.
shape_list = shape.as_list()
if None in shape_list[1:]:
raise ValueError(
"None is only supported in the 1st dimension. Tensor '{0}' has "
"invalid shape '{1}'.".format(_get_tensor_name(tensor),
shape_list))
elif shape_list and shape_list[0] is None:
self._set_batch_size(batch_size=1)
# Get quantization stats. Ensures there is one stat per name if the stats
# are specified.
if self.quantized_input_stats:
quantized_stats = []
invalid_stats = []
for name in self.get_input_arrays():
if name in self.quantized_input_stats:
quantized_stats.append(self.quantized_input_stats[name])
else:
invalid_stats.append(name)
if invalid_stats:
raise ValueError(
"Quantization input stats are not available for input "
"tensors '{0}'.".format(",".join(invalid_stats)))
else:
quantized_stats = None
if self.representative_dataset:
if not isinstance(self.representative_dataset,
RepresentativeDataset):
raise TypeError(
"representative_dataset must be an instance of "
"RepresentativeDataset")
if self.representative_dataset.input_gen is None:
raise ValueError(
"Provide an input generator for representative_dataset")
# TODO(shashishekhar): For now use optimizations order is ignored.
# Both size and latency optimizations decide whether to apply post
# training optimizations.
post_training_optimize = bool(
len(
set(self.optimizations) & set([
Optimize.OPTIMIZE_FOR_LATENCY, Optimize.OPTIMIZE_FOR_SIZE
])))
# Do weights only quantization if there is no dataset for calibration.
weights_only_quantize_flag = (post_training_optimize and
(self.representative_dataset is None))
converter_kwargs = {
"inference_type": self.inference_type,
"inference_input_type": self.inference_input_type,
"input_format": constants.TENSORFLOW_GRAPHDEF,
"output_format": self.output_format,
"quantized_input_stats": quantized_stats,
"default_ranges_stats": self.default_ranges_stats,
"drop_control_dependency": self.drop_control_dependency,
"reorder_across_fake_quant": self.reorder_across_fake_quant,
"change_concat_input_ranges": self.change_concat_input_ranges,
"allow_custom_ops": self.allow_custom_ops,
"post_training_quantize": weights_only_quantize_flag,
"target_ops": self.target_ops,
"dump_graphviz_dir": self.dump_graphviz_dir,
"dump_graphviz_video": self.dump_graphviz_video
}
optimized_graph = None
if self.inference_type == constants.QUANTIZED_UINT8:
optimized_graph = self._graph_def
else:
try:
is_only_flex_enabled = set([OpsSet.SELECT_TF_OPS
]) == self.target_ops
config = _get_grappler_config(
enable_layout_optimizer=is_only_flex_enabled)
optimized_graph = _run_graph_optimizations(
self._graph_def, self._input_tensors, self._output_tensors,
config)
except Exception:
optimized_graph = self._graph_def
# Converts model.
if self._has_valid_tensors():
result = _toco_convert_impl(input_data=optimized_graph,
input_tensors=self._input_tensors,
output_tensors=self._output_tensors,
**converter_kwargs)
else:
result = _toco_convert_graph_def(
input_data=optimized_graph,
input_arrays_with_shape=self._input_arrays_with_shape,
output_arrays=self._output_arrays,
**converter_kwargs)
if self.representative_dataset and post_training_optimize:
calibrate_quantize = _calibrator.Calibrator(result)
result = calibrate_quantize.calibrate_and_quantize(
self.representative_dataset.input_gen)
return result
def convert(self):
"""Converts a TensorFlow GraphDef based on instance variables.
Returns:
The converted data in serialized format.
Raises:
ValueError:
Input shape is not specified.
None value for dimension in input_tensor.
"""
frozen_func = _convert_to_constants.convert_variables_to_constants_v2(
self._func)
input_tensors = [
tensor for tensor in frozen_func.inputs
if tensor.dtype != _dtypes.resource
]
output_tensors = frozen_func.outputs
# Run a Grappler pass.
is_only_flex_enabled = set([OpsSet.SELECT_TF_OPS
]) == self.target_spec.supported_ops
config = _get_grappler_config(
enable_layout_optimizer=is_only_flex_enabled)
graph_def = _run_graph_optimizations(frozen_func.graph.as_graph_def(),
input_tensors,
output_tensors,
config,
graph=frozen_func.graph)
# Checks dimensions in input tensor.
for tensor in input_tensors:
# Note that shape_list might be empty for scalar shapes.
shape_list = tensor.shape.as_list()
if None in shape_list[1:]:
raise ValueError(
"None is only supported in the 1st dimension. Tensor '{0}' has "
"invalid shape '{1}'.".format(_get_tensor_name(tensor),
shape_list))
elif shape_list and shape_list[0] is None:
# Set the batch size to 1 if undefined.
shape = tensor.shape.as_list()
shape[0] = 1
tensor.set_shape(shape)
if self.representative_dataset:
if not isinstance(self.representative_dataset,
RepresentativeDataset):
raise TypeError(
"`representative_dataset` must be an instance of "
"`RepresentativeDataset`")
if self.representative_dataset.input_gen is None:
raise ValueError(
"Provide an input generator for `representative_dataset`")
# TODO(shashishekhar): For now use optimizations order is ignored.
# Both size and latency optimizations decide whether to apply post
# training optimizations.
post_training_optimize = bool(
len(
set(self.optimizations)
& set([
Optimize.OPTIMIZE_FOR_LATENCY, Optimize.OPTIMIZE_FOR_SIZE
])))
# Do weights only quantization if there is no dataset for calibration.
weights_only_quantize_flag = (post_training_optimize and
(self.representative_dataset is None))
converter_kwargs = {
"input_format": constants.TENSORFLOW_GRAPHDEF,
"allow_custom_ops": self.allow_custom_ops,
"post_training_quantize": weights_only_quantize_flag,
"target_ops": self.target_spec.supported_ops,
}
# Converts model.
result = _toco_convert_impl(input_data=graph_def,
input_tensors=input_tensors,
output_tensors=output_tensors,
**converter_kwargs)
if self.representative_dataset and post_training_optimize:
calibrate_quantize = _calibrator.Calibrator(result)
result = calibrate_quantize.calibrate_and_quantize(
self.representative_dataset.input_gen)
return result
def convert(self):
"""Converts a TensorFlow GraphDef based on instance variables.
Returns:
The converted data in serialized format. Either a TFLite Flatbuffer or a
Graphviz graph depending on value in `output_format`.
Raises:
ValueError:
Input shape is not specified.
None value for dimension in input_tensor.
"""
# Checks dimensions in input tensor.
if self._has_valid_tensors():
for tensor in self._input_tensors:
shape = tensor.shape
if not shape:
raise ValueError("Provide an input shape for input array "
"'{0}'.".format(_get_tensor_name(tensor)))
# Note that shape_list might be empty for scalar shapes.
shape_list = shape.as_list()
if None in shape_list[1:]:
raise ValueError(
"None is only supported in the 1st dimension. Tensor '{0}' has "
"invalid shape '{1}'.".format(
_get_tensor_name(tensor), shape_list))
elif shape_list and shape_list[0] is None:
self._set_batch_size(batch_size=1)
# Get quantization stats. Ensures there is one stat per name if the stats
# are specified.
if self.quantized_input_stats:
quantized_stats = []
invalid_stats = []
for name in self.get_input_arrays():
if name in self.quantized_input_stats:
quantized_stats.append(self.quantized_input_stats[name])
else:
invalid_stats.append(name)
if invalid_stats:
raise ValueError("Quantization input stats are not available for input "
"tensors '{0}'.".format(",".join(invalid_stats)))
else:
quantized_stats = None
if self.representative_dataset:
if not isinstance(self.representative_dataset, RepresentativeDataset):
raise TypeError(
"representative_dataset must be an instance of "
"RepresentativeDataset")
if self.representative_dataset.input_gen is None:
raise ValueError(
"Provide an input generator for representative_dataset")
# TODO(shashishekhar): For now use optimizations order is ignored.
# Both size and latency optimizations decide whether to apply post
# training optimizations.
post_training_optimize = bool(
len(set(self.optimizations) & set([Optimize.OPTIMIZE_FOR_LATENCY,
Optimize.OPTIMIZE_FOR_SIZE])))
# Do weights only quantization if there is no dataset for calibration.
weights_only_quantize_flag = (
post_training_optimize and (self.representative_dataset is None))
converter_kwargs = {
"inference_type": self.inference_type,
"inference_input_type": self.inference_input_type,
"input_format": constants.TENSORFLOW_GRAPHDEF,
"output_format": self.output_format,
"quantized_input_stats": quantized_stats,
"default_ranges_stats": self.default_ranges_stats,
"drop_control_dependency": self.drop_control_dependency,
"reorder_across_fake_quant": self.reorder_across_fake_quant,
"change_concat_input_ranges": self.change_concat_input_ranges,
"allow_custom_ops": self.allow_custom_ops,
"post_training_quantize": weights_only_quantize_flag,
"target_ops": self.target_ops,
"dump_graphviz_dir": self.dump_graphviz_dir,
"dump_graphviz_video": self.dump_graphviz_video
}
optimized_graph = None
if self.inference_type == constants.QUANTIZED_UINT8:
optimized_graph = self._graph_def
else:
try:
is_only_flex_enabled = set([OpsSet.SELECT_TF_OPS]) == self.target_ops
config = _get_grappler_config(
enable_layout_optimizer=is_only_flex_enabled)
optimized_graph = _run_graph_optimizations(
self._graph_def, self._input_tensors, self._output_tensors, config)
except Exception:
optimized_graph = self._graph_def
# Converts model.
if self._has_valid_tensors():
result = _toco_convert_impl(
input_data=optimized_graph,
input_tensors=self._input_tensors,
output_tensors=self._output_tensors,
**converter_kwargs)
else:
result = _toco_convert_graph_def(
input_data=optimized_graph,
input_arrays_with_shape=self._input_arrays_with_shape,
output_arrays=self._output_arrays,
**converter_kwargs)
if self.representative_dataset and post_training_optimize:
calibrate_quantize = _calibrator.Calibrator(result)
result = calibrate_quantize.calibrate_and_quantize(
self.representative_dataset.input_gen)
return result
def convert(self):
"""Converts a TensorFlow GraphDef based on instance variables.
Returns:
The converted data in serialized format.
Raises:
ValueError:
Input shape is not specified.
None value for dimension in input_tensor.
"""
frozen_func = _convert_to_constants.convert_variables_to_constants_v2(
self._func)
input_tensors = [
tensor for tensor in frozen_func.inputs
if tensor.dtype != _dtypes.resource
]
output_tensors = frozen_func.outputs
# Run a Grappler pass.
is_only_flex_enabled = set(
[OpsSet.SELECT_TF_OPS]) == self.target_spec.supported_ops
config = _get_grappler_config(enable_layout_optimizer=is_only_flex_enabled)
graph_def = _run_graph_optimizations(
frozen_func.graph.as_graph_def(),
input_tensors,
output_tensors,
config,
graph=frozen_func.graph)
# Checks dimensions in input tensor.
for tensor in input_tensors:
# Note that shape_list might be empty for scalar shapes.
shape_list = tensor.shape.as_list()
if None in shape_list[1:]:
raise ValueError(
"None is only supported in the 1st dimension. Tensor '{0}' has "
"invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list))
elif shape_list and shape_list[0] is None:
# Set the batch size to 1 if undefined.
shape = tensor.shape.as_list()
shape[0] = 1
tensor.set_shape(shape)
if self.representative_dataset:
if not isinstance(self.representative_dataset, RepresentativeDataset):
raise TypeError("`representative_dataset` must be an instance of "
"`RepresentativeDataset`")
if self.representative_dataset.input_gen is None:
raise ValueError(
"Provide an input generator for `representative_dataset`")
# TODO(shashishekhar): For now use optimizations order is ignored.
# Both size and latency optimizations decide whether to apply post
# training optimizations.
post_training_optimize = bool(
len(
set(self.optimizations)
& set([Optimize.OPTIMIZE_FOR_LATENCY, Optimize.OPTIMIZE_FOR_SIZE])))
# Do weights only quantization if there is no dataset for calibration.
weights_only_quantize_flag = (
post_training_optimize and (self.representative_dataset is None))
converter_kwargs = {
"input_format": constants.TENSORFLOW_GRAPHDEF,
"allow_custom_ops": self.allow_custom_ops,
"post_training_quantize": weights_only_quantize_flag,
"target_ops": self.target_spec.supported_ops,
}
# Converts model.
result = _toco_convert_impl(
input_data=graph_def,
input_tensors=input_tensors,
output_tensors=output_tensors,
**converter_kwargs)
if self.representative_dataset and post_training_optimize:
calibrate_quantize = _calibrator.Calibrator(result)
result = calibrate_quantize.calibrate_and_quantize(
self.representative_dataset.input_gen)
return result
