def _graph_def_from_concrete_fn(self, cfs): if len(cfs) != 1: raise NotImplementedError( "Only a single concrete function is supported.") if _get_version(_tf.__version__) >= _StrictVersion("2.2.0"): frozen_fn = _convert_variables_to_constants_v2( cfs[0], lower_control_flow=False, aggressive_inlining=True) else: frozen_fn = _convert_variables_to_constants_v2( cfs[0], lower_control_flow=False) graph_def = frozen_fn.graph.as_graph_def(add_shapes=True) # run a Grappler's constant folding pass. fn_inputs = [ t for t in frozen_fn.inputs if t.dtype != _dtypes.resource ] grappler_optimizers_list = self._get_grappler_optimizers_list() graph_def = _run_graph_optimizations( graph_def, fn_inputs, frozen_fn.outputs, config=_get_grappler_config(grappler_optimizers_list), graph=frozen_fn.graph, ) return graph_def
def get_func_graph(model, input_signature=None, *args, **kwargs): func = keras_utils.trace_model_call(model, input_signature) concrete_func = func.get_concrete_function(*args, **kwargs) frozen_func = tf_utils.convert_to_constants(concrete_func, lower_control_flow=False) graph_def = frozen_func.graph.as_graph_def() utils.maybe_export_graph(_BEFORE_OPT_GRAPH, graph_def) input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != tf.dtypes.resource ] output_tensors = frozen_func.outputs config = config_pb2.ConfigProto() rewrite_options = config.graph_options.rewrite_options #rewrite_options.constant_folding = rewrite_options.ON rewrite_options.optimizers.append('constfold') graph_def = _run_graph_optimizations(graph_def, input_tensors, output_tensors, config=config, graph=frozen_func.graph) utils.maybe_export_graph(_FINAL_TRACED_GRAPH, graph_def) with tf.Graph().as_default() as tf_graph: tf.import_graph_def(graph_def, name='') func_graph = concrete_func.graph return (tf_graph, func_graph.structured_input_signature, func_graph.structured_outputs)
def extract_outputs_from_subclassing_model(model, output_dict, output_names): from tensorflow.core.protobuf import config_pb2 from tensorflow.python.keras.saving import saving_utils as _saving_utils from tensorflow.lite.python.util import run_graph_optimizations as _run_graph_optimizations from ._graph_cvt import convert_variables_to_constants_v2 as _convert_to_constants function = _saving_utils.trace_model_call(model) concrete_func = function.get_concrete_function() output_names.extend([ts_.name for ts_ in concrete_func.outputs]) output_dict.update( build_layer_outputs(model, concrete_func.graph, concrete_func.outputs)) frozen_func = _convert_to_constants(concrete_func, lower_control_flow=True) graph_def = frozen_func.graph.as_graph_def() if TF_GRAPH_OPTIMIZATION: input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != tf.dtypes.resource ] output_tensors = frozen_func.outputs config = config_pb2.ConfigProto() rewrite_options = config.graph_options.rewrite_options rewrite_options.constant_folding = rewrite_options.ON graph_def = _run_graph_optimizations(graph_def, input_tensors, output_tensors, config=config, graph=frozen_func.graph) with tf.Graph().as_default() as tf_graph: tf.import_graph_def(graph_def, name='') return tf_graph
def _graph_def_from_concrete_fn(cfs): if len(cfs) != 1: raise NotImplementedError( "Only a single concrete function is supported.") frozen_fn = _convert_variables_to_constants_v2( cfs[0], lower_control_flow=False) graph_def = frozen_fn.graph.as_graph_def(add_shapes=True) # run a Grappler's constant folding pass. fn_inputs = [ t for t in frozen_fn.inputs if t.dtype != _dtypes.resource ] graph_def = _run_graph_optimizations( graph_def, fn_inputs, frozen_fn.outputs, config=_get_grappler_config(["constfold", "dependency"]), graph=frozen_fn.graph, ) return graph_def
def get_func_graph(model, input_signature=None): # TODO(yuwang) Use trace_model_call from keras function directly. #from tensorflow.python.keras.saving import saving_utils #func = saving_utils.trace_model_call(model, input_signature) func = keras_utils.trace_model_call(model, input_signature) concrete_func = func.get_concrete_function() frozen_func = tf_utils.convert_to_constants(concrete_func, lower_control_flow=False) graph_def = frozen_func.graph.as_graph_def() utils.maybe_export_graph('{}/{}'.format(_EXPORT_DIR, _FROZEN_FUNC_GRAPH), graph_def) input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != tf.dtypes.resource ] output_tensors = frozen_func.outputs config = config_pb2.ConfigProto() rewrite_options = config.graph_options.rewrite_options #rewrite_options.constant_folding = rewrite_options.ON rewrite_options.optimizers.append('constfold') graph_def = _run_graph_optimizations(graph_def, input_tensors, output_tensors, config=config, graph=frozen_func.graph) utils.maybe_export_graph('{}/{}'.format(_EXPORT_DIR, _OPT_TF_GRAPH), graph_def) #logging.vlog(4, 'Optimized GraphDef:\n{}'.format(graph_def)) with tf.Graph().as_default() as tf_graph: tf.import_graph_def(graph_def, name='') func_graph = concrete_func.graph return (tf_graph, func_graph.structured_input_signature, func_graph.structured_outputs)
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Either a TFLite Flatbuffer or a Graphviz graph depending on value in `output_format`. Raises: ValueError: Input shape is not specified. None value for dimension in input_tensor. """ # Checks dimensions in input tensor. if self._has_valid_tensors(): for tensor in self._input_tensors: shape = tensor.shape if not shape: raise ValueError("Provide an input shape for input array " "'{0}'.".format(_get_tensor_name(tensor))) # Note that shape_list might be empty for scalar shapes. shape_list = shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: self._set_batch_size(batch_size=1) # Get quantization stats. Ensures there is one stat per name if the stats # are specified. if self.quantized_input_stats: quantized_stats = [] invalid_stats = [] for name in self.get_input_arrays(): if name in self.quantized_input_stats: quantized_stats.append(self.quantized_input_stats[name]) else: invalid_stats.append(name) if invalid_stats: raise ValueError( "Quantization input stats are not available for input " "tensors '{0}'.".format(",".join(invalid_stats))) else: quantized_stats = None self._validate_quantization() self._validate_representative_dataset() toco_inference_input_type = self.inference_input_type inference_input_type = self.inference_input_type inference_output_type = self.inference_output_type post_training_optimize = self._is_post_training_optimize() if post_training_optimize: # Post training optimizations require that TOCO outputs a float model. if self.inference_type != constants.FLOAT: raise ValueError( "`optimizations` require that `inference_type` is set to float." ) toco_inference_input_type = constants.FLOAT # Set up default values. if inference_input_type is None: inference_input_type = constants.FLOAT if inference_output_type is None: inference_output_type = constants.FLOAT weight_only_quantize = self._is_int8_weight_only_quantize() if weight_only_quantize: # Currently, weight only quantization requires float inputs and outputs. if (inference_input_type != constants.FLOAT or inference_output_type != constants.FLOAT): raise ValueError( "Provide an inference_input_type and inference_output_type of type " "tf.float32.") if not post_training_optimize and self.inference_output_type is not None: raise ValueError( "inference_output_type is currently not supported if optimizations " "are not enabled.") optimized_graph = self._graph_def if self.inference_type != constants.QUANTIZED_UINT8: try: optimized_graph = _run_graph_optimizations( self._graph_def, self._input_tensors, self._output_tensors, config=self._grappler_config()) except Exception: optimized_graph = self._graph_def self._debug_info = _get_debug_info(self._debug_info_func, optimized_graph) converter_kwargs = self._get_base_converter_args() converter_kwargs.update({ "inference_type": self.inference_type, "inference_input_type": toco_inference_input_type, "output_format": self.output_format, "quantized_input_stats": quantized_stats, "default_ranges_stats": self.default_ranges_stats, "drop_control_dependency": self.drop_control_dependency, "reorder_across_fake_quant": self.reorder_across_fake_quant, "change_concat_input_ranges": self.change_concat_input_ranges, "dump_graphviz_dir": self.dump_graphviz_dir, "dump_graphviz_video": self.dump_graphviz_video }) # Converts model. if self._has_valid_tensors(): result = _toco_convert_impl(input_data=optimized_graph, input_tensors=self._input_tensors, output_tensors=self._output_tensors, **converter_kwargs) else: result = _toco_convert_graph_def( input_data=optimized_graph, input_arrays_with_shape=self._input_arrays_with_shape, output_arrays=self._output_arrays, **converter_kwargs) if self._is_calibration_quantize(): result = self._calibrate_quantize_model(result, inference_input_type, inference_output_type) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Raises: ValueError: Multiple concrete functions are specified. Input shape is not specified. Invalid quantization parameters. """ # TODO(b/130297984): Add support for converting multiple function. if len(self._funcs) != 1: raise ValueError( "This converter can only convert a single " "ConcreteFunction. Converting multiple functions is " "under development.") frozen_func = _convert_to_constants.convert_variables_to_constants_v2( self._funcs[0], lower_control_flow=False) input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != _dtypes.resource ] output_tensors = frozen_func.outputs # Run a Grappler pass. graph_def = frozen_func.graph.as_graph_def() graph_def = _run_graph_optimizations(graph_def, input_tensors, output_tensors, config=self._grappler_config(), graph=frozen_func.graph) # Checks dimensions in input tensor. for tensor in input_tensors: # Note that shape_list might be empty for scalar shapes. shape_list = tensor.shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: # Set the batch size to 1 if undefined. shape = tensor.shape.as_list() shape[0] = 1 tensor.set_shape(shape) self._validate_quantization() self._validate_representative_dataset() self._debug_info = _get_debug_info( _build_debug_info_func(self._funcs[0].graph), graph_def) converter_kwargs = self._get_base_converter_args() # Converts model. result = _toco_convert_impl(input_data=graph_def, input_tensors=input_tensors, output_tensors=output_tensors, **converter_kwargs) if self._is_calibration_quantize(): result = self._calibrate_quantize_model(result, constants.FLOAT, constants.FLOAT) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Either a TFLite Flatbuffer or a Graphviz graph depending on value in `output_format`. Raises: ValueError: Input shape is not specified. None value for dimension in input_tensor. """ self._target_ops = self.target_ops # Checks dimensions in input tensor. if self._has_valid_tensors(): for tensor in self._input_tensors: shape = tensor.shape if not shape: raise ValueError("Provide an input shape for input array " "'{0}'.".format(_get_tensor_name(tensor))) # Note that shape_list might be empty for scalar shapes. shape_list = shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format( _get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: self._set_batch_size(batch_size=1) # Get quantization stats. Ensures there is one stat per name if the stats # are specified. if self.quantized_input_stats: quantized_stats = [] invalid_stats = [] for name in self.get_input_arrays(): if name in self.quantized_input_stats: quantized_stats.append(self.quantized_input_stats[name]) else: invalid_stats.append(name) if invalid_stats: raise ValueError("Quantization input stats are not available for input " "tensors '{0}'.".format(",".join(invalid_stats))) else: quantized_stats = None self._validate_representative_dataset() toco_inference_input_type = self.inference_input_type inference_input_type = self.inference_input_type inference_output_type = self.inference_output_type post_training_optimize = self._is_post_training_optimize() if post_training_optimize: # Post training optimizations require that TOCO outputs a float model. if self.inference_type != constants.FLOAT: raise ValueError( "`optimizations` require that `inference_type` is set to float.") toco_inference_input_type = constants.FLOAT # Set up default values. if inference_input_type is None: inference_input_type = constants.FLOAT if inference_output_type is None: inference_output_type = constants.FLOAT weight_only_quantize = self._is_weight_only_quantize() if weight_only_quantize: # Currently, weight only quantization requires float inputs and outputs. if (inference_input_type != constants.FLOAT or inference_output_type != constants.FLOAT): raise ValueError( "Provide an inference_input_type and inference_output_type of type " "tf.float32.") if not post_training_optimize and self.inference_output_type is not None: raise ValueError( "inference_output_type is currently not supported if optimizations " "are not enabled.") converter_kwargs = { "inference_type": self.inference_type, "inference_input_type": toco_inference_input_type, "input_format": constants.TENSORFLOW_GRAPHDEF, "output_format": self.output_format, "quantized_input_stats": quantized_stats, "default_ranges_stats": self.default_ranges_stats, "drop_control_dependency": self.drop_control_dependency, "reorder_across_fake_quant": self.reorder_across_fake_quant, "change_concat_input_ranges": self.change_concat_input_ranges, "allow_custom_ops": self.allow_custom_ops, "post_training_quantize": weight_only_quantize, "target_ops": self.target_ops, "dump_graphviz_dir": self.dump_graphviz_dir, "dump_graphviz_video": self.dump_graphviz_video } optimized_graph = self._graph_def if self.inference_type != constants.QUANTIZED_UINT8: try: optimized_graph = _run_graph_optimizations( self._graph_def, self._input_tensors, self._output_tensors, config=self._grappler_config()) except Exception: optimized_graph = self._graph_def # Converts model. if self._has_valid_tensors(): result = _toco_convert_impl( input_data=optimized_graph, input_tensors=self._input_tensors, output_tensors=self._output_tensors, **converter_kwargs) else: result = _toco_convert_graph_def( input_data=optimized_graph, input_arrays_with_shape=self._input_arrays_with_shape, output_arrays=self._output_arrays, **converter_kwargs) if self._is_calibration_quantize(): result = self._calibrate_quantize_model(result, inference_input_type, inference_output_type) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Raises: ValueError: Multiple concrete functions are specified. Input shape is not specified. Invalid quantization parameters. """ # TODO(b/130297984): Add support for converting multiple function. self._target_ops = self.target_spec.supported_ops if len(self._funcs) != 1: raise ValueError("This converter can only convert a single " "ConcreteFunction. Converting multiple functions is " "under development.") frozen_func = _convert_to_constants.convert_variables_to_constants_v2( self._funcs[0]) input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != _dtypes.resource ] output_tensors = frozen_func.outputs # Run a Grappler pass. graph_def = frozen_func.graph.as_graph_def() graph_def = _run_graph_optimizations( graph_def, input_tensors, output_tensors, config=self._grappler_config(), graph=frozen_func.graph) # Checks dimensions in input tensor. for tensor in input_tensors: # Note that shape_list might be empty for scalar shapes. shape_list = tensor.shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: # Set the batch size to 1 if undefined. shape = tensor.shape.as_list() shape[0] = 1 tensor.set_shape(shape) self._validate_representative_dataset() converter_kwargs = { "input_format": constants.TENSORFLOW_GRAPHDEF, "allow_custom_ops": self.allow_custom_ops, "post_training_quantize": self._is_weight_only_quantize(), "target_ops": self.target_spec.supported_ops, } # Converts model. result = _toco_convert_impl( input_data=graph_def, input_tensors=input_tensors, output_tensors=output_tensors, **converter_kwargs) if self._is_calibration_quantize(): result = self._calibrate_quantize_model(result, constants.FLOAT, constants.FLOAT) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Either a TFLite Flatbuffer or a Graphviz graph depending on value in `output_format`. Raises: ValueError: Input shape is not specified. None value for dimension in input_tensor. """ # Checks dimensions in input tensor. if self._has_valid_tensors(): for tensor in self._input_tensors: shape = tensor.shape if not shape: raise ValueError("Provide an input shape for input array " "'{0}'.".format(_get_tensor_name(tensor))) # Note that shape_list might be empty for scalar shapes. shape_list = shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: self._set_batch_size(batch_size=1) # Get quantization stats. Ensures there is one stat per name if the stats # are specified. if self.quantized_input_stats: quantized_stats = [] invalid_stats = [] for name in self.get_input_arrays(): if name in self.quantized_input_stats: quantized_stats.append(self.quantized_input_stats[name]) else: invalid_stats.append(name) if invalid_stats: raise ValueError( "Quantization input stats are not available for input " "tensors '{0}'.".format(",".join(invalid_stats))) else: quantized_stats = None if self.representative_dataset: if not isinstance(self.representative_dataset, RepresentativeDataset): raise TypeError( "representative_dataset must be an instance of " "RepresentativeDataset") if self.representative_dataset.input_gen is None: raise ValueError( "Provide an input generator for representative_dataset") # TODO(shashishekhar): For now use optimizations order is ignored. # Both size and latency optimizations decide whether to apply post # training optimizations. post_training_optimize = bool( len( set(self.optimizations) & set([ Optimize.OPTIMIZE_FOR_LATENCY, Optimize.OPTIMIZE_FOR_SIZE ]))) # Do weights only quantization if there is no dataset for calibration. weights_only_quantize_flag = (post_training_optimize and (self.representative_dataset is None)) converter_kwargs = { "inference_type": self.inference_type, "inference_input_type": self.inference_input_type, "input_format": constants.TENSORFLOW_GRAPHDEF, "output_format": self.output_format, "quantized_input_stats": quantized_stats, "default_ranges_stats": self.default_ranges_stats, "drop_control_dependency": self.drop_control_dependency, "reorder_across_fake_quant": self.reorder_across_fake_quant, "change_concat_input_ranges": self.change_concat_input_ranges, "allow_custom_ops": self.allow_custom_ops, "post_training_quantize": weights_only_quantize_flag, "target_ops": self.target_ops, "dump_graphviz_dir": self.dump_graphviz_dir, "dump_graphviz_video": self.dump_graphviz_video } optimized_graph = None if self.inference_type == constants.QUANTIZED_UINT8: optimized_graph = self._graph_def else: try: is_only_flex_enabled = set([OpsSet.SELECT_TF_OPS ]) == self.target_ops config = _get_grappler_config( enable_layout_optimizer=is_only_flex_enabled) optimized_graph = _run_graph_optimizations( self._graph_def, self._input_tensors, self._output_tensors, config) except Exception: optimized_graph = self._graph_def # Converts model. if self._has_valid_tensors(): result = _toco_convert_impl(input_data=optimized_graph, input_tensors=self._input_tensors, output_tensors=self._output_tensors, **converter_kwargs) else: result = _toco_convert_graph_def( input_data=optimized_graph, input_arrays_with_shape=self._input_arrays_with_shape, output_arrays=self._output_arrays, **converter_kwargs) if self.representative_dataset and post_training_optimize: calibrate_quantize = _calibrator.Calibrator(result) result = calibrate_quantize.calibrate_and_quantize( self.representative_dataset.input_gen) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Raises: ValueError: Input shape is not specified. None value for dimension in input_tensor. """ frozen_func = _convert_to_constants.convert_variables_to_constants_v2( self._func) input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != _dtypes.resource ] output_tensors = frozen_func.outputs # Run a Grappler pass. is_only_flex_enabled = set([OpsSet.SELECT_TF_OPS ]) == self.target_spec.supported_ops config = _get_grappler_config( enable_layout_optimizer=is_only_flex_enabled) graph_def = _run_graph_optimizations(frozen_func.graph.as_graph_def(), input_tensors, output_tensors, config, graph=frozen_func.graph) # Checks dimensions in input tensor. for tensor in input_tensors: # Note that shape_list might be empty for scalar shapes. shape_list = tensor.shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: # Set the batch size to 1 if undefined. shape = tensor.shape.as_list() shape[0] = 1 tensor.set_shape(shape) if self.representative_dataset: if not isinstance(self.representative_dataset, RepresentativeDataset): raise TypeError( "`representative_dataset` must be an instance of " "`RepresentativeDataset`") if self.representative_dataset.input_gen is None: raise ValueError( "Provide an input generator for `representative_dataset`") # TODO(shashishekhar): For now use optimizations order is ignored. # Both size and latency optimizations decide whether to apply post # training optimizations. post_training_optimize = bool( len( set(self.optimizations) & set([ Optimize.OPTIMIZE_FOR_LATENCY, Optimize.OPTIMIZE_FOR_SIZE ]))) # Do weights only quantization if there is no dataset for calibration. weights_only_quantize_flag = (post_training_optimize and (self.representative_dataset is None)) converter_kwargs = { "input_format": constants.TENSORFLOW_GRAPHDEF, "allow_custom_ops": self.allow_custom_ops, "post_training_quantize": weights_only_quantize_flag, "target_ops": self.target_spec.supported_ops, } # Converts model. result = _toco_convert_impl(input_data=graph_def, input_tensors=input_tensors, output_tensors=output_tensors, **converter_kwargs) if self.representative_dataset and post_training_optimize: calibrate_quantize = _calibrator.Calibrator(result) result = calibrate_quantize.calibrate_and_quantize( self.representative_dataset.input_gen) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Raises: ValueError: Multiple concrete functions are specified. Input shape is not specified. Invalid quantization parameters. """ # TODO(b/130297984): Add support for converting multiple function. if len(self._funcs) != 1: raise ValueError("This converter can only convert a single " "ConcreteFunction. Converting multiple functions is " "under development.") # graph_def is used here to preserve the node bug information frozen_func, graph_def = ( _convert_to_constants.convert_variables_to_constants_v2_as_graph( self._funcs[0], lower_control_flow=False)) input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != _dtypes.resource ] output_tensors = frozen_func.outputs # Run a Grappler pass. graph_def = _run_graph_optimizations( graph_def, input_tensors, output_tensors, config=self._grappler_config(), graph=frozen_func.graph) # Checks dimensions in input tensor. for tensor in input_tensors: # Note that shape_list might be empty for scalar shapes. shape_list = tensor.shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: # Set the batch size to 1 if undefined. shape = tensor.shape.as_list() shape[0] = 1 tensor.set_shape(shape) self._validate_quantization() self._validate_representative_dataset() if self._trackable_obj is None: self._debug_info = _get_debug_info( _build_debug_info_func(self._funcs[0].graph), graph_def) else: self._debug_info = _get_debug_info( _convert_debug_info_func(self._trackable_obj.graph_debug_info), graph_def) converter_kwargs = self._get_base_converter_args() if not self.experimental_new_converter: logging.warning( "Please consider switching to use new converter by setting " "experimental_new_converter to true. " "Old converter (TOCO) is deprecated and flow will be switched on " "by default to use new converter soon.") else: logging.info("Using experimental converter: If you encountered a problem " "please file a bug. You can opt-out " "by setting experimental_new_converter=False") # Converts model. result = _toco_convert_impl( input_data=graph_def, input_tensors=input_tensors, output_tensors=output_tensors, **converter_kwargs) if self._is_calibration_quantize(): result = self._calibrate_quantize_model( result, constants.FLOAT, constants.FLOAT, self.experimental_new_quantizer) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Either a TFLite Flatbuffer or a Graphviz graph depending on value in `output_format`. Raises: ValueError: Input shape is not specified. None value for dimension in input_tensor. """ # Checks dimensions in input tensor. if self._has_valid_tensors(): for tensor in self._input_tensors: shape = tensor.shape if not shape: raise ValueError("Provide an input shape for input array " "'{0}'.".format(_get_tensor_name(tensor))) # Note that shape_list might be empty for scalar shapes. shape_list = shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format( _get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: self._set_batch_size(batch_size=1) # Get quantization stats. Ensures there is one stat per name if the stats # are specified. if self.quantized_input_stats: quantized_stats = [] invalid_stats = [] for name in self.get_input_arrays(): if name in self.quantized_input_stats: quantized_stats.append(self.quantized_input_stats[name]) else: invalid_stats.append(name) if invalid_stats: raise ValueError("Quantization input stats are not available for input " "tensors '{0}'.".format(",".join(invalid_stats))) else: quantized_stats = None self._validate_quantization() self._validate_representative_dataset() toco_inference_input_type = self.inference_input_type inference_input_type = self.inference_input_type inference_output_type = self.inference_output_type post_training_optimize = self._is_post_training_optimize() if post_training_optimize: # Post training optimizations require that TOCO outputs a float model. if self.inference_type != constants.FLOAT: raise ValueError( "`optimizations` require that `inference_type` is set to float.") toco_inference_input_type = constants.FLOAT # Set up default values. if inference_input_type is None: inference_input_type = constants.FLOAT if inference_output_type is None: inference_output_type = constants.FLOAT weight_only_quantize = self._is_int8_weight_only_quantize() if weight_only_quantize: # Currently, weight only quantization requires float inputs and outputs. if (inference_input_type != constants.FLOAT or inference_output_type != constants.FLOAT): raise ValueError( "Provide an inference_input_type and inference_output_type of type " "tf.float32.") if not post_training_optimize and self.inference_output_type is not None: raise ValueError( "inference_output_type is currently not supported if optimizations " "are not enabled.") optimized_graph = self._graph_def # if it is not uint8 or int8 with post-training quantization, it is not # quantization aware training, then graph optimization is applied. # Graph optimization is disabled for quantization aware training. if (self.inference_type != constants.QUANTIZED_UINT8 or (self.inference_type == constants.INT8 and (post_training_optimize or weight_only_quantize))): try: # Run function inlining optimization to ensure any models generated # through the from_frozen_graph path have been inlined. optimized_graph = _run_graph_optimizations( self._graph_def, self._input_tensors, self._output_tensors, config=self._grappler_config(["function"])) except Exception: optimized_graph = self._graph_def self._debug_info = _get_debug_info(self._debug_info_func, optimized_graph) converter_kwargs = self._get_base_converter_args() converter_kwargs.update({ "inference_type": self.inference_type, "inference_input_type": toco_inference_input_type, "output_format": self.output_format, "quantized_input_stats": quantized_stats, "default_ranges_stats": self.default_ranges_stats, "drop_control_dependency": self.drop_control_dependency, "reorder_across_fake_quant": self.reorder_across_fake_quant, "change_concat_input_ranges": self.change_concat_input_ranges, "dump_graphviz_dir": self.dump_graphviz_dir, "dump_graphviz_video": self.dump_graphviz_video, "conversion_summary_dir": self.conversion_summary_dir, "custom_opdefs": self._custom_opdefs, }) if not self.experimental_new_converter: logging.warning( "Please consider switching to use new converter by setting " "experimental_new_converter to true. " "Old converter (TOCO) is deprecated and flow will be switched on " "by default to use new converter soon.") else: logging.info("Using experimental converter: If you encountered a problem " "please file a bug. You can opt-out " "by setting experimental_new_converter=False") # Converts model. if self._has_valid_tensors(): result = _toco_convert_impl( input_data=optimized_graph, input_tensors=self._input_tensors, output_tensors=self._output_tensors, **converter_kwargs) else: result = _toco_convert_graph_def( input_data=optimized_graph, input_arrays_with_shape=self._input_arrays_with_shape, output_arrays=self._output_arrays, **converter_kwargs) if self._is_calibration_quantize(): result = self._calibrate_quantize_model( result, inference_input_type, inference_output_type, self.experimental_new_quantizer) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Either a TFLite Flatbuffer or a Graphviz graph depending on value in `output_format`. Raises: ValueError: Input shape is not specified. None value for dimension in input_tensor. """ # Checks dimensions in input tensor. if self._has_valid_tensors(): for tensor in self._input_tensors: shape = tensor.shape if not shape: raise ValueError("Provide an input shape for input array " "'{0}'.".format(_get_tensor_name(tensor))) # Note that shape_list might be empty for scalar shapes. shape_list = shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format( _get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: self._set_batch_size(batch_size=1) # Get quantization stats. Ensures there is one stat per name if the stats # are specified. if self.quantized_input_stats: quantized_stats = [] invalid_stats = [] for name in self.get_input_arrays(): if name in self.quantized_input_stats: quantized_stats.append(self.quantized_input_stats[name]) else: invalid_stats.append(name) if invalid_stats: raise ValueError("Quantization input stats are not available for input " "tensors '{0}'.".format(",".join(invalid_stats))) else: quantized_stats = None if self.representative_dataset: if not isinstance(self.representative_dataset, RepresentativeDataset): raise TypeError( "representative_dataset must be an instance of " "RepresentativeDataset") if self.representative_dataset.input_gen is None: raise ValueError( "Provide an input generator for representative_dataset") # TODO(shashishekhar): For now use optimizations order is ignored. # Both size and latency optimizations decide whether to apply post # training optimizations. post_training_optimize = bool( len(set(self.optimizations) & set([Optimize.OPTIMIZE_FOR_LATENCY, Optimize.OPTIMIZE_FOR_SIZE]))) # Do weights only quantization if there is no dataset for calibration. weights_only_quantize_flag = ( post_training_optimize and (self.representative_dataset is None)) converter_kwargs = { "inference_type": self.inference_type, "inference_input_type": self.inference_input_type, "input_format": constants.TENSORFLOW_GRAPHDEF, "output_format": self.output_format, "quantized_input_stats": quantized_stats, "default_ranges_stats": self.default_ranges_stats, "drop_control_dependency": self.drop_control_dependency, "reorder_across_fake_quant": self.reorder_across_fake_quant, "change_concat_input_ranges": self.change_concat_input_ranges, "allow_custom_ops": self.allow_custom_ops, "post_training_quantize": weights_only_quantize_flag, "target_ops": self.target_ops, "dump_graphviz_dir": self.dump_graphviz_dir, "dump_graphviz_video": self.dump_graphviz_video } optimized_graph = None if self.inference_type == constants.QUANTIZED_UINT8: optimized_graph = self._graph_def else: try: is_only_flex_enabled = set([OpsSet.SELECT_TF_OPS]) == self.target_ops config = _get_grappler_config( enable_layout_optimizer=is_only_flex_enabled) optimized_graph = _run_graph_optimizations( self._graph_def, self._input_tensors, self._output_tensors, config) except Exception: optimized_graph = self._graph_def # Converts model. if self._has_valid_tensors(): result = _toco_convert_impl( input_data=optimized_graph, input_tensors=self._input_tensors, output_tensors=self._output_tensors, **converter_kwargs) else: result = _toco_convert_graph_def( input_data=optimized_graph, input_arrays_with_shape=self._input_arrays_with_shape, output_arrays=self._output_arrays, **converter_kwargs) if self.representative_dataset and post_training_optimize: calibrate_quantize = _calibrator.Calibrator(result) result = calibrate_quantize.calibrate_and_quantize( self.representative_dataset.input_gen) return result
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Raises: ValueError: Input shape is not specified. None value for dimension in input_tensor. """ frozen_func = _convert_to_constants.convert_variables_to_constants_v2( self._func) input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != _dtypes.resource ] output_tensors = frozen_func.outputs # Run a Grappler pass. is_only_flex_enabled = set( [OpsSet.SELECT_TF_OPS]) == self.target_spec.supported_ops config = _get_grappler_config(enable_layout_optimizer=is_only_flex_enabled) graph_def = _run_graph_optimizations( frozen_func.graph.as_graph_def(), input_tensors, output_tensors, config, graph=frozen_func.graph) # Checks dimensions in input tensor. for tensor in input_tensors: # Note that shape_list might be empty for scalar shapes. shape_list = tensor.shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: # Set the batch size to 1 if undefined. shape = tensor.shape.as_list() shape[0] = 1 tensor.set_shape(shape) if self.representative_dataset: if not isinstance(self.representative_dataset, RepresentativeDataset): raise TypeError("`representative_dataset` must be an instance of " "`RepresentativeDataset`") if self.representative_dataset.input_gen is None: raise ValueError( "Provide an input generator for `representative_dataset`") # TODO(shashishekhar): For now use optimizations order is ignored. # Both size and latency optimizations decide whether to apply post # training optimizations. post_training_optimize = bool( len( set(self.optimizations) & set([Optimize.OPTIMIZE_FOR_LATENCY, Optimize.OPTIMIZE_FOR_SIZE]))) # Do weights only quantization if there is no dataset for calibration. weights_only_quantize_flag = ( post_training_optimize and (self.representative_dataset is None)) converter_kwargs = { "input_format": constants.TENSORFLOW_GRAPHDEF, "allow_custom_ops": self.allow_custom_ops, "post_training_quantize": weights_only_quantize_flag, "target_ops": self.target_spec.supported_ops, } # Converts model. result = _toco_convert_impl( input_data=graph_def, input_tensors=input_tensors, output_tensors=output_tensors, **converter_kwargs) if self.representative_dataset and post_training_optimize: calibrate_quantize = _calibrator.Calibrator(result) result = calibrate_quantize.calibrate_and_quantize( self.representative_dataset.input_gen) return result