def testAggregate(self): a = array_ops.constant([3., 4.]) b = array_ops.constant([5., 6.]) hint = op_hint.OpHint("agg") a0, a1 = array_ops.unstack(a) b0, b1 = array_ops.unstack(b) a0 = hint.add_input(a0, tag="c", aggregate=op_hint.OpHint.AGGREGATE_STACK) b0 = hint.add_input(b0, tag="n", aggregate=op_hint.OpHint.AGGREGATE_STACK) a1 = hint.add_input(a1, tag="c", aggregate=op_hint.OpHint.AGGREGATE_STACK) b1 = hint.add_input(b1, tag="n", aggregate=op_hint.OpHint.AGGREGATE_STACK) c0 = math_ops.add(a0, b0, name="addleft") c1 = math_ops.add(a1, b1, name="addright") c0 = hint.add_output( c0, tag="out", aggregate=op_hint.OpHint.AGGREGATE_STACK) c1 = hint.add_output( c1, tag="out", aggregate=op_hint.OpHint.AGGREGATE_STACK) curr = array_ops.stack([c0, c1]) output = array_ops.identity(curr, name="FINAL_OUTPUT") with self.cached_session() as sess: stubbed_graphdef = op_hint.convert_op_hints_to_stubs( graph_def=sess.graph_def) self.assertEqual( self._getGraphOpTypes( stubbed_graphdef, output_nodes=[op_hint._tensor_name_base(output.name)]), set(["agg", "Const", "Identity"]))
def testScaleAndBiasAndIdentity(self): """This tests a scaled add which has 3 inputs and 2 outputs.""" a = array_ops.constant(1.) x = array_ops.constant([2., 3.]) b = array_ops.constant([4., 5.]) def _scaled_and_bias_and_identity(a, x, b): custom = op_hint.OpHint("scale_and_bias_and_identity") a, x, b = custom.add_inputs(a, x, b) return custom.add_outputs(a * x + b, x) output = array_ops.identity(_scaled_and_bias_and_identity(a, x, b), name="ModelOutput") with self.cached_session() as sess: # make sure one identity for each input (3) and output (2) => 3 + 2 = 5 # +1 for the final output self.assertEqual(self._countIdentities(sess.graph_def.node), 6) stubbed_graphdef = op_hint.convert_op_hints_to_stubs( graph_def=sess.graph_def) self.assertEqual( self._getGraphOpTypes( stubbed_graphdef, output_nodes=[op_hint._tensor_name_base(output.name)]), set(["scale_and_bias_and_identity", "Const", "Identity", "Pack"]))
def testScaleAndBiasAndIdentity(self): """This tests a scaled add which has 3 inputs and 2 outputs.""" with ops.Graph().as_default(): a = array_ops.constant(1.) x = array_ops.constant([2., 3.]) b = array_ops.constant([4., 5.]) def _scaled_and_bias_and_identity(a, x, b): custom = op_hint.OpHint("scale_and_bias_and_identity") a, x, b = custom.add_inputs(a, x, b) return custom.add_outputs(a * x + b, x) output = array_ops.identity(_scaled_and_bias_and_identity(a, x, b), name="ModelOutput") with self.cached_session() as sess: # make sure one identity for each input (3) and output (2) => 3 + 2 = 5 # +1 for the final output self.assertEqual(self._countIdentities(sess.graph_def.node), 6) stubbed_graphdef = op_hint.convert_op_hints_to_stubs( graph_def=sess.graph_def) self.assertEqual( self._getGraphOpTypes( stubbed_graphdef, output_nodes=[op_hint._tensor_name_base(output.name)]), set([ "scale_and_bias_and_identity", "Const", "Identity", "Pack" ]))
def testSwishLiteHint(self): """Makes a custom op swish and makes sure it gets converted as a unit.""" image = array_ops.constant([1., 2., 3., 4.]) swish_scale = array_ops.constant(1.0) def _swish(input_tensor, scale): custom = op_hint.OpHint("cool_activation") input_tensor, scale = custom.add_inputs(input_tensor, scale) output = math_ops.sigmoid(input_tensor) * input_tensor * scale output, = custom.add_outputs(output) return output output = array_ops.identity(_swish(image, swish_scale), name="ModelOutput") with self.cached_session() as sess: # check if identities have been put into the graph (2 input, 1 output, # and 1 final output). self.assertEqual(self._countIdentities(sess.graph_def.node), 4) stubbed_graphdef = op_hint.convert_op_hints_to_stubs( graph_def=sess.graph_def) self.assertEqual( self._getGraphOpTypes( stubbed_graphdef, output_nodes=[op_hint._tensor_name_base(output.name)]), set(["cool_activation", "Const", "Identity"]))
def tfliteInvoke(self, graph, test_inputs, outputs): tf.reset_default_graph() # Turn the input into placeholder of shape 1 tflite_input = tf.placeholder( "float", [1, self.time_steps, self.n_input], name="INPUT_IMAGE_LITE") tf.import_graph_def(graph, name="", input_map={"INPUT_IMAGE": tflite_input}) with tf.Session() as sess: curr = sess.graph_def curr = convert_op_hints_to_stubs(graph_def=curr) curr = optimize_for_inference_lib.optimize_for_inference( curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"], [tf.float32.as_datatype_enum]) tflite = tf.lite.toco_convert( curr, [tflite_input], [outputs], allow_custom_ops=False) interpreter = tf.lite.Interpreter(model_content=tflite) try: interpreter.allocate_tensors() except ValueError: assert False input_index = (interpreter.get_input_details()[0]["index"]) interpreter.set_tensor(input_index, test_inputs) interpreter.invoke() output_index = (interpreter.get_output_details()[0]["index"]) result = interpreter.get_tensor(output_index) # Reset all variables so it will not pollute other inferences. interpreter.reset_all_variables() return result
def testSwishLiteHint(self): """Makes a custom op swish and makes sure it gets converted as a unit.""" with ops.Graph().as_default(): image = array_ops.constant([1., 2., 3., 4.]) swish_scale = array_ops.constant(1.0) def _swish(input_tensor, scale): custom = op_hint.OpHint("cool_activation") input_tensor, scale = custom.add_inputs(input_tensor, scale) output = math_ops.sigmoid(input_tensor) * input_tensor * scale output, = custom.add_outputs(output) return output output = array_ops.identity(_swish(image, swish_scale), name="ModelOutput") with self.cached_session() as sess: # check if identities have been put into the graph (2 input, 1 output, # and 1 final output). self.assertEqual(self._countIdentities(sess.graph_def.node), 4) stubbed_graphdef = op_hint.convert_op_hints_to_stubs( graph_def=sess.graph_def) self.assertEqual( self._getGraphOpTypes( stubbed_graphdef, output_nodes=[op_hint._tensor_name_base(output.name)]), set(["cool_activation", "Const", "Identity"]))
def testTwoFunctions(self): """Tests if two functions are converted correctly.""" with ops.Graph().as_default(): a = array_ops.constant([1.]) b = array_ops.constant([1.]) def _double_values(x): custom = op_hint.OpHint("add_test") x, = custom.add_inputs(x) output = math_ops.multiply(x, x) output, = custom.add_outputs(output) return output output = array_ops.identity(math_ops.add(_double_values(a), _double_values(b)), name="ModelOutput") with self.cached_session() as sess: # make sure one identity for each input (2) and output (2) => 2 + 2 # +1 for the final output self.assertEqual(self._countIdentities(sess.graph_def.node), 5) stubbed_graphdef = op_hint.convert_op_hints_to_stubs( graph_def=sess.graph_def) self.assertEqual( self._getGraphOpTypes( stubbed_graphdef, output_nodes=[op_hint._tensor_name_base(output.name)]), set(["add_test", "Const", "Identity", "AddV2"]))
def tfliteInvoke(self, graph, test_inputs, outputs): tf.reset_default_graph() # Turn the input into placeholder of shape 1 tflite_input = tf.placeholder("float", [1, self.time_steps, self.n_input], name="INPUT_IMAGE_LITE") tf.import_graph_def(graph, name="", input_map={"INPUT_IMAGE": tflite_input}) with tf.Session() as sess: curr = sess.graph_def curr = convert_op_hints_to_stubs(graph_def=curr) curr = optimize_for_inference_lib.optimize_for_inference( curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"], [tf.float32.as_datatype_enum]) converter = tf.lite.TFLiteConverter(curr, [tflite_input], [outputs]) tflite = converter.convert() interpreter = tf.lite.Interpreter(model_content=tflite) interpreter.allocate_tensors() input_index = interpreter.get_input_details()[0]["index"] interpreter.set_tensor(input_index, test_inputs) interpreter.invoke() output_index = interpreter.get_output_details()[0]["index"] result = interpreter.get_tensor(output_index) # Reset all variables so it will not pollute other inferences. interpreter.reset_all_variables() return result
def _convert_op_hints_if_present(sess, graph_def, output_tensors, hinted_outputs_nodes): if is_frozen_graph(sess): raise ValueError("Try to convert op hints, needs unfrozen graph.") output_arrays = [get_tensor_name(tensor) for tensor in output_tensors] graph_def = tf_graph_util.convert_variables_to_constants( sess, graph_def, output_arrays + hinted_outputs_nodes) graph_def = convert_op_hints_to_stubs(graph_def=graph_def) return graph_def
def _convert_op_hints_if_present(sess, output_tensors): if is_frozen_graph(sess): raise ValueError("Try to convert op hints, needs unfrozen graph.") hinted_outputs_nodes = find_all_hinted_output_nodes(sess) output_arrays = [get_tensor_name(tensor) for tensor in output_tensors] graph_def = tf_graph_util.convert_variables_to_constants( sess, sess.graph_def, output_arrays + hinted_outputs_nodes) graph_def = convert_op_hints_to_stubs(graph_def=graph_def) graph_def = tf_graph_util.remove_training_nodes(graph_def) return graph_def
def tfliteInvoke(self, graph, test_inputs, outputs): tf.reset_default_graph() # input text placeholder sentences = tf.placeholder(tf.int32, [TEST_SAMPLES, self.time_steps], name="INPUT_TEXT_LITE") lengths = tf.placeholder(tf.int32, [TEST_SAMPLES], name="INPUT_LENGTH_LITE") tf.import_graph_def(graph, name="", input_map={ "INPUT_TEXT": sentences, "INPUT_LENGTH": lengths }) with tf.Session() as sess: curr = sess.graph_def curr = convert_op_hints_to_stubs(graph_def=curr) curr = optimize_for_inference_lib.optimize_for_inference( curr, ["INPUT_TEXT_LITE", "INPUT_LENGTH_LITE"], ["OUTPUT_CLASS"], [tf.int32.as_datatype_enum, tf.int32.as_datatype_enum]) converter = tf.lite.TFLiteConverter(curr, [sentences, lengths], [outputs]) tflite = converter.convert() interpreter = tf.lite.Interpreter(model_content=tflite) try: interpreter.allocate_tensors() except ValueError: assert False # first input (sentences) input_index = (interpreter.get_input_details()[0]["index"]) interpreter.set_tensor(input_index, test_inputs[0]) # second input (lengths) input_index = (interpreter.get_input_details()[1]["index"]) interpreter.set_tensor(input_index, test_inputs[1]) interpreter.invoke() output_index = (interpreter.get_output_details()[0]["index"]) result = interpreter.get_tensor(output_index) # Reset all variables so it will not pollute other inferences. interpreter.reset_all_variables() return result
def testTwoFunctions(self): """Tests if two functions are converted correctly.""" a = array_ops.constant([1.]) b = array_ops.constant([1.]) def _double_values(x): custom = op_hint.OpHint("add_test") x, = custom.add_inputs(x) output = math_ops.multiply(x, x) output, = custom.add_outputs(output) return output output = array_ops.identity( math_ops.add(_double_values(a), _double_values(b)), name="ModelOutput") with self.cached_session() as sess: # make sure one identity for each input (2) and output (2) => 2 + 2 # +1 for the final output self.assertEqual(self._countIdentities(sess.graph_def.node), 5) stubbed_graphdef = op_hint.convert_op_hints_to_stubs( graph_def=sess.graph_def) self.assertEqual( self._getGraphOpTypes( stubbed_graphdef, output_nodes=[op_hint._tensor_name_base(output.name)]), set(["add_test", "Const", "Identity", "Add"]))
def convert(self): """Converts a TensorFlow GraphDef based on instance variables. Returns: The converted data in serialized format. Either a TFLite Flatbuffer or a Graphviz graph depending on value in `output_format`. Raises: ValueError: Input shape is not specified. None value for dimension in input_tensor. """ # If ophints are present, just convert them if mlir_converter is not # explicitly specified. # For MLIRConverter, The ophint conversion will be handled inside MLIR # passes. if not self.experimental_enable_mlir_converter: hinted_outputs_nodes = _find_all_hinted_output_nodes( graph_def=self._graph_def) if hinted_outputs_nodes: self._graph_def = convert_op_hints_to_stubs( graph_def=self._graph_def) # Checks dimensions in input tensor. if self._has_valid_tensors(): for tensor in self._input_tensors: shape = tensor.shape if not shape: raise ValueError("Provide an input shape for input array " "'{0}'.".format(_get_tensor_name(tensor))) # Note that shape_list might be empty for scalar shapes. shape_list = shape.as_list() if None in shape_list[1:]: raise ValueError( "None is only supported in the 1st dimension. Tensor '{0}' has " "invalid shape '{1}'.".format(_get_tensor_name(tensor), shape_list)) elif shape_list and shape_list[0] is None: self._set_batch_size(batch_size=1) # Get quantization stats. Ensures there is one stat per name if the stats # are specified. if self.quantized_input_stats: quantized_stats = [] invalid_stats = [] for name in self.get_input_arrays(): if name in self.quantized_input_stats: quantized_stats.append(self.quantized_input_stats[name]) else: invalid_stats.append(name) if invalid_stats: raise ValueError( "Quantization input stats are not available for input " "tensors '{0}'.".format(",".join(invalid_stats))) else: quantized_stats = None self._validate_quantization() self._validate_representative_dataset() toco_inference_input_type = self.inference_input_type inference_input_type = self.inference_input_type inference_output_type = self.inference_output_type post_training_optimize = self._is_post_training_optimize() if post_training_optimize: # Post training optimizations require that TOCO outputs a float model. if self.inference_type != constants.FLOAT: raise ValueError( "`optimizations` require that `inference_type` is set to float." ) toco_inference_input_type = constants.FLOAT # Set up default values. if inference_input_type is None: inference_input_type = constants.FLOAT if inference_output_type is None: inference_output_type = constants.FLOAT weight_only_quantize = self._is_int8_weight_only_quantize() if weight_only_quantize: # Currently, weight only quantization requires float inputs and outputs. if (inference_input_type != constants.FLOAT or inference_output_type != constants.FLOAT): raise ValueError( "Provide an inference_input_type and inference_output_type of type " "tf.float32.") if not post_training_optimize and self.inference_output_type is not None: raise ValueError( "inference_output_type is currently not supported if optimizations " "are not enabled.") optimized_graph = self._graph_def if self.inference_type != constants.QUANTIZED_UINT8: try: optimized_graph = _run_graph_optimizations( self._graph_def, self._input_tensors, self._output_tensors, config=self._grappler_config()) except Exception: optimized_graph = self._graph_def self._debug_info = _get_debug_info(self._debug_info_func, optimized_graph) converter_kwargs = self._get_base_converter_args() converter_kwargs.update({ "inference_type": self.inference_type, "inference_input_type": toco_inference_input_type, "output_format": self.output_format, "quantized_input_stats": quantized_stats, "default_ranges_stats": self.default_ranges_stats, "drop_control_dependency": self.drop_control_dependency, "reorder_across_fake_quant": self.reorder_across_fake_quant, "change_concat_input_ranges": self.change_concat_input_ranges, "dump_graphviz_dir": self.dump_graphviz_dir, "dump_graphviz_video": self.dump_graphviz_video }) # Converts model. if self._has_valid_tensors(): result = _toco_convert_impl(input_data=optimized_graph, input_tensors=self._input_tensors, output_tensors=self._output_tensors, **converter_kwargs) else: result = _toco_convert_graph_def( input_data=optimized_graph, input_arrays_with_shape=self._input_arrays_with_shape, output_arrays=self._output_arrays, **converter_kwargs) if self._is_calibration_quantize(): result = self._calibrate_quantize_model(result, inference_input_type, inference_output_type) return result