def test_tensor_data(self): tensors = { "empty_tensor": np.array([], dtype=np.float32), "multi_dim_empty_tensor": np.array([[], []], dtype=np.float32), "scalar": np.array(1., dtype=np.float32), "one_item_array": np.array([1.], dtype=np.float32), "normal_array": np.array([[1., 2.], [2., 3.]], dtype=np.float32) } tf_reset_default_graph() with tf_session() as sess: for n, data in tensors.items(): tf.constant(data, dtype=tf.float32, name=n) for tf_node in sess.graph.get_operations(): name = tf_node.name self.assertTrue(name in tensors.keys()) self.assertTrue("value" in tf_node.node_def.attr) # convert to onnx tensor value tensor_value = tf_utils.tf_to_onnx_tensor( tf_utils.get_tf_node_attr(tf_node, "value"), name=utils.port_name(tf_node.name) ) attr = helper.make_attribute("value", tensor_value) # same as node.get_tensor_value(is_list=False) actual = numpy_helper.to_array(helper.get_attribute_value(attr)) expected = tensors[name] self.assertTrue(np.array_equal(expected, actual))
def setUp(self): self.config = get_test_config() tf_reset_default_graph() # reset name generation on every test utils.INTERNAL_NAME = 1 np.random.seed(1) # Make it reproducible. self.logger = logging.getLogger(self.__class__.__name__)
def test_map_fn(self): def fn0(elem): res = elem + elem * elem return res def fn1(elem): res = elem[0] * elem[1] + elem[0] return res x_val = 100 * np.random.random_sample([2, 10]).astype(np.float32) y_val = 100 * np.random.random_sample([2, 10]).astype(np.float32) # test fn0 x = tf_placeholder(tf.float32, shape=x_val.shape, name="input_0") x_ = tf.identity(x) res_ = tf.map_fn(fn0, x_, dtype=tf.float32) _ = tf.identity(res_, name="output_0") input_names_with_port = ["input_0:0"] output_names_with_port = ["output_0:0"] self._run_test_case(input_names_with_port, output_names_with_port) tf_reset_default_graph() # test fn1 x = tf_placeholder(tf.float32, shape=x_val.shape, name="input_0") y = tf_placeholder(tf.float32, shape=y_val.shape, name="input_1") x_ = tf.identity(x) y_ = tf.identity(y) res_ = tf.map_fn(fn1, (x_, y_), dtype=tf.float32) _ = tf.identity(res_, name="output_0") input_names_with_port = ["input_0:0", "input_1:0"] output_names_with_port = ["output_0:0"] self._run_test_case(input_names_with_port, output_names_with_port)
def convert_to_tflite(self, graph_def, feed_dict, outputs): if not feed_dict: return None # Can't make TFlite model with no inputs tf_reset_default_graph() with tf_session() as sess: tf.import_graph_def(graph_def, name='') sess_inputs = [ sess.graph.get_tensor_by_name(k) for k in feed_dict.keys() ] sess_outputs = [sess.graph.get_tensor_by_name(n) for n in outputs] converter = tf_lite.TFLiteConverter.from_session( sess, sess_inputs, sess_outputs) #converter.optimizations = [tf.lite.Optimize.DEFAULT] converter.target_spec.supported_ops = [ tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops. tf.lite.OpsSet.SELECT_TF_OPS, # enable TensorFlow flex ops. ] from tensorflow.lite.python.convert import ConverterError try: tflite_model = converter.convert() tflite_path = os.path.join(self.test_data_directory, self._testMethodName + ".tflite") dir_name = os.path.dirname(tflite_path) if dir_name: os.makedirs(dir_name, exist_ok=True) with open(tflite_path, 'wb') as f: f.write(tflite_model) return tflite_path except ConverterError: return None
def _run_test_case(self, input_names_with_port, output_names_with_port): try: tf.compat.v1.disable_eager_execution() except: # pylint: disable=bare-except pass graph_def = None with tf_session() as sess: # freeze graph origin_graph = sess.graph variables_lib.global_variables_initializer().run() output_name_without_port = [ n.split(':')[0] for n in output_names_with_port ] graph_def = tf.graph_util.convert_variables_to_constants( sess, sess.graph_def, output_name_without_port) tf_reset_default_graph() tf.import_graph_def(graph_def, name='') # optimize graph input_tensors = { i: sess.graph.get_tensor_by_name(i) for i in input_names_with_port } output_tensors = { i: sess.graph.get_tensor_by_name(i) for i in output_names_with_port } graph_def = tf_optimize(input_tensors, output_tensors, sess.graph_def, True) with tf_session() as sess: if self.config.is_debug_mode: if not os.path.exists(self.test_data_directory): os.makedirs(self.test_data_directory) model_path = os.path.join( self.test_data_directory, self._testMethodName + "_after_tf_optimize.pb") utils.save_protobuf(model_path, graph_def) self.logger.debug("created file %s", model_path) tf_reset_default_graph() tf.import_graph_def(graph_def, name='') with tf_session() as sess: inferred_graph = infer_shape_for_graph(sess.graph) # compare each operation for op in origin_graph.get_operations(): inferred_op = None try: inferred_op = inferred_graph.get_operation_by_name(op.name) except KeyError: continue self._compare_shape_for_op(op, inferred_op)
def run_test_case(self, func, feed_dict, input_names_with_port, output_names_with_port, rtol=1e-07, atol=1e-5, convert_var_to_const=True, constant_fold=True, check_value=True, check_shape=True, check_dtype=True, process_args=None, onnx_feed_dict=None, graph_validator=None, as_session=False, large_model=False): # optional - passed to process_tf_graph if process_args is None: process_args = {} # optional - pass distinct feed_dict to onnx runtime if onnx_feed_dict is None: onnx_feed_dict = feed_dict input_names_with_port = list(feed_dict) tf_reset_default_graph() graph_def = None np.random.seed(1) # Make it reproducible. clean_feed_dict = {utils.node_name(k): v for k, v in feed_dict.items()} if is_tf2() and not as_session: # # use eager to execute the tensorflow func # # numpy doesn't work for all ops, make it tf.Tensor() input_tensors = [ tf.TensorSpec(shape=v.shape, dtype=tf.as_dtype(v.dtype), name=utils.node_name(k)) for k, v in feed_dict.items() ] input_list = [ tf.convert_to_tensor(v, dtype=tf.as_dtype(v.dtype), name=utils.node_name(k)) for k, v in feed_dict.items() ] tf.random.set_seed(1) expected = func(*input_list) if isinstance(expected, (list, tuple)): # list or tuple expected = [x.numpy() for x in expected] else: # single result expected = [expected.numpy()] # now make the eager functions a graph concrete_func = tf.function(func, input_signature=tuple(input_tensors)) concrete_func = concrete_func.get_concrete_function() graph_def = from_function(concrete_func, input_names=list(feed_dict.keys()), output_names=output_names_with_port, large_model=large_model) else: # # use graph to execute the tensorflow func # with tf_session() as sess: tf_set_random_seed(1) input_list = [] for k, v in clean_feed_dict.items(): input_list.append( tf_placeholder(name=k, shape=v.shape, dtype=tf.as_dtype(v.dtype))) func(*input_list) variables_lib.global_variables_initializer().run() tf_tables_initializer().run() output_dict = [] for out_name in output_names_with_port: output_dict.append(sess.graph.get_tensor_by_name(out_name)) expected = sess.run(output_dict, feed_dict=feed_dict) graph_def = freeze_session(sess, input_names=list(feed_dict.keys()), output_names=output_names_with_port) tf_reset_default_graph() with tf_session() as sess: tf.import_graph_def(graph_def, name='') graph_def = tf_optimize(list(feed_dict.keys()), output_names_with_port, graph_def, fold_constant=constant_fold) tf_reset_default_graph() with tf_session() as sess: const_node_values = None if large_model: const_node_values = compress_graph_def(graph_def) tf.import_graph_def(graph_def, name='') if self.config.is_debug_mode: model_path = os.path.join( self.test_data_directory, self._testMethodName + "_after_tf_optimize.pb") utils.save_protobuf(model_path, graph_def) self.logger.debug("created file %s", model_path) g = process_tf_graph(sess.graph, opset=self.config.opset, input_names=list(feed_dict.keys()), output_names=output_names_with_port, target=self.config.target, const_node_values=const_node_values, **process_args) g = optimizer.optimize_graph(g) actual = self.run_backend(g, output_names_with_port, onnx_feed_dict, large_model) for expected_val, actual_val in zip(expected, actual): if check_value: self.assertAllClose(expected_val, actual_val, rtol=rtol, atol=atol) if check_dtype: self.assertEqual(expected_val.dtype, actual_val.dtype) # why need shape checke: issue when compare [] with scalar # https://github.com/numpy/numpy/issues/11071 if check_shape: self.assertEqual(expected_val.shape, actual_val.shape) if graph_validator: self.assertTrue(graph_validator(g)) return g
def run_test(self, name, backend="caffe2", onnx_file=None, opset=None, extra_opset=None, perf=None, fold_const=None): """Run complete test against backend.""" self.perf = perf # get the model if self.url: _, dir_name = self.download_model() logger.info("Downloaded to %s", dir_name) model_path = os.path.join( dir_name, self.local) if self.local != "." else dir_name else: model_path = self.local logger.info("Load model from %s", model_path) input_names = list(self.input_names.keys()) outputs = self.output_names if self.model_type in ["checkpoint"]: graph_def, input_names, outputs = tf_loader.from_checkpoint( model_path, input_names, outputs) elif self.model_type in ["saved_model"]: loaded = tf_loader.from_saved_model( model_path, input_names, outputs, self.tag, self.signatures, self.concrete_function, self.large_model, return_concrete_func=self.large_model) if self.large_model: # Must maintain ref to imported since concrete_func uses weak refs # pylint: disable=unused-variable graph_def, input_names, outputs, concrete_func, imported = loaded else: graph_def, input_names, outputs = loaded elif self.model_type in ["keras"]: graph_def, input_names, outputs = tf_loader.from_keras( model_path, input_names, outputs) else: graph_def, input_names, outputs = tf_loader.from_graphdef( model_path, input_names, outputs) if utils.is_debug_mode(): utils.save_protobuf( os.path.join(TEMP_DIR, name + "_after_tf_optimize.pb"), graph_def) if self.large_model: inputs = {} for k in input_names: v = self.input_names[k] inputs[k.split(":")[0]] = tf.constant(self.make_input(v)) tf_func = tf.function(concrete_func) logger.info("Running TF") tf_results_d = tf_func(**inputs) if self.structured_outputs is None: tf_results = list(tf_results_d.values()) else: tf_results = [ tf_results_d[output] for output in self.structured_outputs ] if self.perf: logger.info("Running TF perf") start = time.time() for _ in range(PERFITER): _ = concrete_func(**inputs) self.tf_runtime = time.time() - start logger.info("TensorFlow OK") inputs = {} shape_override = {} tf_reset_default_graph() from tf2onnx.tf_utils import compress_graph_def const_node_values = None with tf.Graph().as_default() as tf_graph: if self.large_model: const_node_values = compress_graph_def(graph_def) tf.import_graph_def(graph_def, name='') with tf_session(graph=tf_graph) as sess: # create the input data for k in input_names: v = self.input_names[k] t = sess.graph.get_tensor_by_name(k) expected_dtype = tf.as_dtype(t.dtype).name if isinstance(v, six.text_type) and v.startswith("np."): np_value = eval(v) # pylint: disable=eval-used if expected_dtype != np_value.dtype: logger.warning( "dtype mismatch for input %s: expected=%s, actual=%s", k, expected_dtype, np_value.dtype) inputs[k] = np_value.astype(expected_dtype) else: inputs[k] = self.make_input(v).astype(expected_dtype) if self.force_input_shape: for k, v in inputs.items(): shape_override[k] = list(v.shape) # run the model with tensorflow if self.skip_tensorflow: logger.info("TensorFlow SKIPPED") elif not self.large_model: tf_results = self.run_tensorflow(sess, inputs) logger.info("TensorFlow OK") model_proto = None if self.skip_conversion: if self.large_model: external_tensor_storage = ExternalTensorStorage() model_proto = utils.model_proto_from_zip( self.converted_model, external_tensor_storage) else: external_tensor_storage = None model_proto = utils.model_proto_from_file(self.converted_model) logger.info("ONNX loaded from file") else: try: # convert model to onnx onnx_graph = self.to_onnx(sess.graph, opset=opset, extra_opset=extra_opset, shape_override=shape_override, input_names=inputs.keys(), const_node_values=const_node_values) onnx_graph = optimizer.optimize_graph(onnx_graph) print("ONNX", onnx_graph.dump_node_statistics()) external_tensor_storage = ExternalTensorStorage( ) if self.large_model else None model_proto = onnx_graph.make_model( "converted from tf2onnx", external_tensor_storage=external_tensor_storage) logger.info("To_ONNX, OK") if onnx_file: self.create_onnx_file(name, model_proto, inputs, onnx_file, external_tensor_storage) if self.converted_model: if self.large_model: utils.save_onnx_zip(self.converted_model, model_proto, external_tensor_storage) else: utils.save_protobuf(self.converted_model, model_proto) logger.info("Created %s", self.converted_model) except Exception: logger.error("To_ONNX FAIL", exc_info=1) return False try: onnx_results = None if backend == "caffe2": onnx_results = self.run_caffe2(name, model_proto, inputs) elif backend == "onnxruntime": onnx_results = self.run_onnxruntime(name, model_proto, inputs, external_tensor_storage) else: raise ValueError("unknown backend") logger.info("Run_ONNX OK") try: if self.skip_tensorflow: logger.info("Results: skipped tensorflow") else: if self.check_only_shape: for tf_res, onnx_res in zip(tf_results, onnx_results): np.testing.assert_array_equal( tf_res.shape, onnx_res.shape) else: for tf_res, onnx_res in zip(tf_results, onnx_results): np.testing.assert_allclose(tf_res, onnx_res, rtol=self.rtol, atol=self.atol) logger.info("Results: OK") return True except Exception: logger.error("Results", exc_info=1) except Exception: logger.error("Run_ONNX FAIL", exc_info=1) return False
def run_test_case(self, func, feed_dict, input_names_with_port, output_names_with_port, rtol=1e-07, atol=1e-5, convert_var_to_const=True, constant_fold=True, check_value=True, check_shape=True, check_dtype=True, process_args=None, onnx_feed_dict=None, graph_validator=None, as_session=False, large_model=False, premade_placeholders=False): test_tf = not self.config.skip_tf_tests test_tflite = not self.config.skip_tflite_tests run_tfl_consistency_test = test_tf and test_tflite and self.config.run_tfl_consistency_test # optional - passed to process_tf_graph if process_args is None: process_args = {} # optional - pass distinct feed_dict to onnx runtime if onnx_feed_dict is None: onnx_feed_dict = feed_dict input_names_with_port = list(feed_dict) tf_reset_default_graph() if tf_lite is None: test_tflite = False g = None expected, graph_def, initialized_tables = \ self.freeze_and_run_tf(func, feed_dict, output_names_with_port, as_session, premade_placeholders, large_model, constant_fold) if test_tflite: tflite_path = self.convert_to_tflite(graph_def, feed_dict, output_names_with_port) test_tflite = tflite_path is not None if test_tf: tf_reset_default_graph() with tf_session() as sess: const_node_values = None if large_model: const_node_values = compress_graph_def(graph_def) tf.import_graph_def(graph_def, name='') g = process_tf_graph(sess.graph, opset=self.config.opset, input_names=list(feed_dict.keys()), output_names=output_names_with_port, target=self.config.target, const_node_values=const_node_values, initialized_tables=initialized_tables, **process_args) g = optimizer.optimize_graph(g, catch_errors=False) actual = self.run_backend(g, output_names_with_port, onnx_feed_dict, large_model) self.assert_results_equal(expected, actual, rtol, atol, check_value, check_shape, check_dtype) self.assert_shapes_correct(g, self.config.allow_missing_shapes, not self.config.skip_onnx_checker) if graph_validator: self.assertTrue(graph_validator(g)) if test_tflite: tfl_results, tfl_outputs = self.run_tflite(tflite_path, feed_dict) test_tflite = tfl_results is not None if test_tflite: if run_tfl_consistency_test: self.assert_results_equal(expected, tfl_results, rtol, atol, check_value, check_shape, check_dtype) tfl_process_args = process_args.copy() if 'inputs_as_nchw' in tfl_process_args: nchw_inps_with_port = tfl_process_args['inputs_as_nchw'] tfl_process_args['inputs_as_nchw'] = [ i.split(':')[0] for i in nchw_inps_with_port ] input_names_without_port = [ inp.split(':')[0] for inp in feed_dict.keys() ] g = process_tf_graph(None, opset=self.config.opset, input_names=input_names_without_port, output_names=tfl_outputs, target=self.config.target, tflite_path=tflite_path, **tfl_process_args) g = optimizer.optimize_graph(g) onnx_feed_dict_without_port = { k.split(':')[0]: v for k, v in onnx_feed_dict.items() } onnx_from_tfl_res = self.run_backend(g, tfl_outputs, onnx_feed_dict_without_port, postfix="_from_tflite") self.assert_results_equal(tfl_results, onnx_from_tfl_res, rtol, atol, check_value, check_shape, check_dtype) self.assert_shapes_correct(g, self.config.allow_missing_shapes, not self.config.skip_onnx_checker) if graph_validator: self.assertTrue(graph_validator(g)) if g is None: raise unittest.SkipTest("Both tf and tflite marked to skip") return g
def freeze_and_run_tf(self, func, feed_dict, outputs, as_session, premade_placeholders, large_model, constant_fold): np.random.seed(1) # Make it reproducible. clean_feed_dict = {utils.node_name(k): v for k, v in feed_dict.items()} if is_tf2() and not as_session: # # use eager to execute the tensorflow func # # numpy doesn't work for all ops, make it tf.Tensor() input_tensors = [ tf.TensorSpec(shape=v.shape, dtype=tf.as_dtype(v.dtype), name=utils.node_name(k)) for k, v in feed_dict.items() ] input_list = [ tf.convert_to_tensor(v, dtype=tf.as_dtype(v.dtype), name=utils.node_name(k)) for k, v in feed_dict.items() ] tf.random.set_seed(1) result = func(*input_list) if isinstance(result, (list, tuple)): # list or tuple result = [x.numpy() for x in result] else: # single result result = [result.numpy()] # now make the eager functions a graph concrete_func = tf.function(func, input_signature=tuple(input_tensors)) concrete_func = concrete_func.get_concrete_function() graph_def = from_function(concrete_func, input_names=list(feed_dict.keys()), output_names=outputs, large_model=large_model) initialized_tables = None else: # # use graph to execute the tensorflow func # with tf_session() as sess: tf_set_random_seed(1) input_list = [] if not premade_placeholders: for k, v in clean_feed_dict.items(): input_list.append( tf_placeholder(name=k, shape=v.shape, dtype=tf.as_dtype(v.dtype))) func(*input_list) variables_lib.global_variables_initializer().run() tf_tables_initializer().run() output_dict = [] for out_name in outputs: output_dict.append(sess.graph.get_tensor_by_name(out_name)) result = sess.run(output_dict, feed_dict=feed_dict) graph_def = freeze_session(sess, input_names=list(feed_dict.keys()), output_names=outputs) table_names, key_dtypes, value_dtypes = get_hash_table_info( graph_def) initialized_tables = {} for n, k_dtype, val_dtype in zip(table_names, key_dtypes, value_dtypes): h = lookup_ops.hash_table_v2(k_dtype, val_dtype, shared_name=n) k, v = lookup_ops.lookup_table_export_v2( h, k_dtype, val_dtype) initialized_tables[n] = (sess.run(k), sess.run(v)) tf_reset_default_graph() with tf_session() as sess: tf.import_graph_def(graph_def, name='') graph_def = tf_optimize(list(feed_dict.keys()), outputs, graph_def, fold_constant=constant_fold) model_path = os.path.join( self.test_data_directory, self._testMethodName + "_after_tf_optimize.pb") utils.save_protobuf(model_path, graph_def) self.logger.debug("created file %s", model_path) return result, graph_def, initialized_tables
def run_test(self, name, backend="onnxruntime", onnx_file=None, opset=None, extra_opset=None, perf=None): """Run complete test against backend.""" self.perf = perf # get the model if self.url: _, dir_name = self.download_model() logger.info("Downloaded to %s", dir_name) model_path = os.path.join( dir_name, self.local) if self.local != "." else dir_name else: model_path = self.local logger.info("Load model from %s", model_path) input_names = list(self.input_names.keys()) initialized_tables = {} outputs = self.output_names tflite_path = None to_rename = None if self.model_type in ["checkpoint"]: graph_def, input_names, outputs = tf_loader.from_checkpoint( model_path, input_names, outputs) elif self.model_type in ["saved_model"]: loaded = tf_loader.from_saved_model( model_path, None, None, self.tag, self.signatures, self.concrete_function, self.large_model, return_concrete_func=not self.run_tf_frozen, return_initialized_tables=True, return_tensors_to_rename=True) if not self.run_tf_frozen: # Must maintain ref to imported since concrete_func uses weak refs # pylint: disable=unused-variable graph_def, input_names, outputs, concrete_func, imported, initialized_tables, to_rename = loaded else: graph_def, input_names, outputs, initialized_tables, to_rename = loaded elif self.model_type in ["keras"]: graph_def, input_names, outputs = tf_loader.from_keras( model_path, input_names, outputs) elif self.model_type in ["tflite"]: tflite_path = model_path graph_def = None else: graph_def, input_names, outputs = tf_loader.from_graphdef( model_path, input_names, outputs) if utils.is_debug_mode(): utils.save_protobuf( os.path.join(TEMP_DIR, name + "_after_tf_optimize.pb"), graph_def) if tflite_path is not None: inputs = {} for k in input_names: v = self.input_names[k] inputs[k] = self.make_input(v) interpreter = tf.lite.Interpreter(tflite_path) input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() input_name_to_index = { n['name'].split(':')[0]: n['index'] for n in input_details } for k, v in inputs.items(): interpreter.resize_tensor_input(input_name_to_index[k], v.shape) interpreter.allocate_tensors() def run_tflite(): for k, v in inputs.items(): interpreter.set_tensor(input_name_to_index[k], v) interpreter.invoke() result = [ interpreter.get_tensor(output['index']) for output in output_details ] return result tf_results = run_tflite() if self.perf: logger.info("Running TFLite perf") n = 0 start = time.time() stop = start + PERF_TIME while time.time() < stop: for _ in range(PERF_STEP): _ = run_tflite() n += PERF_STEP self.tf_runtime = 1000 * (time.time() - start) / n logger.info("TFLite perf {:.2f}ms/inference, n={}".format( self.tf_runtime, n)) logger.info("TFLite OK") if not self.run_tf_frozen: inputs = {} for k in input_names: v = self.input_names[k] inputs[k.split(":")[0]] = tf.constant(self.make_input(v)) tf_func = tf.function(concrete_func) logger.info("Running TF") tf_results_d = tf_func(**inputs) # If there is only a single output a dict might not be returned if isinstance(tf_results_d, tf.Tensor): tf_results = [tf_results_d] else: tf_results = [ tf_results_d[k] for k in sorted(tf_results_d.keys()) ] tf_results = [tf_res.numpy() for tf_res in tf_results] if self.perf: logger.info("Running TF perf") n = 0 start = time.time() stop = start + PERF_TIME if self.tf_profile is not None: tf.profiler.experimental.start(self.tf_profile) while time.time() < stop: for _ in range(PERF_STEP): _ = concrete_func(**inputs) n += PERF_STEP if self.tf_profile is not None: tf.profiler.experimental.stop() self.tf_runtime = 1000 * (time.time() - start) / n logger.info("TF perf {:.2f}ms/inference, n={}".format( self.tf_runtime, n)) logger.info("TensorFlow OK") shape_override = {} const_node_values = None tf_graph = None if graph_def is not None: inputs = {} tf_reset_default_graph() with tf.Graph().as_default() as tf_graph: from tf2onnx.tf_utils import compress_graph_def if self.large_model: const_node_values = compress_graph_def(graph_def) tf.import_graph_def(graph_def, name='') with tf_session(graph=tf_graph) as sess: # create the input data for k in input_names: v = self.input_names[k] t = sess.graph.get_tensor_by_name(k) expected_dtype = tf.as_dtype(t.dtype).name if isinstance(v, six.text_type) and v.startswith("np."): np_value = eval(v) # pylint: disable=eval-used if expected_dtype != np_value.dtype: logger.warning( "dtype mismatch for input %s: expected=%s, actual=%s", k, expected_dtype, np_value.dtype) inputs[k] = np_value.astype(expected_dtype) else: if expected_dtype == "string": inputs[k] = self.make_input(v).astype( np.str).astype(np.object) else: inputs[k] = self.make_input(v).astype( expected_dtype) if self.force_input_shape: for k, v in inputs.items(): shape_override[k] = list(v.shape) # run the model with tensorflow if self.skip_tensorflow: logger.info("TensorFlow SKIPPED") elif self.run_tf_frozen: if self.tf_profile is not None: tf.profiler.experimental.start(self.tf_profile) tf_results = self.run_tensorflow(sess, inputs) if self.tf_profile is not None: tf.profiler.experimental.stop() logger.info("TensorFlow OK") tf_graph = sess.graph model_proto = None if self.skip_conversion: if self.large_model: external_tensor_storage = ExternalTensorStorage() model_proto = utils.model_proto_from_zip( self.converted_model, external_tensor_storage) else: external_tensor_storage = None model_proto = utils.model_proto_from_file(self.converted_model) logger.info("ONNX loaded from file") else: try: # convert model to onnx onnx_graph = self.to_onnx( tf_graph, opset=opset, extra_opset=extra_opset, shape_override=shape_override, input_names=inputs.keys(), const_node_values=const_node_values, initialized_tables=initialized_tables, tflite_path=tflite_path, tensors_to_rename=to_rename) onnx_graph = optimizer.optimize_graph(onnx_graph) print("ONNX", onnx_graph.dump_node_statistics()) external_tensor_storage = ExternalTensorStorage( ) if self.large_model else None model_proto = onnx_graph.make_model( "converted from tf2onnx", external_tensor_storage=external_tensor_storage) logger.info("To_ONNX, OK") if onnx_file: self.create_onnx_file(name, model_proto, inputs, onnx_file, external_tensor_storage) if self.converted_model: if self.large_model: utils.save_onnx_zip(self.converted_model, model_proto, external_tensor_storage) else: utils.save_protobuf(self.converted_model, model_proto) logger.info("Created %s", self.converted_model) except Exception: logger.error("To_ONNX FAIL", exc_info=1) return False try: onnx_results = None if backend == "onnxruntime": if to_rename is None: struc_outputs = self.output_names else: struc_outputs = [ to_rename.get(k, k) for k in self.output_names ] onnx_results = self.run_onnxruntime(name, model_proto, inputs, struc_outputs, external_tensor_storage) else: raise ValueError("unknown backend") logger.info("Run_ONNX OK") try: if self.skip_tensorflow: logger.info("Results: skipped tensorflow") else: if self.check_only_shape: for tf_res, onnx_res in zip(tf_results, onnx_results): np.testing.assert_array_equal( tf_res.shape, onnx_res.shape) else: for tf_res, onnx_res in zip(tf_results, onnx_results): good_cnt = np.count_nonzero( np.isclose(tf_res, onnx_res, rtol=self.rtol, atol=self.atol)) bad_cnt = tf_res.size - good_cnt if bad_cnt > self.ptol / 100 * tf_res.size: # Prints a nice error message with stats np.testing.assert_allclose(tf_res, onnx_res, rtol=self.rtol, atol=self.atol) logger.info("Results: OK") return True except Exception: logger.error("Results", exc_info=1) except Exception: logger.error("Run_ONNX FAIL", exc_info=1) return False
def run_test(self, name, backend="caffe2", onnx_file=None, opset=None, extra_opset=None, perf=None, fold_const=None): """Run complete test against backend.""" self.perf = perf # get the model if self.url: _, dir_name = self.download_model() logger.info("Downloaded to %s", dir_name) model_path = os.path.join(dir_name, self.local) else: model_path = self.local logger.info("Load model from %s", model_path) input_names = list(self.input_names.keys()) outputs = self.output_names if self.model_type in ["checkpoint"]: graph_def, input_names, outputs = tf_loader.from_checkpoint( model_path, input_names, outputs) elif self.model_type in ["saved_model"]: graph_def, input_names, outputs = tf_loader.from_saved_model( model_path, input_names, outputs) elif self.model_type in ["keras"]: graph_def, input_names, outputs = tf_loader.from_keras( model_path, input_names, outputs) else: graph_def, input_names, outputs = tf_loader.from_graphdef( model_path, input_names, outputs) if utils.is_debug_mode(): utils.save_protobuf( os.path.join(TEMP_DIR, name + "_after_tf_optimize.pb"), graph_def) inputs = {} shape_override = {} tf_reset_default_graph() g = tf.import_graph_def(graph_def, name='') # with tf_session(config=tf.ConfigProto(allow_soft_placement=True), graph=g) as sess: with tf_session(graph=g) as sess: # create the input data for k in input_names: v = self.input_names[k] t = sess.graph.get_tensor_by_name(k) expected_dtype = tf.as_dtype(t.dtype).name if isinstance(v, six.text_type) and v.startswith("np."): np_value = eval(v) # pylint: disable=eval-used if expected_dtype != np_value.dtype: logger.warning( "dtype mismatch for input %s: expected=%s, actual=%s", k, expected_dtype, np_value.dtype) inputs[k] = np_value.astype(expected_dtype) else: inputs[k] = self.make_input(v).astype(expected_dtype) if self.force_input_shape: for k, v in inputs.items(): shape_override[k] = list(v.shape) # run the model with tensorflow if self.skip_tensorflow: logger.info("TensorFlow SKIPPED") else: tf_results = self.run_tensorflow(sess, inputs) logger.info("TensorFlow OK") model_proto = None try: # convert model to onnx onnx_graph = self.to_onnx(sess.graph, opset=opset, extra_opset=extra_opset, shape_override=shape_override, input_names=inputs.keys()) onnx_graph = optimizer.optimize_graph(onnx_graph) model_proto = onnx_graph.make_model("converted from tf2onnx") logger.info("To_ONNX, OK") if onnx_file: self.create_onnx_file(name, model_proto, inputs, onnx_file) except Exception: logger.error("To_ONNX FAIL", exc_info=1) return False try: onnx_results = None if backend == "caffe2": onnx_results = self.run_caffe2(name, model_proto, inputs) elif backend == "onnxruntime": onnx_results = self.run_onnxruntime(name, model_proto, inputs) else: raise ValueError("unknown backend") logger.info("Run_ONNX OK") try: if self.skip_tensorflow: logger.info("Results: skipped tensorflow") else: if self.check_only_shape: for tf_res, onnx_res in zip(tf_results, onnx_results): np.testing.assert_array_equal( tf_res.shape, onnx_res.shape) else: for tf_res, onnx_res in zip(tf_results, onnx_results): np.testing.assert_allclose(tf_res, onnx_res, rtol=self.rtol, atol=self.atol) logger.info("Results: OK") return True except Exception: logger.error("Results", exc_info=1) except Exception: logger.error("Run_ONNX FAIL", exc_info=1) return False
def _run_pb_gen(): '''Load Model from model.py file''' inference_model = model.Model(is_training=False, seq_length=FLAGS.seq_length, batch_size=FLAGS.batch_size, img_height=FLAGS.img_height, img_width=FLAGS.img_width) with tf.compat.v1.Session() as sess: '''Initialize Variables in Model''' init_op = tf.compat.v1.global_variables_initializer() '''Start Session''' sess.run(init_op) '''Get Graph Def''' graph_def = sess.graph.as_graph_def() '''Extract Inputs''' inputs = [] for op in sess.graph.get_operations(): if op.type == "Placeholder": inputs.append(op.name) '''Extract Outputs''' name_list = [] exclsv_list = [] for node in graph_def.node: name_list.append(node.name) exclsv_list.extend(node.input) outputs = list(set(name_list) - set(exclsv_list)) outputs = ['depth_prediction/depth_prediction/truediv'] '''Fix Nodes''' '''See: https://github.com/onnx/tensorflow-onnx/issues/77''' for node in graph_def.node: if node.op == 'RefSwitch': node.op = 'Switch' for index in range(len(node.input)): if 'moving_' in node.input[index]: node.input[index] = node.input[index] + '/read' elif node.op == 'AssignSub': node.op = 'Sub' if 'use_locking' in node.attr: del node.attr['use_locking'] elif node.op == 'AssignAdd': node.op = 'Add' if 'use_locking' in node.attr: del node.attr['use_locking'] elif node.op == 'Assign': node.op = 'Identity' if 'use_locking' in node.attr: del node.attr['use_locking'] if 'validate_shape' in node.attr: del node.attr['validate_shape'] if len(node.input) == 2: # input0: ref: Should be from a Variable node. May be uninitialized. # input1: value: The value to be assigned to the variable. node.input[0] = node.input[1] del node.input[1] elif node.op == 'L2Loss': node.op = 'Abs' '''Sub Graph Extraction''' needed_names = [tf2onnx.utils.node_name(i) for i in inputs ] + [tf2onnx.utils.node_name(i) for i in outputs] sub_graph = tf.compat.v1.graph_util.extract_sub_graph( graph_def, needed_names) '''Freezing Graph (Necessary before Making ONNX Graph)''' frozen_graph = freeze_session(sess, sub_graph, output_names=outputs) frozen_graph = tf.graph_util.remove_training_nodes(frozen_graph) with open("frozen.pb", "wb") as f: f.write(frozen_graph.SerializeToString()) '''Graph_Def to Graph Conversion''' tf_reset_default_graph() graph = tf.import_graph_def(frozen_graph, name='') with tf_session(graph=graph) as sess: '''Extract Inputs''' inputs = [] for op in sess.graph.get_operations(): if op.type == "Placeholder": inputs.append(op.name + ':0') '''Extract Outputs''' outputs = [output + ":0" for output in outputs] print("jrp", outputs) '''ONNX Graph Generation''' onnx_graph = tf2onnx.tfonnx.process_tf_graph(sess.graph, input_names=inputs, output_names=outputs) '''Optimizing Grapph for ONNX Formation''' #opt_graph = tf2onnx.optimizer.optimize_graph(onnx_graph) '''Make ProtoBuff Model''' model_proto = onnx_graph.make_model(str(FLAGS.output_path)) #onnx.checker.check_model(model_proto) '''Store ProtoBuff-file''' tf2onnx.utils.save_onnx_model("./", "saved_model", feed_dict={}, model_proto=model_proto) print('TF-Graph converted to SavedModel!')
def run_test_case(self, func, feed_dict, input_names_with_port, output_names_with_port, rtol=1e-07, atol=1e-5, mtol=None, convert_var_to_const=True, constant_fold=True, check_value=True, check_shape=True, check_dtype=True, process_args=None, onnx_feed_dict=None, graph_validator=None, as_session=False, large_model=False, premade_placeholders=False, use_custom_ops=False, optimize=True): """ This function tests all scenarios available through the command line. The command line always runs the optimizers. However, they may modify the final graph into something different than the tested converter implements. Set `optimize=False` to keep the original set of nodes and helps debugging. However, the same function should be called with `optimize=True` to test what the user would actually get. """ test_tf = not self.config.skip_tf_tests test_tflite = not self.config.skip_tflite_tests test_tfjs = not self.config.skip_tfjs_tests run_tfl_consistency_test = test_tf and test_tflite and self.config.run_tfl_consistency_test # optional - passed to process_tf_graph if process_args is None: process_args = {} # optional - pass distinct feed_dict to onnx runtime if onnx_feed_dict is None: onnx_feed_dict = feed_dict input_names_with_port = list(feed_dict) tf_reset_default_graph() if tf_lite is None: test_tflite = False g = None expected, graph_def, initialized_tables = \ self.freeze_and_run_tf(func, feed_dict, output_names_with_port, as_session, premade_placeholders, large_model, constant_fold) graph_def_path = os.path.join( self.test_data_directory, self._testMethodName + "_after_tf_optimize.pb") utils.save_protobuf(graph_def_path, graph_def) self.logger.debug("created file %s", graph_def_path) if test_tfjs: tfjs_path = self.convert_to_tfjs(graph_def_path, output_names_with_port) if tfjs_path is None: test_tfjs = False if test_tflite: tflite_path = self.convert_to_tflite(graph_def, feed_dict, output_names_with_port) test_tflite = tflite_path is not None and self.tflite_has_supported_types( tflite_path) if test_tf: tf_reset_default_graph() with tf_session() as sess: const_node_values = None if large_model: const_node_values = compress_graph_def(graph_def) tf.import_graph_def(graph_def, name='') g = process_tf_graph(sess.graph, opset=self.config.opset, input_names=list(feed_dict.keys()), output_names=output_names_with_port, target=self.config.target, const_node_values=const_node_values, initialized_tables=initialized_tables, **process_args) if optimize: g = optimizer.optimize_graph(g, catch_errors=False) actual = self.run_backend(g, output_names_with_port, onnx_feed_dict, large_model, use_custom_ops=use_custom_ops) self.assert_results_equal(expected, actual, rtol, atol, mtol, check_value, check_shape, check_dtype) self.assert_shapes_correct(g, self.config.allow_missing_shapes, not self.config.skip_onnx_checker) if graph_validator: self.assertTrue(graph_validator(g)) if test_tflite: tfl_res, tfl_outputs = self.run_tflite(tflite_path, feed_dict) test_tflite = tfl_res is not None if test_tflite: if run_tfl_consistency_test: self.assert_results_equal(expected, tfl_res, rtol, atol, mtol, check_value, check_shape, check_dtype) tfl_process_args = process_args.copy() if 'inputs_as_nchw' in tfl_process_args: nchw_inps_with_port = tfl_process_args['inputs_as_nchw'] tfl_process_args['inputs_as_nchw'] = [ i.split(':')[0] for i in nchw_inps_with_port ] input_names_without_port = [ inp.split(':')[0] for inp in feed_dict.keys() ] g = process_tf_graph(None, opset=self.config.opset, input_names=input_names_without_port, output_names=tfl_outputs, target=self.config.target, tflite_path=tflite_path, **tfl_process_args) if optimize: g = optimizer.optimize_graph(g) onnx_feed_dict_without_port = { k.split(':')[0]: v for k, v in onnx_feed_dict.items() } onnx_tfl_res = self.run_backend(g, tfl_outputs, onnx_feed_dict_without_port, postfix="_from_tflite", use_custom_ops=use_custom_ops) self.assert_results_equal(tfl_res, onnx_tfl_res, rtol, atol, mtol, check_value, check_shape, check_dtype) self.assert_shapes_correct(g, self.config.allow_missing_shapes, not self.config.skip_onnx_checker) if graph_validator: self.assertTrue(graph_validator(g)) if test_tfjs: try: tfjs_res = run_tfjs(tfjs_path, feed_dict) except RuntimeError as e: ignored_errors = [ "is not yet supported", "Operands could not be broadcast together", "unknown dtype null", "must be [NaN", "Cannot read property 'name' of undefined", "Either strides or dilations must be 1", "does not support" ] if any(err in str(e) for err in ignored_errors): test_tfjs = False else: raise e if test_tfjs: g = process_tf_graph(None, opset=self.config.opset, input_names=list(feed_dict.keys()), output_names=None, target=self.config.target, tfjs_path=tfjs_path, **process_args) g = optimizer.optimize_graph(g) onnx_tfjs_res = self.run_backend(g, None, onnx_feed_dict, large_model, postfix="_from_tfjs", use_custom_ops=use_custom_ops) self.assert_results_equal(tfjs_res, onnx_tfjs_res, rtol, atol, mtol, check_value, check_shape, check_dtype=False) self.assert_shapes_correct(g, self.config.allow_missing_shapes, not self.config.skip_onnx_checker) if graph_validator: self.assertTrue(graph_validator(g)) if g is None: raise unittest.SkipTest("tf, tflite, and tfjs marked to skip") return g
def freeze_and_run_tf(self, func, feed_dict, outputs, as_session, premade_placeholders, large_model): np.random.seed(1) # Make it reproducible. clean_feed_dict = {utils.node_name(k): v for k, v in feed_dict.items()} if is_tf2() and not as_session: # # use eager to execute the tensorflow func # # numpy doesn't work for all ops, make it tf.Tensor() input_tensors = [ tf.TensorSpec(shape=v.shape, dtype=tf.as_dtype(v.dtype), name=utils.node_name(k)) for k, v in feed_dict.items() ] input_list = [ tf.convert_to_tensor(v, dtype=tf.as_dtype(v.dtype), name=utils.node_name(k)) for k, v in feed_dict.items() ] tf.random.set_seed(1) result = func(*input_list) if isinstance(result, (list, tuple)): # list or tuple result = [x.numpy() for x in result] else: # single result result = [result.numpy()] # now make the eager functions a graph concrete_func = tf.function(func, input_signature=tuple(input_tensors)) concrete_func = concrete_func.get_concrete_function() graph_def = from_function(concrete_func, input_names=list(feed_dict.keys()), output_names=outputs, large_model=large_model) initialized_tables = None else: # # use graph to execute the tensorflow func # with tf_session() as sess: tf_set_random_seed(1) input_list = [] if not premade_placeholders: for k, v in clean_feed_dict.items(): input_list.append( tf_placeholder(name=k, shape=v.shape, dtype=tf.as_dtype(v.dtype))) func(*input_list) variables_lib.global_variables_initializer().run() tf_tables_initializer().run() output_dict = [] for out_name in outputs: output_dict.append(sess.graph.get_tensor_by_name(out_name)) result = sess.run(output_dict, feed_dict=feed_dict) graph_def = freeze_session(sess, input_names=list(feed_dict.keys()), output_names=outputs) table_info = get_hash_table_info(graph_def) initialized_tables = {} for info in table_info: if info.shared_name is None: continue h = lookup_ops.hash_table_v2(info.key_dtype, info.val_dtype, shared_name=info.shared_name) k, v = lookup_ops.lookup_table_export_v2( h, info.key_dtype, info.val_dtype) initialized_tables[info.shared_name] = (sess.run(k), sess.run(v)) tf_reset_default_graph() with tf_session() as sess: tf.import_graph_def(graph_def, name='') graph_def = tf_optimize(list(feed_dict.keys()), outputs, graph_def) return result, graph_def, initialized_tables