def testEval(self): if not trt_convert.is_tensorrt_enabled(): return model_dir = test.test_src_dir_path('contrib/tensorrt/test/testdata') accuracy_tf_native = self._Run( is_training=False, use_trt=False, batch_size=128, num_epochs=None, model_dir=model_dir)['accuracy'] logging.info('accuracy_tf_native: %f', accuracy_tf_native) self.assertAllClose(accuracy_tf_native, 0.9662) if trt_convert.get_linked_tensorrt_version()[0] < 5: return accuracy_tf_trt = self._Run( is_training=False, use_trt=True, batch_size=128, num_epochs=None, model_dir=model_dir)['accuracy'] logging.info('accuracy_tf_trt: %f', accuracy_tf_trt) self.assertAllClose(accuracy_tf_trt, 0.9677)
def testEval(self): if not trt_convert.is_tensorrt_enabled(): return model_dir = test.test_src_dir_path('contrib/tensorrt/test/testdata') accuracy_tf_native = self._Run( is_training=False, use_trt=False, batch_size=128, num_epochs=None, model_dir=model_dir)['accuracy'] logging.info('accuracy_tf_native: %f', accuracy_tf_native) self.assertAllClose(0.9662, accuracy_tf_native, rtol=1e-3, atol=1e-3) if trt_convert.get_linked_tensorrt_version()[0] < 5: return accuracy_tf_trt = self._Run( is_training=False, use_trt=True, batch_size=128, num_epochs=None, model_dir=model_dir)['accuracy'] logging.info('accuracy_tf_trt: %f', accuracy_tf_trt) self.assertAllClose(0.9675, accuracy_tf_trt, rtol=1e-3, atol=1e-3)
def testCreateInferenceGraph_BasicConversion(self): """Test case for trt_convert.create_inference_graph().""" if not trt_convert.is_tensorrt_enabled(): return # Use GraphDef as input. self._TestCreateInferenceGraph() # Use SavedModel as input. tmp_dir = self.get_temp_dir() input_saved_model_dir = os.path.join(tmp_dir, "in_dir1") output_saved_model_dir = os.path.join(tmp_dir, "out_dir1") self._WriteInputSavedModel(input_saved_model_dir) self._TestCreateInferenceGraph(input_saved_model_dir, output_saved_model_dir)
def testCreateInferenceGraph_BasicConversion(self): """Test case for trt_convert.create_inference_graph().""" if not trt_convert.is_tensorrt_enabled(): return # Use GraphDef as input. self._TestCreateInferenceGraph() # Use SavedModel as input. tmp_dir = self.get_temp_dir() input_saved_model_dir = os.path.join(tmp_dir, "in_dir1") output_saved_model_dir = os.path.join(tmp_dir, "out_dir1") self._WriteInputSavedModel(input_saved_model_dir) self._TestCreateInferenceGraph(input_saved_model_dir, output_saved_model_dir)
def testCreateInferenceGraph_MinimumSegmentSize(self): if not trt_convert.is_tensorrt_enabled(): return output_graph_def = trt_convert.create_inference_graph( self._GetGraphDef(), ["output"], minimum_segment_size=5, is_dynamic_op=False) node_name_to_op = {node.name: node.op for node in output_graph_def.node} self.assertEqual({ "v1/read": "Const", "input": "Placeholder", "add": "Add", "mul": "Mul", "add_1": "Add", "output": "Identity" }, node_name_to_op)
def testCreateInferenceGraph_MinimumSegmentSize(self): if not trt_convert.is_tensorrt_enabled(): return output_graph_def = trt_convert.create_inference_graph( self._GetGraphDef(), ["output"], minimum_segment_size=5, is_dynamic_op=False) node_name_to_op = {node.name: node.op for node in output_graph_def.node} self.assertEqual({ "v1/read": "Const", "input": "Placeholder", "add": "Add", "mul": "Mul", "add_1": "Add", "output": "Identity" }, node_name_to_op)
def testCreateInferenceGraph_DynamicOp(self): if not trt_convert.is_tensorrt_enabled(): return trt_convert.enable_test_value() tmp_dir = self.get_temp_dir() input_saved_model_dir = os.path.join(tmp_dir, "in_dir2") output_saved_model_dir = os.path.join(tmp_dir, "out_dir2") self._WriteInputSavedModel(input_saved_model_dir) output_graph_def = trt_convert.create_inference_graph( None, None, is_dynamic_op=True, maximum_cached_engines=2, input_saved_model_dir=input_saved_model_dir, output_saved_model_dir=output_saved_model_dir, session_config=self._GetConfigProto()) # Test the output GraphDef. with ops.Graph().as_default(): importer.import_graph_def(output_graph_def, name="") with self.test_session(config=self._GetConfigProto()) as sess: # Run with batch size 1, a new engine is created and cached. self._TestRun(sess, 1, True) # Run with batch size 2, a new engine is created and cached. self._TestRun(sess, 2, True) # Run with batch size 3, since the number of cached engines has reached # the max, it should fall back to TF function. self._TestRun(sess, 3, False) # Test the output SavedModel with ops.Graph().as_default(): with self.test_session(config=self._GetConfigProto()) as sess: loader.load(sess, [tag_constants.SERVING], output_saved_model_dir) # Run with batch size 1, a new engine is created and cached. self._TestRun(sess, 1, True) # Run with batch size 2, a new engine is created and cached. self._TestRun(sess, 2, True) # Run with batch size 3, since the number of cached engines has reached # the max, it should fall back to TF function. self._TestRun(sess, 3, False)
def testCreateInferenceGraph_DynamicOp(self): if not trt_convert.is_tensorrt_enabled(): return trt_convert.enable_test_value() tmp_dir = self.get_temp_dir() input_saved_model_dir = os.path.join(tmp_dir, "in_dir2") output_saved_model_dir = os.path.join(tmp_dir, "out_dir2") self._WriteInputSavedModel(input_saved_model_dir) output_graph_def = trt_convert.create_inference_graph( None, None, is_dynamic_op=True, maximum_cached_engines=2, input_saved_model_dir=input_saved_model_dir, output_saved_model_dir=output_saved_model_dir, session_config=self._GetConfigProto()) # Test the output GraphDef. with ops.Graph().as_default(): importer.import_graph_def(output_graph_def, name="") with self.test_session(config=self._GetConfigProto()) as sess: # Run with batch size 1, a new engine is created and cached. self._TestRun(sess, 1, True) # Run with batch size 2, a new engine is created and cached. self._TestRun(sess, 2, True) # Run with batch size 3, since the number of cached engines has reached # the max, it should fall back to TF function. self._TestRun(sess, 3, False) # Test the output SavedModel with ops.Graph().as_default(): with self.test_session(config=self._GetConfigProto()) as sess: loader.load(sess, [tag_constants.SERVING], output_saved_model_dir) # Run with batch size 1, a new engine is created and cached. self._TestRun(sess, 1, True) # Run with batch size 2, a new engine is created and cached. self._TestRun(sess, 2, True) # Run with batch size 3, since the number of cached engines has reached # the max, it should fall back to TF function. self._TestRun(sess, 3, False)
def testCreateInferenceGraph_StaticOp(self): if not trt_convert.is_tensorrt_enabled(): return trt_convert.enable_test_value() tmp_dir = self.get_temp_dir() input_saved_model_dir = os.path.join(tmp_dir, "in_dir3") output_saved_model_dir = os.path.join(tmp_dir, "out_dir3") self._WriteInputSavedModel(input_saved_model_dir) output_graph_def = trt_convert.create_inference_graph( None, None, max_batch_size=1, is_dynamic_op=False, maximum_cached_engines=2, # This is noop, added just for testing. input_saved_model_dir=input_saved_model_dir, output_saved_model_dir=output_saved_model_dir, session_config=self._GetConfigProto()) # Test the output GraphDef. with ops.Graph().as_default(): importer.import_graph_def(output_graph_def, name="") with self.test_session(config=self._GetConfigProto()) as sess: # Run with batch size 1, the default engine embedded in the graphdef # will be used. self._TestRun(sess, 1, True) # Run with batch size 2, which exceed the max_batch_size, it should fall # back to TF function. self._TestRun(sess, 2, False) # Test the output SavedModel with ops.Graph().as_default(): with self.test_session(config=self._GetConfigProto()) as sess: loader.load(sess, [tag_constants.SERVING], output_saved_model_dir) # Run with batch size 1, the default engine embedded in the graphdef # will be used. self._TestRun(sess, 1, True) # Run with batch size 2, which exceed the max_batch_size, it should fall # back to TF function. self._TestRun(sess, 2, False)
def testCreateInferenceGraph_StaticOp(self): if not trt_convert.is_tensorrt_enabled(): return trt_convert.enable_test_value() tmp_dir = self.get_temp_dir() input_saved_model_dir = os.path.join(tmp_dir, "in_dir3") output_saved_model_dir = os.path.join(tmp_dir, "out_dir3") self._WriteInputSavedModel(input_saved_model_dir) output_graph_def = trt_convert.create_inference_graph( None, None, max_batch_size=1, is_dynamic_op=False, maximum_cached_engines=2, # This is noop, added just for testing. input_saved_model_dir=input_saved_model_dir, output_saved_model_dir=output_saved_model_dir, session_config=self._GetConfigProto()) # Test the output GraphDef. with ops.Graph().as_default(): importer.import_graph_def(output_graph_def, name="") with self.test_session(config=self._GetConfigProto()) as sess: # Run with batch size 1, the default engine embedded in the graphdef # will be used. self._TestRun(sess, 1, True) # Run with batch size 2, which exceed the max_batch_size, it should fall # back to TF function. self._TestRun(sess, 2, False) # Test the output SavedModel with ops.Graph().as_default(): with self.test_session(config=self._GetConfigProto()) as sess: loader.load(sess, [tag_constants.SERVING], output_saved_model_dir) # Run with batch size 1, the default engine embedded in the graphdef # will be used. self._TestRun(sess, 1, True) # Run with batch size 2, which exceed the max_batch_size, it should fall # back to TF function. self._TestRun(sess, 2, False)
use_optimizer_options = [False, True] dynamic_engine_options = [False, True] for (use_optimizer, precision_mode, dynamic_engine) in itertools.product(use_optimizer_options, PRECISION_MODES, dynamic_engine_options): if IsQuantizationMode(precision_mode): if use_optimizer: # TODO(aaroey): if use_optimizer is True we need to get the inference # graphdef using custom python wrapper class, which is not currently # supported yet. continue if not dynamic_engine: # TODO(aaroey): construction of static calibration engine is not # supported yet. continue conversion = "OptimizerConversion" if use_optimizer else "ToolConversion" engine_type = ("DynamicEngine" if dynamic_engine else "StaticEngine") test_name = "%s_%s_%s" % (conversion, precision_mode, engine_type) run_params = RunParams(use_optimizer=use_optimizer, precision_mode=precision_mode, dynamic_engine=dynamic_engine, test_name=test_name) setattr(test_class, "testTfTrt_" + test_name, _GetTest(run_params)) if trt_convert.is_tensorrt_enabled(): _AddTests(TfTrtIntegrationTestBase)
return _Test use_optimizer_options = [False, True] dynamic_engine_options = [False, True] for (use_optimizer, precision_mode, dynamic_engine) in itertools.product( use_optimizer_options, PRECISION_MODES, dynamic_engine_options): if IsQuantizationMode(precision_mode): if use_optimizer: # TODO(aaroey): if use_optimizer is True we need to get the inference # graphdef using custom python wrapper class, which is not currently # supported yet. continue if not dynamic_engine: # TODO(aaroey): construction of static calibration engine is not # supported yet. continue conversion = "OptimizerConversion" if use_optimizer else "ToolConversion" engine_type = ("DynamicEngine" if dynamic_engine else "StaticEngine") test_name = "%s_%s_%s" % (conversion, precision_mode, engine_type) run_params = RunParams( use_optimizer=use_optimizer, precision_mode=precision_mode, dynamic_engine=dynamic_engine, test_name=test_name) setattr(test_class, "testTfTrt_" + test_name, _GetTest(run_params)) if trt_convert.is_tensorrt_enabled(): _AddTests(TfTrtIntegrationTestBase)