def ShouldRunTest(self, run_params):
# Only test static engine mode, with or without calibration.
return (get_linked_tensorrt_version()[0] >= 5
and trt_test.IsQuantizationMode(run_params.precision_mode)
and not run_params.convert_online
and not run_params.dynamic_engine
), "test static engine, offline conversion and INT8"
def testEval(self):
if not is_tensorrt_enabled():
return
model_dir = test.test_src_dir_path(
'python/compiler/tensorrt/test/testdata/mnist')
accuracy_tf_native = self._Run(
is_training=False,
use_trt=False,
batch_size=128,
num_epochs=None,
model_dir=model_dir)['accuracy']
logging.info('accuracy_tf_native: %f', accuracy_tf_native)
self.assertAllClose(0.9662, accuracy_tf_native, rtol=3e-3, atol=3e-3)
if get_linked_tensorrt_version()[0] < 5:
return
accuracy_tf_trt = self._Run(
is_training=False,
use_trt=True,
batch_size=128,
num_epochs=None,
model_dir=model_dir)['accuracy']
logging.info('accuracy_tf_trt: %f', accuracy_tf_trt)
self.assertAllClose(0.9675, accuracy_tf_trt, rtol=1e-3, atol=1e-3)
def ShouldRunTest(self, run_params):
if get_linked_tensorrt_version()[0] < 5:
return False
# Only test static engine mode, with or without calibration.
return (trt_test.IsQuantizationMode(run_params.precision_mode)
and not run_params.convert_online
and not run_params.dynamic_engine)
def testTrtGraphConverter_ShapeOp_v2(self):
"""Test case for TrtGraphConverterV2 with ShapeOp."""
if not is_tensorrt_enabled():
return
# TODO(b/185944425): enable the test for TRT before TRT 7.
ver = get_linked_tensorrt_version()
if ver[0] < 7:
return
class ShapeOpModel(tracking.AutoTrackable):
def __init__(self):
self.v = None
@def_function.function(input_signature=[
tensor_spec.TensorSpec(shape=[None, None],
dtype=dtypes.float32)
])
def run(self, x):
q = x + 1
q_shape = array_ops.shape(q)
return array_ops.identity(q_shape, name="output")
np_input = np.random.random_sample([5, 3]).astype(np.float32)
def _InputFunc():
yield (np_input, )
# Create the SavedModel.
root = ShapeOpModel()
expected_output = root.run(np_input)
input_saved_model_dir = self.mkdtemp()
save.save(root, input_saved_model_dir, signatures=root.run)
# Convert the graph to TF-TRT.
conv_params = trt_convert.TrtConversionParams(minimum_segment_size=2)
converter = trt_convert.TrtGraphConverterV2(
input_saved_model_dir=input_saved_model_dir,
conversion_params=conv_params,
use_dynamic_shape=True)
converter.convert()
# Build the graph with the input generator. This runs the TRTEngineOp native
# segment.
converter.build(_InputFunc)
output_saved_model_dir = self.mkdtemp()
converter.save(output_saved_model_dir)
root_with_trt = load.load(output_saved_model_dir)
converted_signature = root_with_trt.signatures["serving_default"]
# Check that the graph is converted to one TRTEngineOp.
self._CheckTrtOps(converted_signature)
# Run the graph.
output_with_trt = converted_signature(
x=ops.convert_to_tensor(np_input))
# Check the result of the run.
self.assertAllClose(expected_output, list(output_with_trt.values())[0])
def ShouldRunTest(self, run_params):
# There is no CombinedNonMaxSuppression op for GPU at the moment, so
# calibration will fail.
# TODO(laigd): fix this.
if trt_test.IsQuantizationMode(run_params.precision_mode):
return False
# Only run for TRT 5.1 and above.
ver = get_linked_tensorrt_version()
return ver[0] > 5 or (ver[0] == 5 and ver[1] >= 1)
def testTrtGraphConverter_AllowEngineNativeSegmentExecution(self):
if not is_tensorrt_enabled():
return
# This test will not work anymore with TRT >= 8. TensorRT does not
# preallocate anymore the max_workspace_size_bytes, but rather allocates as
# it needs up to this value.
# TODO: update the unittest to make this TRTEngine creation fail with TRT8.
ver = get_linked_tensorrt_version()
if ver[0] >= 8:
return
np_input1, np_input2 = self._RandomInput([4, 1, 1])
# Create a model and save it.
input_saved_model_dir = self.mkdtemp()
root = self._GetModelForV2()
save.save(root, input_saved_model_dir,
{_SAVED_MODEL_SIGNATURE_KEY: root.run})
def _InputFn():
yield np_input1, np_input2
# Run TRT conversion and request an unreasonably large workspace.
converter = self._CreateConverterV2(
input_saved_model_dir, max_workspace_size_bytes=10 << 40)
converter.convert()
os.environ["TF_TRT_ALLOW_ENGINE_NATIVE_SEGMENT_EXECUTION"] = "False"
with self.assertRaisesRegex(
errors.AbortedError,
r"User disallowed engine native segment execution"):
try:
converter.build(input_fn=_InputFn)
finally:
# Always reset the environment variable.
os.environ["TF_TRT_ALLOW_ENGINE_NATIVE_SEGMENT_EXECUTION"] = "True"
converter.build(input_fn=_InputFn)
def IsTensorRTVersionGreaterEqual(major, minor=0, patch=0):
ver = get_linked_tensorrt_version()
return ver[0] > major or (ver[0] == major and ver[1] > minor) or (
ver[0] == major and ver[1] == minor and ver[2] >= patch)
def ShouldRunTest(self, run_params):
# Only run for TRT 6 and above.
ver = get_linked_tensorrt_version()
return ver[0] >= 6 and (not run_params.use_calibration)
def ShouldRunTest(self, run_params):
# Only run for TRT 6 and above.
ver = get_linked_tensorrt_version()
return run_params.is_v2 and ver[0] >= 6 and (
not run_params.use_calibration
), "test v2 >=TRT6 and non-calibration"
def is_linked_tensorrt_version_greater_equal(major, minor=0, patch=0): ver = _pywrap_py_utils.get_linked_tensorrt_version() return _is_tensorrt_version_greater_equal(ver, (major, minor, patch))
