def main():
args = get_args()
logging.basicConfig(level=logging.get_verbosity_level(args.verbose))
if args.debug:
utils.set_debug_mode(True)
Test.cache_dir = args.cache
Test.target = args.target
tests = load_tests_from_yaml(args.config)
if args.list:
logger.info(sorted(tests.keys()))
return 0
if args.tests:
test_keys = args.tests.split(",")
else:
test_keys = list(tests.keys())
failed = 0
count = 0
for test in test_keys:
logger.info("===================================")
t = tests[test]
if args.tests is None:
if t.disabled and not args.include_disabled:
logger.info("Skip %s: disabled", test)
continue
condition, reason = t.check_opset_constraints(args.opset, args.extra_opset)
if not condition:
logger.info("Skip %s: %s", test, reason)
continue
count += 1
try:
logger.info("Running %s", test)
ret = t.run_test(test, backend=args.backend, onnx_file=args.onnx_file,
opset=args.opset, extra_opset=args.extra_opset, perf=args.perf,
fold_const=args.fold_const)
except Exception:
logger.error("Failed to run %s", test, exc_info=1)
ret = None
finally:
if not utils.is_debug_mode():
utils.delete_directory(TEMP_DIR)
if not ret:
failed += 1
logger.info("===================================")
logger.info("RESULT: %s failed of %s, backend=%s", failed, count, args.backend)
if args.perf:
with open(args.perf, "w") as f:
f.write("test,tensorflow,onnx\n")
for test in test_keys:
t = tests[test]
if t.perf:
f.write("{},{},{}\n".format(test, t.tf_runtime, t.onnx_runtime))
return failed
def main():
# suppress log info of tensorflow so that result of test can be seen much easier
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.logging.set_verbosity(tf.logging.WARN)
args = get_args()
Test.cache_dir = args.cache
Test.target = args.target
tests = tests_from_yaml(args.config)
if args.list:
print(sorted(tests.keys()))
return 0
if args.tests:
test_keys = args.tests.split(",")
else:
test_keys = list(tests.keys())
failed = 0
count = 0
for test in test_keys:
t = tests[test]
if args.tests is None and t.disabled and not args.include_disabled:
continue
count += 1
try:
ret = t.run_test(test,
backend=args.backend,
debug=args.debug,
onnx_file=args.onnx_file,
opset=args.opset,
extra_opset=args.extra_opset,
perf=args.perf,
fold_const=args.fold_const)
except Exception as ex:
ret = None
print(ex)
finally:
if not args.debug:
utils.delete_directory(TEMP_DIR)
if not ret:
failed += 1
print("=== RESULT: {} failed of {}, backend={}".format(
failed, count, args.backend))
if args.perf:
with open(args.perf, "w") as f:
f.write("test,tensorflow,onnx\n")
for test in test_keys:
t = tests[test]
if t.perf:
f.write("{},{},{}\n".format(test, t.tf_runtime,
t.onnx_runtime))
return failed
def main():
args = get_args()
logging.basicConfig(level=logging.get_verbosity_level(args.verbose))
if args.debug:
utils.set_debug_mode(True)
Test.cache_dir = args.cache
Test.target = args.target
tests = tests_from_yaml(args.config)
if args.list:
print(sorted(tests.keys()))
return 0
if args.tests:
test_keys = args.tests.split(",")
else:
test_keys = list(tests.keys())
failed = 0
count = 0
for test in test_keys:
t = tests[test]
if args.tests is None and t.disabled and not args.include_disabled:
continue
count += 1
try:
ret = t.run_test(test,
backend=args.backend,
onnx_file=args.onnx_file,
opset=args.opset,
extra_opset=args.extra_opset,
perf=args.perf,
fold_const=args.fold_const)
except Exception as ex:
ret = None
tb = traceback.format_exc()
print(ex, tb)
finally:
if not utils.is_debug_mode():
utils.delete_directory(TEMP_DIR)
if not ret:
failed += 1
print("=== RESULT: {} failed of {}, backend={}".format(
failed, count, args.backend))
if args.perf:
with open(args.perf, "w") as f:
f.write("test,tensorflow,onnx\n")
for test in test_keys:
t = tests[test]
if t.perf:
f.write("{},{},{}\n".format(test, t.tf_runtime,
t.onnx_runtime))
return failed
def tearDown(self):
if not self.config.is_debug_mode:
utils.delete_directory(self.test_data_directory)
