def test_no_reduce_outputs(self):
with tempfile.TemporaryDirectory() as outdir:
run_polygraphy_debug(
[
"reduce",
ONNX_MODELS["reducable"].path,
"--output=reduced.onnx",
"--show-output",
"--no-reduce-outputs",
"--mode=linear",
"--check",
TestReduce.FAKE_REDUCE_CHECKER,
"polygraphy_debug.onnx",
"--fail-node",
"onnx_graphsurgeon_node_3",
],
disable_verbose=True,
cwd=outdir,
)
model = onnx_from_path(os.path.join(outdir, "reduced.onnx"))
assert len(model.graph.node) == 4
assert len(model.graph.input) == 1
assert len(model.graph.output) == 2
assert model.graph.input[0].name == "Y0"
node_names = [node.name for node in model.graph.node]
assert "onnx_graphsurgeon_node_7" in node_names
def test_reduce_custom_fail_message(self, fail_code_arg):
with tempfile.TemporaryDirectory() as outdir:
# fake_reduce_checker will alternate error messages based on whether an arbitrary node is present in the model.
run_polygraphy_debug(
[
"reduce",
ONNX_MODELS["reducable"].path,
"--output=reduced.onnx",
"--show-output",
"--fail-regex",
"REALLY BAD",
"BAD NODE",
] + fail_code_arg + [
"--check",
TestReduce.FAKE_REDUCE_CHECKER,
"polygraphy_debug.onnx",
"--fail-node",
"onnx_graphsurgeon_node_5",
"--fail-return-code=0",
],
disable_verbose=True,
cwd=outdir,
)
model = onnx_from_path(os.path.join(outdir, "reduced.onnx"))
assert len(model.graph.node) == 1
assert model.graph.node[0].name == "onnx_graphsurgeon_node_5"
def test_can_isolate_node(self, fail_node, mode):
with tempfile.TemporaryDirectory() as outdir:
run_polygraphy_debug(["reduce", ONNX_MODELS["reducable"].path, "--output=reduced.onnx", "--mode", mode, "--show-output",
"--min-good=good_reduced.onnx",
"--check", TestReduce.FAKE_REDUCE_CHECKER, "polygraphy_debug.onnx", "--fail-node", fail_node],
disable_verbose=True, cwd=outdir)
model = onnx_from_path(os.path.join(outdir, "reduced.onnx"))
min_good_path = os.path.join(outdir, "good_reduced.onnx")
good_model = None
if os.path.exists(min_good_path):
good_model = onnx_from_path(min_good_path)
# The model should only contain one node - the failing one.
# One exception - since bisect depends on node ordering, it sometimes doesn't
# reduce branches to the maximum possible extent.
if mode == "bisect" and fail_node == "onnx_graphsurgeon_node_1":
assert len(model.graph.node) == 3
elif mode == "bisect" and fail_node == "onnx_graphsurgeon_node_7":
assert len(model.graph.node) == 2
else:
assert len(model.graph.node) == 1
node_names = [node.name for node in model.graph.node]
assert fail_node in node_names
# For now we're just doing a very basic sanity check for --min-good
if good_model:
assert model != good_model
def test_reduce_with_constant(self):
# Should be no failure when models including Constant nodes use fallback
# shape inference; Constant nodes will be lowered to constant tensors.
with tempfile.TemporaryDirectory() as outdir:
run_polygraphy_debug(
[
"reduce",
ONNX_MODELS["reducable_with_const"].path,
"--no-shape-inference",
"--mode=linear",
"--output=reduced.onnx",
] + [
"--check",
TestReduce.FAKE_REDUCE_CHECKER,
"polygraphy_debug.onnx",
"--fail-node",
"onnx_graphsurgeon_node_3",
],
disable_verbose=True,
cwd=outdir,
)
model = onnx_from_path(os.path.join(outdir, "reduced.onnx"))
graph = gs.import_onnx(model)
assert len(graph.nodes) == 1
assert graph.nodes[0].name == "onnx_graphsurgeon_node_3"
# Outputs of Constant nodes should not become Variables; thus the model should have no inputs.
assert not graph.inputs
def test_reduce_custom_return_code(self):
with tempfile.TemporaryDirectory() as outdir:
run_polygraphy_debug(["reduce", ONNX_MODELS["reducable"].path, "--output=reduced.onnx", "--show-output", "--fail-code=1", # Only 1s are real failures.
"--check", TestReduce.FAKE_REDUCE_CHECKER, "polygraphy_debug.onnx", "--fail-node", "onnx_graphsurgeon_node_5", "--default-return-code=2"],
disable_verbose=True, cwd=outdir)
model = onnx_from_path(os.path.join(outdir, "reduced.onnx"))
assert len(model.graph.node) == 1
assert model.graph.node[0].name == "onnx_graphsurgeon_node_5"
def test_no_reduce_required_branches(self, fail_nodes):
with tempfile.TemporaryDirectory() as outdir:
run_polygraphy_debug(["reduce", ONNX_MODELS["reducable"].path, "--output=reduced.onnx", "--show-output",
"--check", TestReduce.FAKE_REDUCE_CHECKER, "polygraphy_debug.onnx",
"--fail-node"] + fail_nodes,
disable_verbose=True, cwd=outdir)
model = onnx_from_path(os.path.join(outdir, "reduced.onnx"))
node_names = [node.name for node in model.graph.node]
assert all(fail_node in node_names for fail_node in fail_nodes)
assert len(model.graph.node) <= 3 # The branch on the opposite side of the model should be removed.
def test_good_bad(self):
with tempfile.TemporaryDirectory() as outdir:
# Also includes --show-output sanity test
status = run_polygraphy_debug(
[
"build",
ONNX_MODELS["identity"].path,
"--save-tactics=replay.json",
"--show-output",
"--artifacts-dir",
outdir,
"--until=good",
"--artifacts",
"replay.json",
"--check",
"true",
],
cwd=outdir,
)
assert "Passed: 1/1 | Pass Rate: 100.0%" in status.stdout
status = run_polygraphy_debug(
[
"build",
ONNX_MODELS["identity"].path,
"--save-tactics=replay.json",
"--artifacts-dir",
outdir,
"--until=bad",
"--artifacts",
"replay.json",
"--check",
"false",
],
cwd=outdir,
)
assert "Passed: 0/1 | Pass Rate: 0.0%" in status.stdout
def check_outdir(subdir):
files = glob.glob(os.path.join(outdir, subdir, "*"))
assert len(files) == 1
basenames = list(map(os.path.basename, files))
assert len([
f for f in basenames
if f.startswith("replay") and f.endswith(".json")
]) == 1
check_outdir("good")
check_outdir("bad")
def test_ignore_fail_code(self):
# Sanity check to make sure the command normally fails.
status = run_polygraphy_debug(
["repeat", "--until=5", "--check", "false"])
assert "Passed: 0/5 | Pass Rate: 0.0%" in status.stdout
status = run_polygraphy_debug([
"repeat", "--until=5", "--ignore-fail-code=2", "--check", "false"
])
assert "Passed: 0/5 | Pass Rate: 0.0%" in status.stdout
status = run_polygraphy_debug([
"repeat", "--until=5", "--ignore-fail-code=1", "--check", "false"
])
assert "Passed: 5/5 | Pass Rate: 100.0%" in status.stdout
def test_sanity(self, mode, direction, check_status, model):
with tempfile.TemporaryDirectory() as outdir:
run_polygraphy_debug(
[
"precision",
"--mode",
mode,
"--direction",
direction,
ONNX_MODELS[model].path,
"--int8",
"--check",
check_status,
],
cwd=outdir,
)
def test_good_bad(self, replay_dir):
replay_dir, expected_output = replay_dir
good = os.path.join(replay_dir, "good")
bad = os.path.join(replay_dir, "bad")
status = run_polygraphy_debug(["diff-tactics", "--good", good, "--bad", bad], disable_verbose=True)
self.check_output(status, expected_output)
def test_iteration_info(self):
with tempfile.TemporaryDirectory() as outdir:
iter_info = os.path.join(outdir, "iter_info.json")
check_script = os.path.join(outdir, "check.py")
# Hacky Python script to make sure the iteration is actually incremented
check_num = """
import json
import os
iter_info = json.load(open('{:}', "r"))
file_name = str(iter_info["iteration"]) + ".txt"
path = os.path.abspath(file_name)
print(path)
assert not os.path.exists(path)
with open(path, "w") as f:
f.write("File")
""".format(iter_info)
with open(check_script, "w") as f:
f.write(dedent(check_num))
status = run_polygraphy_debug(["repeat", "--until=5", "--iteration-info", iter_info, "--show-output",
"--check", sys.executable, check_script], cwd=outdir)
assert "FAILED" not in status.stdout
assert "Passed: 5/5 | Pass Rate: 100.0%" in status.stdout
# Iteration info should be cleaned up afterwards
assert not os.path.exists(iter_info)
def test_reduce_shape_inference(self, opts):
with tempfile.TemporaryDirectory() as outdir:
status = run_polygraphy_debug(["reduce", ONNX_MODELS["dynamic_identity"].path, "--output=reduced.onnx",
"--show-output", "--model-input-shapes=X:[1,2,5,5]"] + opts
+ ["--check", "false"],
disable_verbose=True, cwd=outdir)
model = onnx_from_path(os.path.join(outdir, "reduced.onnx"))
graph = gs.import_onnx(model)
assert tuple(graph.inputs[0].shape) == (1, 2, 5, 5)
assert tuple(graph.outputs[0].shape) == (1, 2, 5, 5)
def test_good_bad_file(self, replay_dir):
replay_dir, expected_output = replay_dir
def find_file(dirpath, filename):
return glob.glob(os.path.join(dirpath, "**", filename),
recursive=True)[0]
good = find_file(os.path.join(replay_dir, "good"), "0.json")
bad = find_file(os.path.join(replay_dir, "bad"), "1.json")
status = run_polygraphy_debug(
["diff-tactics", "--good", good, "--bad", bad],
disable_verbose=True)
self.check_output(status, expected_output, expected_num=1)
def test_until(self, until, check, expected_iters):
status = run_polygraphy_debug(
["repeat", "--until", until, "--check", check])
assert "Finished {:} iteration(s)".format(
expected_iters) in status.stdout
def test_dir(self, replay_dir):
replay_dir, expected_output = replay_dir
status = run_polygraphy_debug(["diff-tactics", "--dir", replay_dir],
disable_verbose=True)
self.check_output(status, expected_output)
