def test_no_input(monkeypatch):
set_command_line_flags(["argv0", "--env=llvm-ic-v0"])
monkeypatch.setattr("sys.stdin", StringIO(""))
with capture_output() as out:
with pytest.raises(SystemExit):
main(["argv0", "-"])
assert "No inputs to validate" in out.stderr
def test_invalid_csv_format(monkeypatch):
stdin = "invalid\ncsv\nformat"
set_command_line_flags(["argv0", "--env=llvm-ic-v0"])
monkeypatch.setattr("sys.stdin", StringIO(stdin))
with capture_output() as out:
with pytest.raises(SystemExit):
main(["argv0", "-"])
assert "Expected 4 columns in the first row of CSV" in out.stderr
def test_validate_cbench_null_options(monkeypatch, benchmarks: List[str]):
stdin = "\n".join([
"benchmark,reward,walltime,commandline",
] + [f"{b},,0,opt input.bc -o output.bc" for b in benchmarks])
set_command_line_flags(["argv0", "--env=llvm-v0"])
monkeypatch.setattr("sys.stdin", StringIO(stdin))
with capture_output() as out:
main(["argv0", "-"])
assert not out.stderr
assert out.stdout.count("✅") == len(benchmarks) # Every benchmark passed.
def test_invalid_csv_format(monkeypatch):
input = "invalid\ncsv\nformat"
flags.FLAGS.unparse_flags()
flags.FLAGS(["argv0", "--env=llvm-ic-v0", "--dataset=cBench-v0"])
monkeypatch.setattr("sys.stdin", StringIO(input))
with capture_output() as out:
with pytest.raises(SystemExit):
main(["argv0"])
assert "Failed to parse input:" in out.stderr
def test_okay_llvm_result(monkeypatch):
stdin = """
benchmark,reward,commandline,walltime
benchmark://cbench-v1/crc32,0,opt input.bc -o output.bc,0.3
""".strip()
set_command_line_flags(["argv0", "--env=llvm-ic-v0"])
monkeypatch.setattr("sys.stdin", StringIO(stdin))
with capture_output() as out:
main(["argv0", "-"])
assert "✅ cbench-v1/crc32 " in out.stdout
assert not out.stderr
def test_invalid_reward_llvm_result(monkeypatch):
stdin = """
benchmark,reward,commandline,walltime
benchmark://cbench-v1/crc32,0.5,opt input.bc -o output.bc,0.3
""".strip()
set_command_line_flags(["argv0", "--env=llvm-ic-v0"])
monkeypatch.setattr("sys.stdin", StringIO(stdin))
with capture_output() as out:
with pytest.raises(SystemExit):
main(["argv0", "-"])
assert ("❌ cbench-v1/crc32 Expected reward 0.5 but received reward 0.0\n"
in out.stdout)
assert not out.stderr
def test_okay_llvm_result(monkeypatch):
input = """
benchmark,reward,commandline,walltime
benchmark://cBench-v0/dijkstra,0,opt input.bc -o output.bc,0.3
""".strip()
flags.FLAGS.unparse_flags()
flags.FLAGS(["argv0", "--env=llvm-ic-v0", "--dataset=cBench-v0"])
monkeypatch.setattr("sys.stdin", StringIO(input))
with capture_output() as out:
main(["argv0"])
assert out.stdout.startswith("✅ cBench-v0/dijkstra ")
assert not out.stderr
def test_multiple_valid_inputs(monkeypatch):
stdin = """
benchmark,reward,walltime,commandline
benchmark://cbench-v1/crc32,,0,opt input.bc -o output.bc
benchmark://cbench-v1/crc32,,0,opt input.bc -o output.bc
benchmark://cbench-v1/crc32,,0,opt input.bc -o output.bc
""".strip()
set_command_line_flags(["argv0", "--env=llvm-v0"])
monkeypatch.setattr("sys.stdin", StringIO(stdin))
with capture_output() as out:
main(["argv0", "-"])
assert not out.stderr
assert out.stdout.count("✅") == 3 # Every benchmark passed.
def test_okay_llvm_result_file_input():
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "test.csv"
with open(str(path), "w") as f:
f.write("""
benchmark,reward,commandline,walltime
benchmark://cbench-v1/crc32,0,opt input.bc -o output.bc,0.3
""".strip())
set_command_line_flags(["argv0", "--env=llvm-ic-v0"])
with capture_output() as out:
main(["argv0", str(path)])
assert "✅ cbench-v1/crc32 " in out.stdout
assert not out.stderr
def test_invalid_reward_llvm_result(monkeypatch):
input = """
benchmark,reward,commandline,walltime
benchmark://cBench-v0/dijkstra,0.5,opt input.bc -o output.bc,0.3
""".strip()
flags.FLAGS.unparse_flags()
flags.FLAGS(["argv0", "--env=llvm-ic-v0", "--dataset=cBench-v0"])
monkeypatch.setattr("sys.stdin", StringIO(input))
with capture_output() as out:
with pytest.raises(SystemExit):
main(["argv0"])
assert out.stdout.startswith(
"❌ cBench-v0/dijkstra Expected reward 0.5000 but received reward 0.0000\n"
)
assert not out.stderr
def test_validate_cBenh_null_options(monkeypatch):
input = """
benchmark,reward,walltime,commandline
benchmark://cBench-v0/gsm,,0,opt input.bc -o output.bc
benchmark://cBench-v0/lame,,0,opt input.bc -o output.bc
benchmark://cBench-v0/stringsearch,,0,opt input.bc -o output.bc
benchmark://cBench-v0/ghostscript,,0,opt input.bc -o output.bc
benchmark://cBench-v0/qsort,,0,opt input.bc -o output.bc
benchmark://cBench-v0/sha,,0,opt input.bc -o output.bc
benchmark://cBench-v0/ispell,,0,opt input.bc -o output.bc
benchmark://cBench-v0/blowfish,,0,opt input.bc -o output.bc
benchmark://cBench-v0/adpcm,,0,opt input.bc -o output.bc
benchmark://cBench-v0/tiffdither,,0,opt input.bc -o output.bc
benchmark://cBench-v0/bzip2,,0,opt input.bc -o output.bc
benchmark://cBench-v0/stringsearch2,,0,opt input.bc -o output.bc
benchmark://cBench-v0/bitcount,,0,opt input.bc -o output.bc
benchmark://cBench-v0/jpeg-d,,0,opt input.bc -o output.bc
benchmark://cBench-v0/jpeg-c,,0,opt input.bc -o output.bc
benchmark://cBench-v0/dijkstra,,0,opt input.bc -o output.bc
benchmark://cBench-v0/rijndael,,0,opt input.bc -o output.bc
benchmark://cBench-v0/patricia,,0,opt input.bc -o output.bc
benchmark://cBench-v0/tiff2rgba,,0,opt input.bc -o output.bc
benchmark://cBench-v0/crc32,,0,opt input.bc -o output.bc
benchmark://cBench-v0/tiff2bw,,0,opt input.bc -o output.bc
benchmark://cBench-v0/tiffmedian,,0,opt input.bc -o output.bc
benchmark://cBench-v0/susan,,0,opt input.bc -o output.bc
""".strip()
flags.FLAGS.unparse_flags()
flags.FLAGS(["argv0", "--env=llvm-v0", "--dataset=cBench-v0"])
monkeypatch.setattr("sys.stdin", StringIO(input))
with capture_output() as out:
main(["argv0"])
assert out.stdout.count("✅") == 23 # Every benchmark passed.
assert not out.stderr
