def test_no_input(monkeypatch): set_command_line_flags(["argv0", "--env=llvm-ic-v0"]) monkeypatch.setattr("sys.stdin", StringIO("")) with capture_output() as out: with pytest.raises(SystemExit): main(["argv0", "-"]) assert "No inputs to validate" in out.stderr
def test_invalid_csv_format(monkeypatch): stdin = "invalid\ncsv\nformat" set_command_line_flags(["argv0", "--env=llvm-ic-v0"]) monkeypatch.setattr("sys.stdin", StringIO(stdin)) with capture_output() as out: with pytest.raises(SystemExit): main(["argv0", "-"]) assert "Expected 4 columns in the first row of CSV" in out.stderr
def test_validate_cbench_null_options(monkeypatch, benchmarks: List[str]): stdin = "\n".join([ "benchmark,reward,walltime,commandline", ] + [f"{b},,0,opt input.bc -o output.bc" for b in benchmarks]) set_command_line_flags(["argv0", "--env=llvm-v0"]) monkeypatch.setattr("sys.stdin", StringIO(stdin)) with capture_output() as out: main(["argv0", "-"]) assert not out.stderr assert out.stdout.count("✅") == len(benchmarks) # Every benchmark passed.
def test_invalid_csv_format(monkeypatch): input = "invalid\ncsv\nformat" flags.FLAGS.unparse_flags() flags.FLAGS(["argv0", "--env=llvm-ic-v0", "--dataset=cBench-v0"]) monkeypatch.setattr("sys.stdin", StringIO(input)) with capture_output() as out: with pytest.raises(SystemExit): main(["argv0"]) assert "Failed to parse input:" in out.stderr
def test_okay_llvm_result(monkeypatch): stdin = """ benchmark,reward,commandline,walltime benchmark://cbench-v1/crc32,0,opt input.bc -o output.bc,0.3 """.strip() set_command_line_flags(["argv0", "--env=llvm-ic-v0"]) monkeypatch.setattr("sys.stdin", StringIO(stdin)) with capture_output() as out: main(["argv0", "-"]) assert "✅ cbench-v1/crc32 " in out.stdout assert not out.stderr
def test_invalid_reward_llvm_result(monkeypatch): stdin = """ benchmark,reward,commandline,walltime benchmark://cbench-v1/crc32,0.5,opt input.bc -o output.bc,0.3 """.strip() set_command_line_flags(["argv0", "--env=llvm-ic-v0"]) monkeypatch.setattr("sys.stdin", StringIO(stdin)) with capture_output() as out: with pytest.raises(SystemExit): main(["argv0", "-"]) assert ("❌ cbench-v1/crc32 Expected reward 0.5 but received reward 0.0\n" in out.stdout) assert not out.stderr
def test_okay_llvm_result(monkeypatch): input = """ benchmark,reward,commandline,walltime benchmark://cBench-v0/dijkstra,0,opt input.bc -o output.bc,0.3 """.strip() flags.FLAGS.unparse_flags() flags.FLAGS(["argv0", "--env=llvm-ic-v0", "--dataset=cBench-v0"]) monkeypatch.setattr("sys.stdin", StringIO(input)) with capture_output() as out: main(["argv0"]) assert out.stdout.startswith("✅ cBench-v0/dijkstra ") assert not out.stderr
def test_multiple_valid_inputs(monkeypatch): stdin = """ benchmark,reward,walltime,commandline benchmark://cbench-v1/crc32,,0,opt input.bc -o output.bc benchmark://cbench-v1/crc32,,0,opt input.bc -o output.bc benchmark://cbench-v1/crc32,,0,opt input.bc -o output.bc """.strip() set_command_line_flags(["argv0", "--env=llvm-v0"]) monkeypatch.setattr("sys.stdin", StringIO(stdin)) with capture_output() as out: main(["argv0", "-"]) assert not out.stderr assert out.stdout.count("✅") == 3 # Every benchmark passed.
def test_okay_llvm_result_file_input(): with tempfile.TemporaryDirectory() as d: path = Path(d) / "test.csv" with open(str(path), "w") as f: f.write(""" benchmark,reward,commandline,walltime benchmark://cbench-v1/crc32,0,opt input.bc -o output.bc,0.3 """.strip()) set_command_line_flags(["argv0", "--env=llvm-ic-v0"]) with capture_output() as out: main(["argv0", str(path)]) assert "✅ cbench-v1/crc32 " in out.stdout assert not out.stderr
def test_invalid_reward_llvm_result(monkeypatch): input = """ benchmark,reward,commandline,walltime benchmark://cBench-v0/dijkstra,0.5,opt input.bc -o output.bc,0.3 """.strip() flags.FLAGS.unparse_flags() flags.FLAGS(["argv0", "--env=llvm-ic-v0", "--dataset=cBench-v0"]) monkeypatch.setattr("sys.stdin", StringIO(input)) with capture_output() as out: with pytest.raises(SystemExit): main(["argv0"]) assert out.stdout.startswith( "❌ cBench-v0/dijkstra Expected reward 0.5000 but received reward 0.0000\n" ) assert not out.stderr
def test_validate_cBenh_null_options(monkeypatch): input = """ benchmark,reward,walltime,commandline benchmark://cBench-v0/gsm,,0,opt input.bc -o output.bc benchmark://cBench-v0/lame,,0,opt input.bc -o output.bc benchmark://cBench-v0/stringsearch,,0,opt input.bc -o output.bc benchmark://cBench-v0/ghostscript,,0,opt input.bc -o output.bc benchmark://cBench-v0/qsort,,0,opt input.bc -o output.bc benchmark://cBench-v0/sha,,0,opt input.bc -o output.bc benchmark://cBench-v0/ispell,,0,opt input.bc -o output.bc benchmark://cBench-v0/blowfish,,0,opt input.bc -o output.bc benchmark://cBench-v0/adpcm,,0,opt input.bc -o output.bc benchmark://cBench-v0/tiffdither,,0,opt input.bc -o output.bc benchmark://cBench-v0/bzip2,,0,opt input.bc -o output.bc benchmark://cBench-v0/stringsearch2,,0,opt input.bc -o output.bc benchmark://cBench-v0/bitcount,,0,opt input.bc -o output.bc benchmark://cBench-v0/jpeg-d,,0,opt input.bc -o output.bc benchmark://cBench-v0/jpeg-c,,0,opt input.bc -o output.bc benchmark://cBench-v0/dijkstra,,0,opt input.bc -o output.bc benchmark://cBench-v0/rijndael,,0,opt input.bc -o output.bc benchmark://cBench-v0/patricia,,0,opt input.bc -o output.bc benchmark://cBench-v0/tiff2rgba,,0,opt input.bc -o output.bc benchmark://cBench-v0/crc32,,0,opt input.bc -o output.bc benchmark://cBench-v0/tiff2bw,,0,opt input.bc -o output.bc benchmark://cBench-v0/tiffmedian,,0,opt input.bc -o output.bc benchmark://cBench-v0/susan,,0,opt input.bc -o output.bc """.strip() flags.FLAGS.unparse_flags() flags.FLAGS(["argv0", "--env=llvm-v0", "--dataset=cBench-v0"]) monkeypatch.setattr("sys.stdin", StringIO(input)) with capture_output() as out: main(["argv0"]) assert out.stdout.count("✅") == 23 # Every benchmark passed. assert not out.stderr