def test_state_equality_same():
a = CompilerEnvState(benchmark="benchmark://cbench-v0/foo",
walltime=10,
commandline="-a -b -c")
b = CompilerEnvState(benchmark="benchmark://cbench-v0/foo",
walltime=10,
commandline="-a -b -c")
assert a == b # testing __eq__
assert not a != b # noqa testing __ne__
def test_state_to_json_from_dict_no_reward():
original_state = CompilerEnvState(benchmark="benchmark://cbench-v0/foo",
walltime=100,
commandline="-a -b -c")
state_from_dict = CompilerEnvState(**json.loads(original_state.json()))
assert state_from_dict.benchmark == "benchmark://cbench-v0/foo"
assert state_from_dict.walltime == 100
assert state_from_dict.reward is None
assert state_from_dict.commandline == "-a -b -c"
def test_state_equality_differnt_walltime():
"""Test that walltime is not compared."""
a = CompilerEnvState(benchmark="benchmark://cbench-v0/foo",
walltime=10,
commandline="-a -b -c")
b = CompilerEnvState(benchmark="benchmark://cbench-v0/foo",
walltime=5,
commandline="-a -b -c")
assert a == b # testing __eq__
assert not a != b # noqa testing __ne__
def test_validation_result_join_two_inputs_different_errors():
a = ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=1,
),
walltime=3,
errors=[
ValidationError(
type="Syntax Error",
data={"data": [1, 2, 3]},
)
],
)
b = ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=10,
),
walltime=3,
errors=[
ValidationError(
type="Type Error",
data={"a": "b"},
)
],
)
c = ValidationResult.join([a, b])
assert c == ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=10,
),
walltime=3,
errors=[
ValidationError(
type="Syntax Error",
data={"data": [1, 2, 3]},
),
ValidationError(
type="Type Error",
data={"a": "b"},
),
],
)
# Test walltime, which is excluded from equality comparisons.
assert c.walltime == 6
def test_state_equality_one_sided_reward():
a = CompilerEnvState(
benchmark="benchmark://cbench-v0/foo",
walltime=5,
commandline="-a -b -c",
reward=2,
)
b = CompilerEnvState(benchmark="benchmark://cbench-v0/foo",
walltime=5,
commandline="-a -b -c")
assert a == b # testing __eq__
assert b == a # testing __eq__
assert not a != b # noqa testing __ne__
assert not b != a # noqa testing __ne__
def test_state_invalid_walltime():
with pytest.raises(PydanticValidationError,
match="Walltime cannot be negative"):
CompilerEnvState(
benchmark="benchmark://cbench-v0/foo",
walltime=-1,
reward=1.5,
commandline="",
)
def test_validation_result_equality_different_walltimes():
a = ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=1,
),
walltime=3,
)
b = ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=10,
),
walltime=10,
)
assert a == b
def test_compiler_env_state_reader_no_header():
buf = StringIO("benchmark://cbench-v0/foo,2.0,5.0,-a -b -c\n")
reader = CompilerEnvStateReader(buf)
assert list(reader) == [
CompilerEnvState(
benchmark="benchmark://cbench-v0/foo",
walltime=5,
commandline="-a -b -c",
reward=2,
)
]
def test_validate_state_no_reward():
state = CompilerEnvState(
benchmark="benchmark://cbench-v1/crc32",
walltime=1,
commandline="opt input.bc -o output.bc",
)
with gym.make("llvm-v0") as env:
result = env.validate(state)
assert result.okay()
assert not result.reward_validated
assert str(result) == "✅ cbench-v1/crc32"
def test_compiler_env_state_reader_with_header_out_of_order_columns():
buf = StringIO("commandline,reward,benchmark,walltime\n"
"-a -b -c,2.0,benchmark://cbench-v0/foo,5.0\n")
reader = CompilerEnvStateReader(buf)
assert list(reader) == [
CompilerEnvState(
benchmark="benchmark://cbench-v0/foo",
walltime=5,
commandline="-a -b -c",
reward=2,
)
]
def test_validate_state_without_state_reward():
"""Validating state when state has no reward value."""
state = CompilerEnvState(
benchmark="benchmark://cbench-v1/crc32",
walltime=1,
commandline="opt input.bc -o output.bc",
)
with gym.make("llvm-v0", reward_space="IrInstructionCount") as env:
result = env.validate(state)
assert result.okay()
assert not result.reward_validated
assert not result.reward_validation_failed
def test_state_serialize_deserialize_equality_no_reward():
original_state = CompilerEnvState(benchmark="benchmark://cbench-v0/foo",
walltime=100,
commandline="-a -b -c")
buf = StringIO()
CompilerEnvStateWriter(buf).write_state(original_state)
buf.seek(0) # Rewind the buffer for reading.
state_from_csv = next(iter(CompilerEnvStateReader(buf)))
assert state_from_csv.benchmark == "benchmark://cbench-v0/foo"
assert state_from_csv.walltime == 100
assert state_from_csv.reward is None
assert state_from_csv.commandline == "-a -b -c"
def test_validation_result_equality_different_errors_order():
a = ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=1,
),
walltime=3,
errors=[
ValidationError(
type="Syntax Error",
data={"data": [1, 2, 3]},
),
ValidationError(
type="Runtime Error",
data={"a": "b"},
),
],
)
b = ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=1,
),
walltime=3,
errors=[
ValidationError(
type="Runtime Error",
data={"a": "b"},
),
ValidationError(
type="Syntax Error",
data={"data": [1, 2, 3]},
),
],
)
assert a == b
def test_compiler_env_state_writer_no_header():
buf = StringIO()
writer = CompilerEnvStateWriter(buf, header=False)
writer.write_state(
CompilerEnvState(
benchmark="benchmark://cbench-v0/foo",
walltime=5,
commandline="-a -b -c",
reward=2,
),
flush=True,
)
assert buf.getvalue() == "benchmark://cbench-v0/foo,2.0,5.0,-a -b -c\n"
def test_validate_state_with_reward():
state = CompilerEnvState(
benchmark="benchmark://cbench-v1/crc32",
walltime=1,
reward=0,
commandline="opt input.bc -o output.bc",
)
with gym.make("llvm-v0", reward_space="IrInstructionCount") as env:
result = env.validate(state)
assert result.okay()
assert result.reward_validated
assert not result.reward_validation_failed
assert str(result) == "✅ cbench-v1/crc32 0.0000"
def test_validate_states_lambda_callback(inorder, nproc):
state = CompilerEnvState(
benchmark="benchmark://cbench-v1/crc32",
walltime=1,
commandline="opt input.bc -o output.bc",
)
results = list(
validate_states(
make_env=lambda: gym.make("llvm-v0"),
states=[state],
inorder=inorder,
nproc=nproc,
))
assert len(results) == 1
assert results[0].okay()
def test_validation_result_json():
result = ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=1,
),
walltime=3,
errors=[
ValidationError(
type="Syntax Error",
data={"data": [1, 2, 3]},
)
],
)
assert ValidationResult(**json.loads(result.json())) == result
def test_validation_result_join_one_input():
result = ValidationResult(
state=CompilerEnvState(
benchmark="benchmark://example-v0/test",
commandline="test",
walltime=1,
),
walltime=3,
errors=[
ValidationError(
type="Syntax Error",
data={"data": [1, 2, 3]},
)
],
)
joined_result = ValidationResult.join([result])
assert result == joined_result
def test_compiler_env_state_writer_with_statement(tmpwd: Path, flush: bool):
path = Path("results.csv")
assert not path.is_file() # Sanity check.
f = open(path, "w")
with CompilerEnvStateWriter(f) as writer:
writer.write_state(
CompilerEnvState(
benchmark="benchmark://cbench-v0/foo",
walltime=5,
commandline="-a -b -c",
reward=2,
),
flush=flush,
)
assert f.closed
with open(path) as f:
assert f.read() == ("benchmark,reward,walltime,commandline\n"
"benchmark://cbench-v0/foo,2.0,5.0,-a -b -c\n")
def test_validate_state_without_env_reward():
"""Validating state when environment has no reward space."""
state = CompilerEnvState(
benchmark="benchmark://cbench-v1/crc32",
walltime=1,
reward=0,
commandline="opt input.bc -o output.bc",
)
with gym.make("llvm-v0") as env:
with pytest.warns(
UserWarning,
match=(
"Validating state with reward, "
"but environment has no reward space set"
),
):
result = env.validate(state)
assert result.okay()
assert not result.reward_validated
assert not result.reward_validation_failed
def test_state_equality_different_types():
state = CompilerEnvState(benchmark="benchmark://cbench-v0/foo",
walltime=10,
commandline="-a -b -c")
assert not state == 5 # noqa testing __eq__
assert state != 5 # testing __ne__
def test_state_from_dict_empty():
with pytest.raises(PydanticValidationError):
CompilerEnvState(**{})
def test_state_invalid_benchmark_uri():
with pytest.raises(PydanticValidationError, match="benchmark"):
CompilerEnvState(benchmark="invalid",
walltime=100,
reward=1.5,
commandline="")