def test_wrapped_step_multi_step(env: LlvmEnv):
env = TimeLimit(env, max_episode_steps=5)
env.reset(benchmark="benchmark://cbench-v1/dijkstra")
env.step([0, 0, 0])
assert env.benchmark == "benchmark://cbench-v1/dijkstra"
assert env.actions == [0, 0, 0]
def test_invalid_runtime_count(env: LlvmEnv):
env = RuntimePointEstimateReward(env, runtime_count=-10)
with pytest.raises(
ValueError,
match="runtimes_per_observation_count must be >= 1. Received: -10"
):
env.reset()
def test_wrapped_step_multi_step(env: LlvmEnv):
"""Test passing a list of actions to step()."""
env = CompilerEnvWrapper(env)
env.reset()
env.multistep([0, 0, 0])
assert env.actions == [0, 0, 0]
def test_deep_copy(env: LlvmEnv):
"""Test that deep copy creates an independent copy."""
env.reset()
with deepcopy(env) as cpy:
assert cpy.state == env.state
env.step(env.action_space.sample())
assert cpy.state != env.state
def test_invalid_warmup_count(env: LlvmEnv):
env = RuntimePointEstimateReward(env, warmup_count=-10)
with pytest.raises(
ValueError,
match="warmup_runs_count_per_runtime_observation must be >= 0. Received: -10",
):
env.reset()
def test_reward_range_not_runnable_benchmark(env: LlvmEnv):
env = RuntimePointEstimateReward(env, runtime_count=3)
with pytest.raises(
BenchmarkInitError, match=r"^Benchmark is not runnable: benchmark://npb-v0/1$"
):
env.reset(benchmark="benchmark://npb-v0/1")
def test_csmith_positive_buildtimes(env: LlvmEnv,
csmith_dataset: CsmithDataset):
benchmark = next(csmith_dataset.benchmarks())
env.reset(benchmark=benchmark)
val = env.observation["Buildtime"]
print(val.tolist())
assert np.all(np.greater(val, 0))
def test_reset_invalid_ir(env: LlvmEnv):
"""Test that setting the $CXX to an invalid binary raises an error."""
benchmark = llvm.make_benchmark(INVALID_IR_PATH)
with pytest.raises(BenchmarkInitError,
match="Failed to compute .text size cost"):
env.reset(benchmark=benchmark)
def test_benchmark_constructor_arg(env: LlvmEnv):
env.close() # Fixture only required to pull in dataset.
with gym.make("llvm-v0", benchmark="cbench-v1/dijkstra") as env:
assert env.benchmark == "benchmark://cbench-v1/dijkstra"
env.reset()
assert env.benchmark == "benchmark://cbench-v1/dijkstra"
def test_failing_build_cmd(env: LlvmEnv, tmpdir):
"""Test that Runtime observation raises an error if build command fails."""
with open(tmpdir / "program.c", "w") as f:
f.write("""
#include <stdio.h>
int main(int argc, char** argv) {
printf("Hello\\n");
for (int i = 0; i < 10; ++i) {
argc += 2;
}
return argc - argc;
}
""")
benchmark = env.make_benchmark(Path(tmpdir) / "program.c")
benchmark.proto.dynamic_config.build_cmd.argument.extend(
["$CC", "$IN", "-invalid-cc-argument"])
benchmark.proto.dynamic_config.build_cmd.outfile.extend(["a.out"])
benchmark.proto.dynamic_config.build_cmd.timeout_seconds = 10
benchmark.proto.dynamic_config.run_cmd.argument.extend(["./a.out"])
benchmark.proto.dynamic_config.run_cmd.timeout_seconds = 10
env.reset(benchmark=benchmark)
with pytest.raises(
ServiceError,
match=
(r"Command '\$CC \$IN -invalid-cc-argument' failed with exit code 1: "
r"clang: error: unknown argument: '-invalid-cc-argument'"),
):
env.observation.Runtime()
def test_wrapped_step_custom_args(env: LlvmEnv, wrapper_type):
"""Test passing the custom CompilerGym step() keyword arguments."""
class MyWrapper(wrapper_type):
def convert_observation(self, observation):
return observation # pass thru
def action(self, action):
return action # pass thru
def convert_reward(self, reward):
return reward
env = MyWrapper(env)
env.reset()
(ir, ic), (icr, icroz), _, _ = env.multistep(
actions=[0, 0, 0],
observation_spaces=["Ir", "IrInstructionCount"],
reward_spaces=["IrInstructionCount", "IrInstructionCountOz"],
)
assert isinstance(ir, str)
assert isinstance(ic, int)
assert isinstance(icr, float)
assert isinstance(icroz, float)
assert env.unwrapped.observation.spaces["Ir"].space.contains(ir)
assert env.unwrapped.observation.spaces[
"IrInstructionCount"].space.contains(ic)
def test_env_spec_make_workaround(env: LlvmEnv):
"""Demonstrate how #587 would be fixed, by updating the 'kwargs' dict."""
env.reset(benchmark="cbench-v1/bitcount")
env.spec._kwargs[ # pylint: disable=protected-access
"benchmark"
] = "cbench-v1/bitcount"
with env.spec.make() as new_env:
assert new_env.benchmark == env.benchmark
def test_env_spec_make(env: LlvmEnv):
"""Test that demonstrates a failure in gym compatibility: env.spec does
not encode mutable state like benchmark, reward space, and observation
space.
"""
env.reset(benchmark="cbench-v1/bitcount")
with env.spec.make() as new_env:
assert new_env.benchmark == env.benchmark
def test_no_module_id_builtin_benchmark(env: LlvmEnv):
"""Test that the module and source IDs are stripped in shipped benchmark."""
env.reset("cbench-v1/crc32")
ir = env.ir
print(ir) # For debugging in case of error.
assert "; ModuleID = '-'\n" in ir
assert '\nsource_filename = "-"\n' in ir
def test_clgen_random_select(env: LlvmEnv, clgen_dataset: CLgenDataset,
index: int, tmpwd: Path):
uri = next(islice(clgen_dataset.benchmark_uris(), index, None))
benchmark = clgen_dataset.benchmark(uri)
env.reset(benchmark=benchmark)
assert benchmark.source
benchmark.write_sources_to_directory(tmpwd)
assert (tmpwd / "kernel.cl").is_file()
def test_reset_to_force_benchmark(env: LlvmEnv):
"""Test that calling reset() with a benchmark forces that benchmark to
be used for every subsequent episode.
"""
env.reset(benchmark="benchmark://cbench-v1/crc32")
assert env.benchmark == "benchmark://cbench-v1/crc32"
for _ in range(10):
env.reset()
assert env.benchmark == "benchmark://cbench-v1/crc32"
def test_poj104_random_select(env: LlvmEnv, poj104_dataset: POJ104Dataset,
index: int, tmpwd: Path):
uri = next(islice(poj104_dataset.benchmark_uris(), index, None))
benchmark = poj104_dataset.benchmark(uri)
env.reset(benchmark=benchmark)
assert benchmark.source
benchmark.write_sources_to_directory(tmpwd)
assert (tmpwd / "source.cc").is_file()
def test_observation_equivalence(env: LlvmEnv, tmpdir, observation_space: str):
"""Test that compute_observation() produces the same result as the environment."""
tmpdir = Path(tmpdir)
env.reset()
env.write_bitcode(tmpdir / "ir.bc")
observation = compute_observation(
env.observation.spaces[observation_space], tmpdir / "ir.bc")
assert observation == env.observation[observation_space]
def test_default_runtime_observation_count_fork(env: LlvmEnv):
"""Test that default property values propagate on fork()."""
env.reset()
rc = env.runtime_observation_count
wc = env.runtime_warmup_runs_count
with env.fork() as fkd:
assert fkd.runtime_observation_count == rc
assert fkd.runtime_warmup_runs_count == wc
def test_invalid_runtime_count(env: LlvmEnv):
env.reset()
with pytest.raises(ValueError,
match=r"runtimes_per_observation_count must be >= 1"):
env.runtime_observation_count = 0
with pytest.raises(ValueError,
match=r"runtimes_per_observation_count must be >= 1"):
env.runtime_observation_count = -1
def test_send_param_invalid_reply_count(env: LlvmEnv, mocker):
"""Test that an error is raised when # replies != # params."""
env.reset()
mocker.patch.object(env, "service")
with pytest.raises(
OSError,
match="Sent 1 parameter but received 0 responses from the service"
):
env.send_param("param", "")
def test_timeout_expired(env: LlvmEnv, tmpdir):
tmpdir = Path(tmpdir)
env.reset(benchmark="cbench-v1/jpeg-c") # larger benchmark
env.write_bitcode(tmpdir / "ir.bc")
space = env.observation.spaces["Programl"]
with pytest.raises(
TimeoutError,
match="Failed to compute Programl observation in 0.1 seconds"):
compute_observation(space, tmpdir / "ir.bc", timeout=0.1)
def test_Counter_step(env: LlvmEnv):
with Counter(env) as env:
env.reset()
env.step(0)
assert env.counters == {
"close": 0,
"fork": 0,
"reset": 1,
"step": 1,
}
def test_step_session_id_not_found(env: LlvmEnv):
"""Test that step() recovers gracefully from an unknown session error from
the service."""
env._session_id = 15 # pylint: disable=protected-access
observation, reward, done, info = env.step(0)
assert done
assert info["error_details"] == "Session not found: 15"
assert observation is None
assert reward is None
assert not env.in_episode
def test_csmith_random_select(env: LlvmEnv, csmith_dataset: CsmithDataset,
index: int, tmpwd: Path):
uri = next(islice(csmith_dataset.benchmark_uris(), index, None))
benchmark = csmith_dataset.benchmark(uri)
assert isinstance(benchmark, CsmithBenchmark)
env.reset(benchmark=benchmark)
assert benchmark.source
benchmark.write_sources_to_directory(tmpwd)
assert (tmpwd / "source.c").is_file()
def test_observation_programl_equivalence(env: LlvmEnv, tmpdir):
"""Test that compute_observation() produces the same result as the environment."""
tmpdir = Path(tmpdir)
env.reset()
env.write_bitcode(tmpdir / "ir.bc")
G = compute_observation(env.observation.spaces["Programl"],
tmpdir / "ir.bc")
networkx.algorithms.isomorphism.is_isomorphic(G,
env.observation.Programl())
def test_ValidateBenchmarkAfterEveryStep_valid(env: LlvmEnv):
env.reset()
type(env.benchmark).ivalidate = lambda *_: iter(())
env = ValidateBenchmarkAfterEveryStep(env, reward_penalty=-5)
_, reward, done, info = env.step(0)
assert reward != -5
assert not done
assert "error_details" not in info
def test_anghabench_random_select(
env: LlvmEnv, anghabench_dataset: AnghaBenchDataset, index: int, tmpwd: Path
):
uri = next(islice(anghabench_dataset.benchmark_uris(), index, None))
benchmark = anghabench_dataset.benchmark(uri)
env.reset(benchmark=benchmark)
assert benchmark.source
benchmark.write_sources_to_directory(tmpwd)
assert (tmpwd / "function.c").is_file()
def test_wrapped_env_in_episode(env: LlvmEnv, wrapper_type):
class MyWrapper(wrapper_type):
def convert_observation(self, observation):
return observation
env = MyWrapper(env)
assert not env.in_episode
env.reset()
assert env.in_episode
def test_ValidateBenchmarkAfterEveryStep_invalid(env: LlvmEnv, reward_penalty):
env.reset()
type(env.benchmark).ivalidate = lambda *_: iter(["Oh no!"])
env = ValidateBenchmarkAfterEveryStep(env, reward_penalty=reward_penalty)
_, reward, done, info = env.step(0)
assert reward == reward_penalty
assert done
assert info["error_details"] == "Oh no!"
