def test_wrapped_step_multi_step(env: LlvmEnv): env = TimeLimit(env, max_episode_steps=5) env.reset(benchmark="benchmark://cbench-v1/dijkstra") env.step([0, 0, 0]) assert env.benchmark == "benchmark://cbench-v1/dijkstra" assert env.actions == [0, 0, 0]
def test_invalid_runtime_count(env: LlvmEnv): env = RuntimePointEstimateReward(env, runtime_count=-10) with pytest.raises( ValueError, match="runtimes_per_observation_count must be >= 1. Received: -10" ): env.reset()
def test_wrapped_step_multi_step(env: LlvmEnv): """Test passing a list of actions to step().""" env = CompilerEnvWrapper(env) env.reset() env.multistep([0, 0, 0]) assert env.actions == [0, 0, 0]
def test_deep_copy(env: LlvmEnv): """Test that deep copy creates an independent copy.""" env.reset() with deepcopy(env) as cpy: assert cpy.state == env.state env.step(env.action_space.sample()) assert cpy.state != env.state
def test_invalid_warmup_count(env: LlvmEnv): env = RuntimePointEstimateReward(env, warmup_count=-10) with pytest.raises( ValueError, match="warmup_runs_count_per_runtime_observation must be >= 0. Received: -10", ): env.reset()
def test_reward_range_not_runnable_benchmark(env: LlvmEnv): env = RuntimePointEstimateReward(env, runtime_count=3) with pytest.raises( BenchmarkInitError, match=r"^Benchmark is not runnable: benchmark://npb-v0/1$" ): env.reset(benchmark="benchmark://npb-v0/1")
def test_csmith_positive_buildtimes(env: LlvmEnv, csmith_dataset: CsmithDataset): benchmark = next(csmith_dataset.benchmarks()) env.reset(benchmark=benchmark) val = env.observation["Buildtime"] print(val.tolist()) assert np.all(np.greater(val, 0))
def test_reset_invalid_ir(env: LlvmEnv): """Test that setting the $CXX to an invalid binary raises an error.""" benchmark = llvm.make_benchmark(INVALID_IR_PATH) with pytest.raises(BenchmarkInitError, match="Failed to compute .text size cost"): env.reset(benchmark=benchmark)
def test_benchmark_constructor_arg(env: LlvmEnv): env.close() # Fixture only required to pull in dataset. with gym.make("llvm-v0", benchmark="cbench-v1/dijkstra") as env: assert env.benchmark == "benchmark://cbench-v1/dijkstra" env.reset() assert env.benchmark == "benchmark://cbench-v1/dijkstra"
def test_failing_build_cmd(env: LlvmEnv, tmpdir): """Test that Runtime observation raises an error if build command fails.""" with open(tmpdir / "program.c", "w") as f: f.write(""" #include <stdio.h> int main(int argc, char** argv) { printf("Hello\\n"); for (int i = 0; i < 10; ++i) { argc += 2; } return argc - argc; } """) benchmark = env.make_benchmark(Path(tmpdir) / "program.c") benchmark.proto.dynamic_config.build_cmd.argument.extend( ["$CC", "$IN", "-invalid-cc-argument"]) benchmark.proto.dynamic_config.build_cmd.outfile.extend(["a.out"]) benchmark.proto.dynamic_config.build_cmd.timeout_seconds = 10 benchmark.proto.dynamic_config.run_cmd.argument.extend(["./a.out"]) benchmark.proto.dynamic_config.run_cmd.timeout_seconds = 10 env.reset(benchmark=benchmark) with pytest.raises( ServiceError, match= (r"Command '\$CC \$IN -invalid-cc-argument' failed with exit code 1: " r"clang: error: unknown argument: '-invalid-cc-argument'"), ): env.observation.Runtime()
def test_wrapped_step_custom_args(env: LlvmEnv, wrapper_type): """Test passing the custom CompilerGym step() keyword arguments.""" class MyWrapper(wrapper_type): def convert_observation(self, observation): return observation # pass thru def action(self, action): return action # pass thru def convert_reward(self, reward): return reward env = MyWrapper(env) env.reset() (ir, ic), (icr, icroz), _, _ = env.multistep( actions=[0, 0, 0], observation_spaces=["Ir", "IrInstructionCount"], reward_spaces=["IrInstructionCount", "IrInstructionCountOz"], ) assert isinstance(ir, str) assert isinstance(ic, int) assert isinstance(icr, float) assert isinstance(icroz, float) assert env.unwrapped.observation.spaces["Ir"].space.contains(ir) assert env.unwrapped.observation.spaces[ "IrInstructionCount"].space.contains(ic)
def test_env_spec_make_workaround(env: LlvmEnv): """Demonstrate how #587 would be fixed, by updating the 'kwargs' dict.""" env.reset(benchmark="cbench-v1/bitcount") env.spec._kwargs[ # pylint: disable=protected-access "benchmark" ] = "cbench-v1/bitcount" with env.spec.make() as new_env: assert new_env.benchmark == env.benchmark
def test_env_spec_make(env: LlvmEnv): """Test that demonstrates a failure in gym compatibility: env.spec does not encode mutable state like benchmark, reward space, and observation space. """ env.reset(benchmark="cbench-v1/bitcount") with env.spec.make() as new_env: assert new_env.benchmark == env.benchmark
def test_no_module_id_builtin_benchmark(env: LlvmEnv): """Test that the module and source IDs are stripped in shipped benchmark.""" env.reset("cbench-v1/crc32") ir = env.ir print(ir) # For debugging in case of error. assert "; ModuleID = '-'\n" in ir assert '\nsource_filename = "-"\n' in ir
def test_clgen_random_select(env: LlvmEnv, clgen_dataset: CLgenDataset, index: int, tmpwd: Path): uri = next(islice(clgen_dataset.benchmark_uris(), index, None)) benchmark = clgen_dataset.benchmark(uri) env.reset(benchmark=benchmark) assert benchmark.source benchmark.write_sources_to_directory(tmpwd) assert (tmpwd / "kernel.cl").is_file()
def test_reset_to_force_benchmark(env: LlvmEnv): """Test that calling reset() with a benchmark forces that benchmark to be used for every subsequent episode. """ env.reset(benchmark="benchmark://cbench-v1/crc32") assert env.benchmark == "benchmark://cbench-v1/crc32" for _ in range(10): env.reset() assert env.benchmark == "benchmark://cbench-v1/crc32"
def test_poj104_random_select(env: LlvmEnv, poj104_dataset: POJ104Dataset, index: int, tmpwd: Path): uri = next(islice(poj104_dataset.benchmark_uris(), index, None)) benchmark = poj104_dataset.benchmark(uri) env.reset(benchmark=benchmark) assert benchmark.source benchmark.write_sources_to_directory(tmpwd) assert (tmpwd / "source.cc").is_file()
def test_observation_equivalence(env: LlvmEnv, tmpdir, observation_space: str): """Test that compute_observation() produces the same result as the environment.""" tmpdir = Path(tmpdir) env.reset() env.write_bitcode(tmpdir / "ir.bc") observation = compute_observation( env.observation.spaces[observation_space], tmpdir / "ir.bc") assert observation == env.observation[observation_space]
def test_default_runtime_observation_count_fork(env: LlvmEnv): """Test that default property values propagate on fork().""" env.reset() rc = env.runtime_observation_count wc = env.runtime_warmup_runs_count with env.fork() as fkd: assert fkd.runtime_observation_count == rc assert fkd.runtime_warmup_runs_count == wc
def test_invalid_runtime_count(env: LlvmEnv): env.reset() with pytest.raises(ValueError, match=r"runtimes_per_observation_count must be >= 1"): env.runtime_observation_count = 0 with pytest.raises(ValueError, match=r"runtimes_per_observation_count must be >= 1"): env.runtime_observation_count = -1
def test_send_param_invalid_reply_count(env: LlvmEnv, mocker): """Test that an error is raised when # replies != # params.""" env.reset() mocker.patch.object(env, "service") with pytest.raises( OSError, match="Sent 1 parameter but received 0 responses from the service" ): env.send_param("param", "")
def test_timeout_expired(env: LlvmEnv, tmpdir): tmpdir = Path(tmpdir) env.reset(benchmark="cbench-v1/jpeg-c") # larger benchmark env.write_bitcode(tmpdir / "ir.bc") space = env.observation.spaces["Programl"] with pytest.raises( TimeoutError, match="Failed to compute Programl observation in 0.1 seconds"): compute_observation(space, tmpdir / "ir.bc", timeout=0.1)
def test_Counter_step(env: LlvmEnv): with Counter(env) as env: env.reset() env.step(0) assert env.counters == { "close": 0, "fork": 0, "reset": 1, "step": 1, }
def test_step_session_id_not_found(env: LlvmEnv): """Test that step() recovers gracefully from an unknown session error from the service.""" env._session_id = 15 # pylint: disable=protected-access observation, reward, done, info = env.step(0) assert done assert info["error_details"] == "Session not found: 15" assert observation is None assert reward is None assert not env.in_episode
def test_csmith_random_select(env: LlvmEnv, csmith_dataset: CsmithDataset, index: int, tmpwd: Path): uri = next(islice(csmith_dataset.benchmark_uris(), index, None)) benchmark = csmith_dataset.benchmark(uri) assert isinstance(benchmark, CsmithBenchmark) env.reset(benchmark=benchmark) assert benchmark.source benchmark.write_sources_to_directory(tmpwd) assert (tmpwd / "source.c").is_file()
def test_observation_programl_equivalence(env: LlvmEnv, tmpdir): """Test that compute_observation() produces the same result as the environment.""" tmpdir = Path(tmpdir) env.reset() env.write_bitcode(tmpdir / "ir.bc") G = compute_observation(env.observation.spaces["Programl"], tmpdir / "ir.bc") networkx.algorithms.isomorphism.is_isomorphic(G, env.observation.Programl())
def test_ValidateBenchmarkAfterEveryStep_valid(env: LlvmEnv): env.reset() type(env.benchmark).ivalidate = lambda *_: iter(()) env = ValidateBenchmarkAfterEveryStep(env, reward_penalty=-5) _, reward, done, info = env.step(0) assert reward != -5 assert not done assert "error_details" not in info
def test_anghabench_random_select( env: LlvmEnv, anghabench_dataset: AnghaBenchDataset, index: int, tmpwd: Path ): uri = next(islice(anghabench_dataset.benchmark_uris(), index, None)) benchmark = anghabench_dataset.benchmark(uri) env.reset(benchmark=benchmark) assert benchmark.source benchmark.write_sources_to_directory(tmpwd) assert (tmpwd / "function.c").is_file()
def test_wrapped_env_in_episode(env: LlvmEnv, wrapper_type): class MyWrapper(wrapper_type): def convert_observation(self, observation): return observation env = MyWrapper(env) assert not env.in_episode env.reset() assert env.in_episode
def test_ValidateBenchmarkAfterEveryStep_invalid(env: LlvmEnv, reward_penalty): env.reset() type(env.benchmark).ivalidate = lambda *_: iter(["Oh no!"]) env = ValidateBenchmarkAfterEveryStep(env, reward_penalty=reward_penalty) _, reward, done, info = env.step(0) assert reward == reward_penalty assert done assert info["error_details"] == "Oh no!"