def test_inst2vec_observation_space( env: LlvmEnv, cbench_crc32_inst2vec_embedding_indices: List[int]): env.reset("cbench-v1/crc32") key = "Inst2vec" space = env.observation.spaces[key] assert isinstance(space.space, Sequence) value: np.ndarray = env.observation[key] print(value) # For debugging in case of error. assert isinstance(value, np.ndarray) assert value.dtype == np.float32 height, width = value.shape assert width == len(env.inst2vec.embeddings[0]) assert height == len(cbench_crc32_inst2vec_embedding_indices) # Check a handful of values. np.testing.assert_array_almost_equal( value.tolist(), [ env.inst2vec.embeddings[idx] for idx in cbench_crc32_inst2vec_embedding_indices ], ) assert space.deterministic assert not space.platform_dependent
def test_programl_json_observation_space(env: LlvmEnv): env.reset("cbench-v1/crc32") key = "ProgramlJson" space = env.observation.spaces[key] assert isinstance(space.space, Sequence) graph: Dict[str, Any] = env.observation[key] assert isinstance(graph, dict)
def test_instruction_count_reward_spaces(env: LlvmEnv): env.reset(benchmark="cBench-v0/crc32") key = "IrInstructionCount" space = env.reward.spaces[key] assert str(space) == "RewardSpaceSpec(IrInstructionCount)" assert env.reward[key] == 0 assert space.range == (-np.inf, np.inf) assert space.deterministic assert not space.platform_dependent assert space.success_threshold is None key = "IrInstructionCountO3" space = env.reward.spaces[key] assert str(space) == "RewardSpaceSpec(IrInstructionCountO3)" assert env.reward[key] == 0 assert space.range == (-np.inf, np.inf) assert space.deterministic assert not space.platform_dependent assert space.success_threshold == 1 key = "IrInstructionCountOz" space = env.reward.spaces[key] assert str(space) == "RewardSpaceSpec(IrInstructionCountOz)" assert env.reward[key] == 0 assert space.range == (-np.inf, np.inf) assert space.deterministic assert not space.platform_dependent assert space.success_threshold == 1
def test_add_derived_space(env: LlvmEnv): env.reset() with pytest.deprecated_call( match= "Use the derived_observation_spaces argument to CompilerEnv constructor." ): env.observation.add_derived_space( id="IrLen", base_id="Ir", space=Box(name="IrLen", low=0, high=float("inf"), shape=(1, ), dtype=int), translate=lambda base: [15], ) value = env.observation["IrLen"] assert isinstance(value, list) assert value == [15] # Repeat the above test using the generated bound method. value = env.observation.IrLen() assert isinstance(value, list) assert value == [15]
def test_observation_spaces(env: LlvmEnv): env.reset("cbench-v1/crc32") assert set(env.observation.spaces.keys()) == { "Ir", "IrSha1", "Bitcode", "BitcodeFile", "InstCount", "InstCountDict", "InstCountNorm", "InstCountNormDict", "Autophase", "AutophaseDict", "Programl", "ProgramlJson", "CpuInfo", "Inst2vecPreprocessedText", "Inst2vecEmbeddingIndices", "Inst2vec", "IrInstructionCount", "IrInstructionCountO0", "IrInstructionCountO3", "IrInstructionCountOz", "ObjectTextSizeBytes", "ObjectTextSizeO0", "ObjectTextSizeO3", "ObjectTextSizeOz", "Runtime", "Buildtime", "IsBuildable", "IsRunnable", }
def test_native_test_size_reward_spaces(env: LlvmEnv): env.reset(benchmark="cBench-v0/crc32") key = "ObjectTextSizeBytes" space = env.reward.spaces[key] assert str(space) == "RewardSpaceSpec(ObjectTextSizeBytes)" assert env.reward[key] == 0 assert space.range == (-np.inf, np.inf) assert space.deterministic assert space.platform_dependent assert space.success_threshold is None key = "ObjectTextSizeO3" space = env.reward.spaces[key] assert str(space) == "RewardSpaceSpec(ObjectTextSizeO3)" assert env.reward[key] == 0 assert space.range == (-np.inf, np.inf) assert space.deterministic assert space.platform_dependent assert space.success_threshold == 1 key = "ObjectTextSizeOz" space = env.reward.spaces[key] assert str(space) == "RewardSpaceSpec(ObjectTextSizeOz)" assert env.reward[key] == 0 assert space.range == (-np.inf, np.inf) assert space.deterministic assert space.platform_dependent assert space.success_threshold == 1
def test_cpuinfo_observation_space(env: LlvmEnv): env.reset("cbench-v1/crc32") key = "CpuInfo" space = env.observation.spaces[key] assert isinstance(space.space, DictSpace) value: Dict[str, Any] = env.observation[key] print(value) # For debugging in case of error. assert isinstance(value, dict) # Test each expected key, removing it as we go. assert isinstance(value.pop("name"), str) assert isinstance(value.pop("cores_count"), int) assert isinstance(value.pop("l1i_cache_size"), int) assert isinstance(value.pop("l1i_cache_count"), int) assert isinstance(value.pop("l1d_cache_size"), int) assert isinstance(value.pop("l1d_cache_count"), int) assert isinstance(value.pop("l2_cache_size"), int) assert isinstance(value.pop("l2_cache_count"), int) assert isinstance(value.pop("l3_cache_size"), int) assert isinstance(value.pop("l3_cache_count"), int) assert isinstance(value.pop("l4_cache_size"), int) assert isinstance(value.pop("l4_cache_count"), int) # Anything left in the JSON dictionary now is an unexpected key. assert not value invalid = "invalid value" with pytest.raises(KeyError) as ctx: _ = env.observation[invalid] assert str(ctx.value) == f"'{invalid}'" assert space.deterministic assert space.platform_dependent
def test_apply_state(env: LlvmEnv): """Test that apply() on a clean environment produces same state.""" env.reward_space = "IrInstructionCount" env.reset(benchmark="cbench-v1/crc32") env.step(env.action_space.flags.index("-mem2reg")) with gym.make("llvm-v0", reward_space="IrInstructionCount") as other: other.apply(env.state) assert other.state == env.state
def test_change_benchmark_mid_episode(env: LlvmEnv): """Test that changing the benchmark while in an episode has no effect until the next call to reset().""" env.reset(benchmark="benchmark://cBench-v0/crc32") assert env.benchmark == "benchmark://cBench-v0/crc32" env.benchmark = "benchmark://cBench-v0/dijkstra" assert env.benchmark == "benchmark://cBench-v0/crc32" env.reset() assert env.benchmark == "benchmark://cBench-v0/dijkstra"
def test_state_to_csv_from_csv(env: LlvmEnv): env.reset(benchmark="cBench-v0/crc32") env.episode_reward = 10 state = env.state assert state.reward == 10 state_from_csv = CompilerEnvState.from_csv(env.state.to_csv()) assert state_from_csv.reward == 10 assert state == state_from_csv
def test_buildtime_observation_space_not_runnable(env: LlvmEnv): env.reset("chstone-v0/gsm") key = "Buildtime" space = env.observation.spaces[key] assert isinstance(space.space, Sequence) assert not space.deterministic assert space.platform_dependent assert env.observation[key] is None
def test_runtime_observation_space_invalid_observation_count(env: LlvmEnv): """Test setting an invalid custom observation count for LLVM runtimes.""" env.reset("cbench-v1/crc32") val = env.runtime_observation_count with pytest.raises( ValueError, match="runtimes_per_observation_count must be >= 1. Received: -5"): env.runtime_observation_count = -5 assert env.runtime_observation_count == val # unchanged
def test_step_multiple_actions_list(env: LlvmEnv): """Pass a list of actions to step().""" env.reset(benchmark="cbench-v1/crc32") actions = [ env.action_space.flags.index("-mem2reg"), env.action_space.flags.index("-reg2mem"), ] _, _, done, _ = env.multistep(actions) assert not done assert env.actions == actions
def test_autophase_dict_observation_space(env: LlvmEnv): env.reset("cBench-v0/crc32") key = "AutophaseDict" space = env.observation.spaces[key] assert isinstance(space.space, DictSpace) value: Dict[str, int] = env.observation[key] assert len(value) == 56 assert space.deterministic assert not space.platform_dependent
def test_eager_observation_space(env: LlvmEnv): env.observation_space = "Autophase" assert env.observation_space.id == "Autophase" env.observation_space = None assert env.observation_space is None invalid = "invalid value" with pytest.raises(LookupError) as ctx: env.observation_space = invalid assert str(ctx.value) == f"Observation space not found: {invalid}"
def test_ir_sha1(env: LlvmEnv, tmpwd: Path): env.reset(benchmark="cbench-v1/crc32") before = env.ir_sha1 _, _, done, info = env.step(env.action_space.flags.index("-mem2reg")) assert not done, info assert not info[ "action_had_no_effect"], "sanity check failed, action had no effect" after = env.ir_sha1 assert before != after
def test_runtime_observation_space_not_runnable(env: LlvmEnv): env.reset("chstone-v0/gsm") key = "Runtime" space = env.observation.spaces[key] assert isinstance(space.space, Sequence) value: np.ndarray = env.observation[key] print(value.tolist()) # For debugging in case of error. assert isinstance(value, np.ndarray) assert value.shape == (0, ) assert space.space.contains(value)
def test_is_runnable_observation_space(env: LlvmEnv): env.reset("cbench-v1/crc32") key = "IsRunnable" space = env.observation.spaces[key] assert isinstance(space.space, Scalar) assert space.deterministic assert space.platform_dependent value: int = env.observation[key] print(value) # For debugging in case of error. assert isinstance(value, int) assert value == 1
def test_instcount_norm_dict_observation_space(env: LlvmEnv): env.reset("cbench-v1/crc32") key = "InstCountNormDict" space = env.observation.spaces[key] assert isinstance(space.space, DictSpace) assert space.deterministic assert not space.platform_dependent value: Dict[str, int] = env.observation[key] print(value) # For debugging in case of error. assert len(value) == 69
def test_is_buildable_observation_space_not_buildable(env: LlvmEnv): env.reset("chstone-v0/gsm") key = "IsBuildable" space = env.observation.spaces[key] assert isinstance(space.space, Scalar) assert space.deterministic assert space.platform_dependent value: int = env.observation[key] print(value) # For debugging in case of error. assert isinstance(value, int) assert value == 0
def test_reward_space(env: LlvmEnv): env.reward_space = "IrInstructionCount" assert env.reward_space.id == "IrInstructionCount" env.reward_space = None assert env.reward_space is None invalid = "invalid value" with pytest.raises(LookupError) as ctx: env.reward_space = invalid assert str(ctx.value) == f"Reward space not found: {invalid}"
def test_reward_spaces(env: LlvmEnv): env.reset(benchmark="cBench-v0/crc32") assert set(env.reward.spaces.keys()) == { "IrInstructionCount", "IrInstructionCountO3", "IrInstructionCountOz", "ObjectTextSizeBytes", "ObjectTextSizeO3", "ObjectTextSizeOz", }
def test_step_multiple_actions_generator(env: LlvmEnv): """Pass an iterable of actions to step().""" env.reset(benchmark="cbench-v1/crc32") actions = ( env.action_space.flags.index("-mem2reg"), env.action_space.flags.index("-reg2mem"), ) _, _, done, _ = env.step(actions) assert not done assert env.actions == [ env.action_space.flags.index("-mem2reg"), env.action_space.flags.index("-reg2mem"), ]
def test_buildtime_observation_space_not_runnable(env: LlvmEnv): env.reset("chstone-v0/gsm") key = "Buildtime" space = env.observation.spaces[key] assert isinstance(space.space, Sequence) assert not space.deterministic assert space.platform_dependent value: np.ndarray = env.observation[key] print(value) # For debugging in case of error. assert value.shape == (0, ) assert space.space.contains(value)
def test_connection_dies_default_reward_negated(env: LlvmEnv): env.reward_space = "IrInstructionCount" env.reset(benchmark="cBench-v0/crc32") env.reward_space.default_negates_returns = True env.reward_space.default_value = 2.5 env.episode_reward = 10 env.service.close() observation, reward, done, _ = env.step(0) assert done assert reward == -7.5 # negates reward.
def test_default_observation_space(env: LlvmEnv): env.observation_space = "Autophase" assert env.observation_space.shape == (56, ) assert env.observation_space_spec.id == "Autophase" env.observation_space = None assert env.observation_space is None assert env.observation_space_spec is None invalid = "invalid value" with pytest.raises(LookupError, match=f"Observation space not found: {invalid}"): env.observation_space = invalid
def test_state_serialize_deserialize_equality(env: LlvmEnv): env.reset(benchmark="cbench-v1/crc32") env.episode_reward = 10 state = env.state assert state.reward == 10 buf = StringIO() CompilerEnvStateWriter(buf).write_state(state) buf.seek(0) # Rewind the buffer for reading. state_from_csv = next(iter(CompilerEnvStateReader(buf))) assert state_from_csv.reward == 10 assert state == state_from_csv
def test_ir_observation_space(env: LlvmEnv): env.reset("cBench-v0/crc32") key = "Ir" space = env.observation.spaces[key] assert isinstance(space.space, Sequence) assert space.space.dtype == str assert space.space.size_range == (0, None) value: str = env.observation[key] assert isinstance(value, str) assert space.space.contains(value) assert space.deterministic assert not space.platform_dependent
def test_bitcode_file_equivalence(env: LlvmEnv, benchmark_uri: str): """Test that LLVM produces the same bitcode as a file and as a byte array.""" env.reset(benchmark=benchmark_uri) bitcode = env.observation.Bitcode() bitcode_file = env.observation.BitcodeFile() try: with open(bitcode_file, "rb") as f: bitcode_from_file = f.read() assert bitcode.tobytes() == bitcode_from_file finally: os.unlink(bitcode_file)
def test_ir_observation_space(env: LlvmEnv): env.reset("cbench-v1/crc32") key = "Ir" space = env.observation.spaces[key] assert isinstance(space.space, Sequence) assert space.space.dtype == str assert space.space.size_range == (0, np.iinfo(np.int64).max) value: str = env.observation[key] print(value) # For debugging in case of error. assert isinstance(value, str) assert space.space.contains(value) assert space.deterministic assert not space.platform_dependent