def test_inst2vec_observation_space(
env: LlvmEnv, cbench_crc32_inst2vec_embedding_indices: List[int]):
env.reset("cbench-v1/crc32")
key = "Inst2vec"
space = env.observation.spaces[key]
assert isinstance(space.space, Sequence)
value: np.ndarray = env.observation[key]
print(value) # For debugging in case of error.
assert isinstance(value, np.ndarray)
assert value.dtype == np.float32
height, width = value.shape
assert width == len(env.inst2vec.embeddings[0])
assert height == len(cbench_crc32_inst2vec_embedding_indices)
# Check a handful of values.
np.testing.assert_array_almost_equal(
value.tolist(),
[
env.inst2vec.embeddings[idx]
for idx in cbench_crc32_inst2vec_embedding_indices
],
)
assert space.deterministic
assert not space.platform_dependent
def test_programl_json_observation_space(env: LlvmEnv):
env.reset("cbench-v1/crc32")
key = "ProgramlJson"
space = env.observation.spaces[key]
assert isinstance(space.space, Sequence)
graph: Dict[str, Any] = env.observation[key]
assert isinstance(graph, dict)
def test_instruction_count_reward_spaces(env: LlvmEnv):
env.reset(benchmark="cBench-v0/crc32")
key = "IrInstructionCount"
space = env.reward.spaces[key]
assert str(space) == "RewardSpaceSpec(IrInstructionCount)"
assert env.reward[key] == 0
assert space.range == (-np.inf, np.inf)
assert space.deterministic
assert not space.platform_dependent
assert space.success_threshold is None
key = "IrInstructionCountO3"
space = env.reward.spaces[key]
assert str(space) == "RewardSpaceSpec(IrInstructionCountO3)"
assert env.reward[key] == 0
assert space.range == (-np.inf, np.inf)
assert space.deterministic
assert not space.platform_dependent
assert space.success_threshold == 1
key = "IrInstructionCountOz"
space = env.reward.spaces[key]
assert str(space) == "RewardSpaceSpec(IrInstructionCountOz)"
assert env.reward[key] == 0
assert space.range == (-np.inf, np.inf)
assert space.deterministic
assert not space.platform_dependent
assert space.success_threshold == 1
def test_add_derived_space(env: LlvmEnv):
env.reset()
with pytest.deprecated_call(
match=
"Use the derived_observation_spaces argument to CompilerEnv constructor."
):
env.observation.add_derived_space(
id="IrLen",
base_id="Ir",
space=Box(name="IrLen",
low=0,
high=float("inf"),
shape=(1, ),
dtype=int),
translate=lambda base: [15],
)
value = env.observation["IrLen"]
assert isinstance(value, list)
assert value == [15]
# Repeat the above test using the generated bound method.
value = env.observation.IrLen()
assert isinstance(value, list)
assert value == [15]
def test_observation_spaces(env: LlvmEnv):
env.reset("cbench-v1/crc32")
assert set(env.observation.spaces.keys()) == {
"Ir",
"IrSha1",
"Bitcode",
"BitcodeFile",
"InstCount",
"InstCountDict",
"InstCountNorm",
"InstCountNormDict",
"Autophase",
"AutophaseDict",
"Programl",
"ProgramlJson",
"CpuInfo",
"Inst2vecPreprocessedText",
"Inst2vecEmbeddingIndices",
"Inst2vec",
"IrInstructionCount",
"IrInstructionCountO0",
"IrInstructionCountO3",
"IrInstructionCountOz",
"ObjectTextSizeBytes",
"ObjectTextSizeO0",
"ObjectTextSizeO3",
"ObjectTextSizeOz",
"Runtime",
"Buildtime",
"IsBuildable",
"IsRunnable",
}
def test_native_test_size_reward_spaces(env: LlvmEnv):
env.reset(benchmark="cBench-v0/crc32")
key = "ObjectTextSizeBytes"
space = env.reward.spaces[key]
assert str(space) == "RewardSpaceSpec(ObjectTextSizeBytes)"
assert env.reward[key] == 0
assert space.range == (-np.inf, np.inf)
assert space.deterministic
assert space.platform_dependent
assert space.success_threshold is None
key = "ObjectTextSizeO3"
space = env.reward.spaces[key]
assert str(space) == "RewardSpaceSpec(ObjectTextSizeO3)"
assert env.reward[key] == 0
assert space.range == (-np.inf, np.inf)
assert space.deterministic
assert space.platform_dependent
assert space.success_threshold == 1
key = "ObjectTextSizeOz"
space = env.reward.spaces[key]
assert str(space) == "RewardSpaceSpec(ObjectTextSizeOz)"
assert env.reward[key] == 0
assert space.range == (-np.inf, np.inf)
assert space.deterministic
assert space.platform_dependent
assert space.success_threshold == 1
def test_cpuinfo_observation_space(env: LlvmEnv):
env.reset("cbench-v1/crc32")
key = "CpuInfo"
space = env.observation.spaces[key]
assert isinstance(space.space, DictSpace)
value: Dict[str, Any] = env.observation[key]
print(value) # For debugging in case of error.
assert isinstance(value, dict)
# Test each expected key, removing it as we go.
assert isinstance(value.pop("name"), str)
assert isinstance(value.pop("cores_count"), int)
assert isinstance(value.pop("l1i_cache_size"), int)
assert isinstance(value.pop("l1i_cache_count"), int)
assert isinstance(value.pop("l1d_cache_size"), int)
assert isinstance(value.pop("l1d_cache_count"), int)
assert isinstance(value.pop("l2_cache_size"), int)
assert isinstance(value.pop("l2_cache_count"), int)
assert isinstance(value.pop("l3_cache_size"), int)
assert isinstance(value.pop("l3_cache_count"), int)
assert isinstance(value.pop("l4_cache_size"), int)
assert isinstance(value.pop("l4_cache_count"), int)
# Anything left in the JSON dictionary now is an unexpected key.
assert not value
invalid = "invalid value"
with pytest.raises(KeyError) as ctx:
_ = env.observation[invalid]
assert str(ctx.value) == f"'{invalid}'"
assert space.deterministic
assert space.platform_dependent
def test_apply_state(env: LlvmEnv):
"""Test that apply() on a clean environment produces same state."""
env.reward_space = "IrInstructionCount"
env.reset(benchmark="cbench-v1/crc32")
env.step(env.action_space.flags.index("-mem2reg"))
with gym.make("llvm-v0", reward_space="IrInstructionCount") as other:
other.apply(env.state)
assert other.state == env.state
def test_change_benchmark_mid_episode(env: LlvmEnv):
"""Test that changing the benchmark while in an episode has no effect until
the next call to reset()."""
env.reset(benchmark="benchmark://cBench-v0/crc32")
assert env.benchmark == "benchmark://cBench-v0/crc32"
env.benchmark = "benchmark://cBench-v0/dijkstra"
assert env.benchmark == "benchmark://cBench-v0/crc32"
env.reset()
assert env.benchmark == "benchmark://cBench-v0/dijkstra"
def test_state_to_csv_from_csv(env: LlvmEnv):
env.reset(benchmark="cBench-v0/crc32")
env.episode_reward = 10
state = env.state
assert state.reward == 10
state_from_csv = CompilerEnvState.from_csv(env.state.to_csv())
assert state_from_csv.reward == 10
assert state == state_from_csv
def test_buildtime_observation_space_not_runnable(env: LlvmEnv):
env.reset("chstone-v0/gsm")
key = "Buildtime"
space = env.observation.spaces[key]
assert isinstance(space.space, Sequence)
assert not space.deterministic
assert space.platform_dependent
assert env.observation[key] is None
def test_runtime_observation_space_invalid_observation_count(env: LlvmEnv):
"""Test setting an invalid custom observation count for LLVM runtimes."""
env.reset("cbench-v1/crc32")
val = env.runtime_observation_count
with pytest.raises(
ValueError,
match="runtimes_per_observation_count must be >= 1. Received: -5"):
env.runtime_observation_count = -5
assert env.runtime_observation_count == val # unchanged
def test_step_multiple_actions_list(env: LlvmEnv):
"""Pass a list of actions to step()."""
env.reset(benchmark="cbench-v1/crc32")
actions = [
env.action_space.flags.index("-mem2reg"),
env.action_space.flags.index("-reg2mem"),
]
_, _, done, _ = env.multistep(actions)
assert not done
assert env.actions == actions
def test_autophase_dict_observation_space(env: LlvmEnv):
env.reset("cBench-v0/crc32")
key = "AutophaseDict"
space = env.observation.spaces[key]
assert isinstance(space.space, DictSpace)
value: Dict[str, int] = env.observation[key]
assert len(value) == 56
assert space.deterministic
assert not space.platform_dependent
def test_eager_observation_space(env: LlvmEnv):
env.observation_space = "Autophase"
assert env.observation_space.id == "Autophase"
env.observation_space = None
assert env.observation_space is None
invalid = "invalid value"
with pytest.raises(LookupError) as ctx:
env.observation_space = invalid
assert str(ctx.value) == f"Observation space not found: {invalid}"
def test_ir_sha1(env: LlvmEnv, tmpwd: Path):
env.reset(benchmark="cbench-v1/crc32")
before = env.ir_sha1
_, _, done, info = env.step(env.action_space.flags.index("-mem2reg"))
assert not done, info
assert not info[
"action_had_no_effect"], "sanity check failed, action had no effect"
after = env.ir_sha1
assert before != after
def test_runtime_observation_space_not_runnable(env: LlvmEnv):
env.reset("chstone-v0/gsm")
key = "Runtime"
space = env.observation.spaces[key]
assert isinstance(space.space, Sequence)
value: np.ndarray = env.observation[key]
print(value.tolist()) # For debugging in case of error.
assert isinstance(value, np.ndarray)
assert value.shape == (0, )
assert space.space.contains(value)
def test_is_runnable_observation_space(env: LlvmEnv):
env.reset("cbench-v1/crc32")
key = "IsRunnable"
space = env.observation.spaces[key]
assert isinstance(space.space, Scalar)
assert space.deterministic
assert space.platform_dependent
value: int = env.observation[key]
print(value) # For debugging in case of error.
assert isinstance(value, int)
assert value == 1
def test_instcount_norm_dict_observation_space(env: LlvmEnv):
env.reset("cbench-v1/crc32")
key = "InstCountNormDict"
space = env.observation.spaces[key]
assert isinstance(space.space, DictSpace)
assert space.deterministic
assert not space.platform_dependent
value: Dict[str, int] = env.observation[key]
print(value) # For debugging in case of error.
assert len(value) == 69
def test_is_buildable_observation_space_not_buildable(env: LlvmEnv):
env.reset("chstone-v0/gsm")
key = "IsBuildable"
space = env.observation.spaces[key]
assert isinstance(space.space, Scalar)
assert space.deterministic
assert space.platform_dependent
value: int = env.observation[key]
print(value) # For debugging in case of error.
assert isinstance(value, int)
assert value == 0
def test_reward_space(env: LlvmEnv):
env.reward_space = "IrInstructionCount"
assert env.reward_space.id == "IrInstructionCount"
env.reward_space = None
assert env.reward_space is None
invalid = "invalid value"
with pytest.raises(LookupError) as ctx:
env.reward_space = invalid
assert str(ctx.value) == f"Reward space not found: {invalid}"
def test_reward_spaces(env: LlvmEnv):
env.reset(benchmark="cBench-v0/crc32")
assert set(env.reward.spaces.keys()) == {
"IrInstructionCount",
"IrInstructionCountO3",
"IrInstructionCountOz",
"ObjectTextSizeBytes",
"ObjectTextSizeO3",
"ObjectTextSizeOz",
}
def test_step_multiple_actions_generator(env: LlvmEnv):
"""Pass an iterable of actions to step()."""
env.reset(benchmark="cbench-v1/crc32")
actions = (
env.action_space.flags.index("-mem2reg"),
env.action_space.flags.index("-reg2mem"),
)
_, _, done, _ = env.step(actions)
assert not done
assert env.actions == [
env.action_space.flags.index("-mem2reg"),
env.action_space.flags.index("-reg2mem"),
]
def test_buildtime_observation_space_not_runnable(env: LlvmEnv):
env.reset("chstone-v0/gsm")
key = "Buildtime"
space = env.observation.spaces[key]
assert isinstance(space.space, Sequence)
assert not space.deterministic
assert space.platform_dependent
value: np.ndarray = env.observation[key]
print(value) # For debugging in case of error.
assert value.shape == (0, )
assert space.space.contains(value)
def test_connection_dies_default_reward_negated(env: LlvmEnv):
env.reward_space = "IrInstructionCount"
env.reset(benchmark="cBench-v0/crc32")
env.reward_space.default_negates_returns = True
env.reward_space.default_value = 2.5
env.episode_reward = 10
env.service.close()
observation, reward, done, _ = env.step(0)
assert done
assert reward == -7.5 # negates reward.
def test_default_observation_space(env: LlvmEnv):
env.observation_space = "Autophase"
assert env.observation_space.shape == (56, )
assert env.observation_space_spec.id == "Autophase"
env.observation_space = None
assert env.observation_space is None
assert env.observation_space_spec is None
invalid = "invalid value"
with pytest.raises(LookupError,
match=f"Observation space not found: {invalid}"):
env.observation_space = invalid
def test_state_serialize_deserialize_equality(env: LlvmEnv):
env.reset(benchmark="cbench-v1/crc32")
env.episode_reward = 10
state = env.state
assert state.reward == 10
buf = StringIO()
CompilerEnvStateWriter(buf).write_state(state)
buf.seek(0) # Rewind the buffer for reading.
state_from_csv = next(iter(CompilerEnvStateReader(buf)))
assert state_from_csv.reward == 10
assert state == state_from_csv
def test_ir_observation_space(env: LlvmEnv):
env.reset("cBench-v0/crc32")
key = "Ir"
space = env.observation.spaces[key]
assert isinstance(space.space, Sequence)
assert space.space.dtype == str
assert space.space.size_range == (0, None)
value: str = env.observation[key]
assert isinstance(value, str)
assert space.space.contains(value)
assert space.deterministic
assert not space.platform_dependent
def test_bitcode_file_equivalence(env: LlvmEnv, benchmark_uri: str):
"""Test that LLVM produces the same bitcode as a file and as a byte array."""
env.reset(benchmark=benchmark_uri)
bitcode = env.observation.Bitcode()
bitcode_file = env.observation.BitcodeFile()
try:
with open(bitcode_file, "rb") as f:
bitcode_from_file = f.read()
assert bitcode.tobytes() == bitcode_from_file
finally:
os.unlink(bitcode_file)
def test_ir_observation_space(env: LlvmEnv):
env.reset("cbench-v1/crc32")
key = "Ir"
space = env.observation.spaces[key]
assert isinstance(space.space, Sequence)
assert space.space.dtype == str
assert space.space.size_range == (0, np.iinfo(np.int64).max)
value: str = env.observation[key]
print(value) # For debugging in case of error.
assert isinstance(value, str)
assert space.space.contains(value)
assert space.deterministic
assert not space.platform_dependent
