def test_cpuinfo_observation_space(env: CompilerEnv):
env.reset("cBench-v0/crc32")
space = "CpuInfo"
assert isinstance(env.observation.spaces[space], Sequence)
value: Dict[str, Any] = env.observation[space]
assert isinstance(value, dict)
# Test each expected key, removing it as we go.
assert isinstance(value.pop("name"), str)
assert isinstance(value.pop("cores_count"), int)
assert isinstance(value.pop("l1i_cache_size"), int)
assert isinstance(value.pop("l1i_cache_count"), int)
assert isinstance(value.pop("l1d_cache_size"), int)
assert isinstance(value.pop("l1d_cache_count"), int)
assert isinstance(value.pop("l2_cache_size"), int)
assert isinstance(value.pop("l2_cache_count"), int)
assert isinstance(value.pop("l3_cache_size"), int)
assert isinstance(value.pop("l3_cache_count"), int)
assert isinstance(value.pop("l4_cache_size"), int)
assert isinstance(value.pop("l4_cache_count"), int)
# Anything left in the JSON dictionary now is an unexpected key.
assert not value
invalid = "invalid value"
with pytest.raises(KeyError) as ctx:
_ = env.observation[invalid]
assert str(ctx.value) == f"'{invalid}'"
space = "CpuInfoDict"
value: Dict[str, Any] = env.observation[space]
print(value)
def test_autophase_dict_observation_space(env: CompilerEnv):
env.reset("cBench-v0/crc32")
key = "AutophaseDict"
space = env.observation.spaces[key]
assert isinstance(space, DictSpace)
value: Dict[str, int] = env.observation[key]
assert len(value) == 56
def test_eager_reward(env: CompilerEnv):
env.eager_reward_space = "codesize"
env.reset()
observation, reward, done, info = env.step(0)
assert observation is None
assert reward == 0
assert not done
def test_service_env_dies_reset(env: CompilerEnv):
env.observation_space = "Autophase"
env.reward_space = "IrInstructionCount"
env.reset("cbench-v1/crc32")
# Kill the service. Note killing the service for a ManagedConnection will
# result in a ServiceError because we have not ended the session we started
# with env.reset() above. For UnmanagedConnection, this error will not be
# raised.
try:
env.service.close()
except ServiceError as e:
assert "Service exited with returncode " in str(e)
# Check that the environment doesn't fall over.
observation, reward, done, info = env.step(0)
assert done, info["error_details"]
assert not env.in_episode
# Check that default values are returned.
np.testing.assert_array_equal(observation, np.zeros(AUTOPHASE_FEATURE_DIM))
assert reward == 0
# Reset the environment and check that it works.
env.reset(benchmark="cbench-v1/crc32")
assert env.in_episode
observation, reward, done, info = env.step(0)
assert not done, info["error_details"]
assert observation is not None
assert reward is not None
def test_invalid_benchmark_missing_file(env: CompilerEnv):
benchmark = Benchmark(uri="benchmark://new", )
with pytest.raises(ValueError) as ctx:
env.reset(benchmark=benchmark)
assert str(ctx.value) == "No program set"
def test_reward_spaces(env: CompilerEnv):
env.reset("cBench-v0/crc32")
assert set(env.reward.ranges.keys()) == {
"IrInstructionCount",
"IrInstructionCountO3",
"IrInstructionCountOz",
"IrInstructionCountOzDiff",
}
reward_space = "IrInstructionCount"
assert env.reward.ranges[reward_space] == (-np.inf, 0)
assert env.reward[reward_space] < 0
reward_space = "IrInstructionCountO3"
assert env.reward.ranges[reward_space] == (0, np.inf)
assert env.reward[reward_space] > 0
reward_space = "IrInstructionCountOz"
assert env.reward.ranges[reward_space] == (0, np.inf)
assert env.reward[reward_space] > 0
reward_space = "IrInstructionCountOzDiff"
assert env.reward.ranges[reward_space] == (-np.inf, np.inf)
invalid = "invalid value"
with pytest.raises(KeyError) as ctx:
_ = env.reward[invalid]
assert str(ctx.value) == f"'{invalid}'"
def test_observation_spaces(env: CompilerEnv):
"""Test that the environment reports the service's observation spaces."""
env.reset()
assert env.observation.spaces.keys() == {
"ir", "features", "runtime", "size"
}
assert env.observation.spaces["ir"].space == Sequence(
name="ir",
size_range=(0, np.iinfo(np.int64).max),
dtype=str,
opaque_data_format=None,
)
assert env.observation.spaces["features"].space == Box(name="features",
shape=(3, ),
low=0,
high=100000,
dtype=int)
assert env.observation.spaces["runtime"].space == Scalar(name="runtime",
min=0,
max=np.inf,
dtype=float)
assert env.observation.spaces["size"].space == Scalar(name="size",
min=0,
max=np.inf,
dtype=float)
def test_add_benchmark_invalid_path(env: CompilerEnv):
with tempfile.TemporaryDirectory() as d:
tmp = Path(d) / "not_a_file"
with pytest.raises(FileNotFoundError) as ctx:
env.reset(benchmark=Benchmark.from_file("benchmark://foo", tmp))
# Use endswith() because on macOS there may be a /private prefix.
assert str(ctx.value).endswith(str(tmp))
def test_add_benchmark_invalid_path(env: CompilerEnv):
with tempfile.TemporaryDirectory() as d:
tmp = Path(d) / "not_a_file"
with pytest.raises(FileNotFoundError) as ctx:
env.reset(benchmark=Benchmark(uri="benchmark://foo",
program=File(uri=f"file:///{tmp}")))
assert str(ctx.value) == f'File not found: "{tmp}"'
def test_autophase_crc32_feature_vector(env: CompilerEnv):
env.benchmark = "cBench-v0/crc32"
env.reset()
features = env.observation["Autophase"]
assert features[0] == 0 # BBNumArgsHi
assert features[1] == 0 # BBNumArgsLo
assert features[2] == 16 # onePred
assert features[3] == 12 # onePredOneSuc
assert features[4] == 2 # onePredTwoSuc
assert features[5] == 16 # oneSuccessor
assert features[6] == 8 # twoPred
assert features[7] == 2 # twoPredOneSuc
assert features[8] == 4 # twoEach
assert features[9] == 8 # twoSuccessor
assert features[10] == 0 # morePreds
assert features[11] == 0 # BB03Phi
assert features[12] == 0 # BBHiPhi
assert features[13] == 29 # BBNoPhi
assert features[14] == 0 # BeginPhi
assert features[15] == 24 # BranchCount
assert features[16] == 9 # returnInt
assert features[17] == 2 # CriticalCount
assert features[18] == 32 # NumEdges
assert features[19] == 38 # const32Bit
assert features[20] == 21 # const64Bit
assert features[21] == 14 # numConstZeroes
assert features[22] == 30 # numConstOnes
assert features[23] == 16 # UncondBranches
assert features[24] == 13 # binaryConstArg
assert features[25] == 0 # NumAShrInst
assert features[26] == 5 # NumAddInst
assert features[27] == 24 # NumAllocaInst
assert features[28] == 3 # NumAndInst
assert features[29] == 3 # BlockMid
assert features[30] == 26 # BlockLow
assert features[31] == 0 # NumBitCastInst
assert features[32] == 24 # NumBrInst
assert features[33] == 13 # NumCallInst
assert features[34] == 5 # NumGetElementPtrInst
assert features[35] == 10 # NumICmpInst
assert features[36] == 3 # NumLShrInst
assert features[37] == 51 # NumLoadInst
assert features[38] == 0 # NumMulInst
assert features[39] == 1 # NumOrInst
assert features[40] == 0 # NumPHIInst
assert features[41] == 5 # NumRetInst
assert features[42] == 0 # NumSExtInst
assert features[43] == 0 # NumSelectInst
assert features[44] == 0 # NumShlInst
assert features[45] == 38 # NumStoreInst
assert features[46] == 0 # NumSubInst
assert features[47] == 1 # NumTruncInst
assert features[48] == 8 # NumXorInst
assert features[49] == 5 # NumZExtInst
assert features[50] == 29 # TotalBlocks
assert features[51] == 196 # TotalInsts
assert features[52] == 131 # TotalMemInst
assert features[53] == 13 # TotalFuncs
assert features[54] == 0 # ArgsPhi
assert features[55] == 81 # testUnary
def test_add_benchmark_invalid_protocol(env: CompilerEnv):
with pytest.raises(ValueError) as ctx:
env.reset(benchmark=Benchmark(uri="benchmark://foo",
program=File(
uri="https://invalid/protocol")))
assert (str(ctx.value) ==
'Unsupported benchmark URI protocol: "https://invalid/protocol"')
def test_observations(env: CompilerEnv):
"""Test observation spaces."""
env.reset()
assert len(env.observation["ir"]) > 0
assert all(env.observation["Inst2vec"] >= 0)
assert all(env.observation["Autophase"] >= 0)
assert len(env.observation["Programl"]) > 0
def test_service_env_dies_reset(env: CompilerEnv):
env.observation_space = "Autophase"
env.reward_space = "IrInstructionCount"
env.reset("cBench-v0/crc32")
# Kill the service.
env.service.close()
# Check that the environment doesn't fall over.
observation, reward, done, info = env.step(0)
assert done, info["error_details"]
assert not env.in_episode
# Check that default values are returned.
np.testing.assert_array_equal(observation, np.zeros(AUTOPHASE_FEATURE_DIM))
assert reward == 0
# Reset the environment and check that it works.
env.reset(benchmark="cBench-v0/crc32")
assert env.in_episode
observation, reward, done, info = env.step(0)
assert not done, info["error_details"]
assert observation is not None
assert reward is not None
def test_default_reward(env: CompilerEnv):
"""Test default reward space."""
env.reward_space = "runtime"
env.reset()
observation, reward, done, info = env.step(0)
assert observation is None
assert reward == 0
assert not done
def test_add_benchmark_invalid_protocol(env: CompilerEnv):
with pytest.raises(ValueError) as ctx:
env.reset(benchmark=Benchmark(
BenchmarkProto(uri="benchmark://foo",
program=File(uri="https://invalid/protocol")), ))
assert str(ctx.value) == (
"Invalid benchmark data URI. "
'Only the file:/// protocol is supported: "https://invalid/protocol"')
def test_double_reset(env: CompilerEnv):
"""Test that reset() can be called twice."""
env.reset()
assert env.in_episode
env.step(env.action_space.sample())
env.reset()
env.step(env.action_space.sample())
assert env.in_episode
def test_observation_spaces(env: CompilerEnv):
env.reset()
assert env.observation.spaces.keys() == {"ir", "features"}
assert env.observation.spaces["ir"].space == Sequence(
size_range=(0, None), dtype=str, opaque_data_format="")
assert env.observation.spaces["features"].space == Box(shape=(3, ),
low=-100,
high=100,
dtype=np.int64)
def test_benchmark_path_invalid_protocol(env: CompilerEnv):
benchmark = Benchmark(uri="benchmark://new",
program=File(uri="invalid_protocol://test"))
with pytest.raises(ValueError) as ctx:
env.reset(benchmark=benchmark)
assert (str(ctx.value) ==
'Unsupported benchmark URI protocol: "invalid_protocol://test"')
def test_invalid_benchmark_data(env: CompilerEnv):
benchmark = Benchmark(
uri="benchmark://new",
program=File(contents="Invalid bitcode".encode("utf-8")))
with pytest.raises(ValueError) as ctx:
env.reset(benchmark=benchmark)
assert str(ctx.value) == 'Failed to parse LLVM bitcode: "benchmark://new"'
def test_reset_to_force_benchmark(env: CompilerEnv):
"""Reset that calling reset() with a benchmark forces that benchmark to
be used for every subsequent episode.
"""
env.benchmark = None
env.reset(benchmark="benchmark://cBench-v0/crc32")
assert env.benchmark == "benchmark://cBench-v0/crc32"
for _ in range(10):
env.reset()
assert env.benchmark == "benchmark://cBench-v0/crc32"
def test_double_reset_with_step(env: CompilerEnv):
"""Test that reset() can be called twice with a step."""
env.reset()
assert env.in_episode
_, _, done, info = env.step(env.action_space.sample())
assert not done, info
env.reset()
_, _, done, info = env.step(env.action_space.sample())
assert not done, info
assert env.in_episode
def test_fork(env: CompilerEnv):
env.reset()
env.step(0)
env.step(1)
other_env = env.fork()
try:
assert env.benchmark == other_env.benchmark
assert other_env.actions == [0, 1]
finally:
other_env.close()
def test_benchmark_path_not_found(env: CompilerEnv):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
benchmark = Benchmark(uri="benchmark://new",
program=File(uri=f"file:///{tmpdir}/not_found"))
with pytest.raises(FileNotFoundError) as ctx:
env.reset(benchmark=benchmark)
assert str(ctx.value) == f'File not found: "{tmpdir}/not_found"'
def random_search(env: CompilerEnv):
best = float("inf")
for _ in range(FLAGS.gcc_search_budget):
env.reset()
env.choices = [
random.randint(-1, min(FLAGS.max_range,
len(opt) - 1))
for opt in env.gcc_spec.options
]
best = min(objective(env), best)
return best
def test_ir_observation_space(env: CompilerEnv):
env.reset("cBench-v0/crc32")
key = "Ir"
space = env.observation.spaces[key]
assert isinstance(space, Sequence)
assert space.dtype == str
assert space.size_range == (0, None)
value: str = env.observation[key]
assert isinstance(value, str)
assert space.contains(value)
def test_step(env: CompilerEnv, observation_space: str, reward_space: str):
"""Request every combination of observation and reward in a fresh environment."""
env.reward_space = None
env.observation_space = None
env.reset(benchmark="cbench-v1/crc32")
observation = env.observation[observation_space]
assert observation is not None
reward = env.reward[reward_space]
assert reward is not None
def test_benchmark_path_empty_file(env: CompilerEnv):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
(tmpdir / "test.bc").touch()
benchmark = Benchmark(uri="benchmark://new",
program=File(uri=f"file:///{tmpdir}/test.bc"))
with pytest.raises(ValueError) as ctx:
env.reset(benchmark=benchmark)
assert str(ctx.value) == f'File is empty: "{tmpdir}/test.bc"'
def test_step(env: CompilerEnv, action_name: str):
"""Run each action on a single benchmark."""
env.reward_space = "IrInstructionCount"
env.observation_space = "Autophase"
env.reset(benchmark="cBench-v0/crc32")
observation, reward, done, _ = env.step(
env.action_space.from_string(action_name))
assert isinstance(observation, np.ndarray)
assert observation.shape == (AUTOPHASE_FEATURE_DIM, )
assert isinstance(reward, float)
assert isinstance(done, bool)
def test_inst2vec_embedding_indices_observation_space(
env: CompilerEnv, cbench_crc32_inst2vec_embedding_indices: List[int]):
env.reset("cBench-v0/crc32")
space = "Inst2vecEmbeddingIndices"
assert isinstance(env.observation.spaces[space], Sequence)
value: List[int] = env.observation[space]
print(value)
assert isinstance(value, list)
for item in value:
assert isinstance(item, int)
assert value == cbench_crc32_inst2vec_embedding_indices
def test_invalid_benchmark_path_contents(env: CompilerEnv):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
with open(str(tmpdir / "test.bc"), "w") as f:
f.write("Invalid bitcode")
benchmark = Benchmark(uri="benchmark://new",
program=File(uri=f"file:///{tmpdir}/test.bc"))
with pytest.raises(ValueError) as ctx:
env.reset(benchmark=benchmark)
assert str(ctx.value) == 'Failed to parse LLVM bitcode: "benchmark://new"'
