def test_cpuinfo_observation_space(env: CompilerEnv): env.reset("cBench-v0/crc32") space = "CpuInfo" assert isinstance(env.observation.spaces[space], Sequence) value: Dict[str, Any] = env.observation[space] assert isinstance(value, dict) # Test each expected key, removing it as we go. assert isinstance(value.pop("name"), str) assert isinstance(value.pop("cores_count"), int) assert isinstance(value.pop("l1i_cache_size"), int) assert isinstance(value.pop("l1i_cache_count"), int) assert isinstance(value.pop("l1d_cache_size"), int) assert isinstance(value.pop("l1d_cache_count"), int) assert isinstance(value.pop("l2_cache_size"), int) assert isinstance(value.pop("l2_cache_count"), int) assert isinstance(value.pop("l3_cache_size"), int) assert isinstance(value.pop("l3_cache_count"), int) assert isinstance(value.pop("l4_cache_size"), int) assert isinstance(value.pop("l4_cache_count"), int) # Anything left in the JSON dictionary now is an unexpected key. assert not value invalid = "invalid value" with pytest.raises(KeyError) as ctx: _ = env.observation[invalid] assert str(ctx.value) == f"'{invalid}'" space = "CpuInfoDict" value: Dict[str, Any] = env.observation[space] print(value)
def test_autophase_dict_observation_space(env: CompilerEnv): env.reset("cBench-v0/crc32") key = "AutophaseDict" space = env.observation.spaces[key] assert isinstance(space, DictSpace) value: Dict[str, int] = env.observation[key] assert len(value) == 56
def test_eager_reward(env: CompilerEnv): env.eager_reward_space = "codesize" env.reset() observation, reward, done, info = env.step(0) assert observation is None assert reward == 0 assert not done
def test_service_env_dies_reset(env: CompilerEnv): env.observation_space = "Autophase" env.reward_space = "IrInstructionCount" env.reset("cbench-v1/crc32") # Kill the service. Note killing the service for a ManagedConnection will # result in a ServiceError because we have not ended the session we started # with env.reset() above. For UnmanagedConnection, this error will not be # raised. try: env.service.close() except ServiceError as e: assert "Service exited with returncode " in str(e) # Check that the environment doesn't fall over. observation, reward, done, info = env.step(0) assert done, info["error_details"] assert not env.in_episode # Check that default values are returned. np.testing.assert_array_equal(observation, np.zeros(AUTOPHASE_FEATURE_DIM)) assert reward == 0 # Reset the environment and check that it works. env.reset(benchmark="cbench-v1/crc32") assert env.in_episode observation, reward, done, info = env.step(0) assert not done, info["error_details"] assert observation is not None assert reward is not None
def test_invalid_benchmark_missing_file(env: CompilerEnv): benchmark = Benchmark(uri="benchmark://new", ) with pytest.raises(ValueError) as ctx: env.reset(benchmark=benchmark) assert str(ctx.value) == "No program set"
def test_reward_spaces(env: CompilerEnv): env.reset("cBench-v0/crc32") assert set(env.reward.ranges.keys()) == { "IrInstructionCount", "IrInstructionCountO3", "IrInstructionCountOz", "IrInstructionCountOzDiff", } reward_space = "IrInstructionCount" assert env.reward.ranges[reward_space] == (-np.inf, 0) assert env.reward[reward_space] < 0 reward_space = "IrInstructionCountO3" assert env.reward.ranges[reward_space] == (0, np.inf) assert env.reward[reward_space] > 0 reward_space = "IrInstructionCountOz" assert env.reward.ranges[reward_space] == (0, np.inf) assert env.reward[reward_space] > 0 reward_space = "IrInstructionCountOzDiff" assert env.reward.ranges[reward_space] == (-np.inf, np.inf) invalid = "invalid value" with pytest.raises(KeyError) as ctx: _ = env.reward[invalid] assert str(ctx.value) == f"'{invalid}'"
def test_observation_spaces(env: CompilerEnv): """Test that the environment reports the service's observation spaces.""" env.reset() assert env.observation.spaces.keys() == { "ir", "features", "runtime", "size" } assert env.observation.spaces["ir"].space == Sequence( name="ir", size_range=(0, np.iinfo(np.int64).max), dtype=str, opaque_data_format=None, ) assert env.observation.spaces["features"].space == Box(name="features", shape=(3, ), low=0, high=100000, dtype=int) assert env.observation.spaces["runtime"].space == Scalar(name="runtime", min=0, max=np.inf, dtype=float) assert env.observation.spaces["size"].space == Scalar(name="size", min=0, max=np.inf, dtype=float)
def test_add_benchmark_invalid_path(env: CompilerEnv): with tempfile.TemporaryDirectory() as d: tmp = Path(d) / "not_a_file" with pytest.raises(FileNotFoundError) as ctx: env.reset(benchmark=Benchmark.from_file("benchmark://foo", tmp)) # Use endswith() because on macOS there may be a /private prefix. assert str(ctx.value).endswith(str(tmp))
def test_add_benchmark_invalid_path(env: CompilerEnv): with tempfile.TemporaryDirectory() as d: tmp = Path(d) / "not_a_file" with pytest.raises(FileNotFoundError) as ctx: env.reset(benchmark=Benchmark(uri="benchmark://foo", program=File(uri=f"file:///{tmp}"))) assert str(ctx.value) == f'File not found: "{tmp}"'
def test_autophase_crc32_feature_vector(env: CompilerEnv): env.benchmark = "cBench-v0/crc32" env.reset() features = env.observation["Autophase"] assert features[0] == 0 # BBNumArgsHi assert features[1] == 0 # BBNumArgsLo assert features[2] == 16 # onePred assert features[3] == 12 # onePredOneSuc assert features[4] == 2 # onePredTwoSuc assert features[5] == 16 # oneSuccessor assert features[6] == 8 # twoPred assert features[7] == 2 # twoPredOneSuc assert features[8] == 4 # twoEach assert features[9] == 8 # twoSuccessor assert features[10] == 0 # morePreds assert features[11] == 0 # BB03Phi assert features[12] == 0 # BBHiPhi assert features[13] == 29 # BBNoPhi assert features[14] == 0 # BeginPhi assert features[15] == 24 # BranchCount assert features[16] == 9 # returnInt assert features[17] == 2 # CriticalCount assert features[18] == 32 # NumEdges assert features[19] == 38 # const32Bit assert features[20] == 21 # const64Bit assert features[21] == 14 # numConstZeroes assert features[22] == 30 # numConstOnes assert features[23] == 16 # UncondBranches assert features[24] == 13 # binaryConstArg assert features[25] == 0 # NumAShrInst assert features[26] == 5 # NumAddInst assert features[27] == 24 # NumAllocaInst assert features[28] == 3 # NumAndInst assert features[29] == 3 # BlockMid assert features[30] == 26 # BlockLow assert features[31] == 0 # NumBitCastInst assert features[32] == 24 # NumBrInst assert features[33] == 13 # NumCallInst assert features[34] == 5 # NumGetElementPtrInst assert features[35] == 10 # NumICmpInst assert features[36] == 3 # NumLShrInst assert features[37] == 51 # NumLoadInst assert features[38] == 0 # NumMulInst assert features[39] == 1 # NumOrInst assert features[40] == 0 # NumPHIInst assert features[41] == 5 # NumRetInst assert features[42] == 0 # NumSExtInst assert features[43] == 0 # NumSelectInst assert features[44] == 0 # NumShlInst assert features[45] == 38 # NumStoreInst assert features[46] == 0 # NumSubInst assert features[47] == 1 # NumTruncInst assert features[48] == 8 # NumXorInst assert features[49] == 5 # NumZExtInst assert features[50] == 29 # TotalBlocks assert features[51] == 196 # TotalInsts assert features[52] == 131 # TotalMemInst assert features[53] == 13 # TotalFuncs assert features[54] == 0 # ArgsPhi assert features[55] == 81 # testUnary
def test_add_benchmark_invalid_protocol(env: CompilerEnv): with pytest.raises(ValueError) as ctx: env.reset(benchmark=Benchmark(uri="benchmark://foo", program=File( uri="https://invalid/protocol"))) assert (str(ctx.value) == 'Unsupported benchmark URI protocol: "https://invalid/protocol"')
def test_observations(env: CompilerEnv): """Test observation spaces.""" env.reset() assert len(env.observation["ir"]) > 0 assert all(env.observation["Inst2vec"] >= 0) assert all(env.observation["Autophase"] >= 0) assert len(env.observation["Programl"]) > 0
def test_service_env_dies_reset(env: CompilerEnv): env.observation_space = "Autophase" env.reward_space = "IrInstructionCount" env.reset("cBench-v0/crc32") # Kill the service. env.service.close() # Check that the environment doesn't fall over. observation, reward, done, info = env.step(0) assert done, info["error_details"] assert not env.in_episode # Check that default values are returned. np.testing.assert_array_equal(observation, np.zeros(AUTOPHASE_FEATURE_DIM)) assert reward == 0 # Reset the environment and check that it works. env.reset(benchmark="cBench-v0/crc32") assert env.in_episode observation, reward, done, info = env.step(0) assert not done, info["error_details"] assert observation is not None assert reward is not None
def test_default_reward(env: CompilerEnv): """Test default reward space.""" env.reward_space = "runtime" env.reset() observation, reward, done, info = env.step(0) assert observation is None assert reward == 0 assert not done
def test_add_benchmark_invalid_protocol(env: CompilerEnv): with pytest.raises(ValueError) as ctx: env.reset(benchmark=Benchmark( BenchmarkProto(uri="benchmark://foo", program=File(uri="https://invalid/protocol")), )) assert str(ctx.value) == ( "Invalid benchmark data URI. " 'Only the file:/// protocol is supported: "https://invalid/protocol"')
def test_double_reset(env: CompilerEnv): """Test that reset() can be called twice.""" env.reset() assert env.in_episode env.step(env.action_space.sample()) env.reset() env.step(env.action_space.sample()) assert env.in_episode
def test_observation_spaces(env: CompilerEnv): env.reset() assert env.observation.spaces.keys() == {"ir", "features"} assert env.observation.spaces["ir"].space == Sequence( size_range=(0, None), dtype=str, opaque_data_format="") assert env.observation.spaces["features"].space == Box(shape=(3, ), low=-100, high=100, dtype=np.int64)
def test_benchmark_path_invalid_protocol(env: CompilerEnv): benchmark = Benchmark(uri="benchmark://new", program=File(uri="invalid_protocol://test")) with pytest.raises(ValueError) as ctx: env.reset(benchmark=benchmark) assert (str(ctx.value) == 'Unsupported benchmark URI protocol: "invalid_protocol://test"')
def test_invalid_benchmark_data(env: CompilerEnv): benchmark = Benchmark( uri="benchmark://new", program=File(contents="Invalid bitcode".encode("utf-8"))) with pytest.raises(ValueError) as ctx: env.reset(benchmark=benchmark) assert str(ctx.value) == 'Failed to parse LLVM bitcode: "benchmark://new"'
def test_reset_to_force_benchmark(env: CompilerEnv): """Reset that calling reset() with a benchmark forces that benchmark to be used for every subsequent episode. """ env.benchmark = None env.reset(benchmark="benchmark://cBench-v0/crc32") assert env.benchmark == "benchmark://cBench-v0/crc32" for _ in range(10): env.reset() assert env.benchmark == "benchmark://cBench-v0/crc32"
def test_double_reset_with_step(env: CompilerEnv): """Test that reset() can be called twice with a step.""" env.reset() assert env.in_episode _, _, done, info = env.step(env.action_space.sample()) assert not done, info env.reset() _, _, done, info = env.step(env.action_space.sample()) assert not done, info assert env.in_episode
def test_fork(env: CompilerEnv): env.reset() env.step(0) env.step(1) other_env = env.fork() try: assert env.benchmark == other_env.benchmark assert other_env.actions == [0, 1] finally: other_env.close()
def test_benchmark_path_not_found(env: CompilerEnv): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) benchmark = Benchmark(uri="benchmark://new", program=File(uri=f"file:///{tmpdir}/not_found")) with pytest.raises(FileNotFoundError) as ctx: env.reset(benchmark=benchmark) assert str(ctx.value) == f'File not found: "{tmpdir}/not_found"'
def random_search(env: CompilerEnv): best = float("inf") for _ in range(FLAGS.gcc_search_budget): env.reset() env.choices = [ random.randint(-1, min(FLAGS.max_range, len(opt) - 1)) for opt in env.gcc_spec.options ] best = min(objective(env), best) return best
def test_ir_observation_space(env: CompilerEnv): env.reset("cBench-v0/crc32") key = "Ir" space = env.observation.spaces[key] assert isinstance(space, Sequence) assert space.dtype == str assert space.size_range == (0, None) value: str = env.observation[key] assert isinstance(value, str) assert space.contains(value)
def test_step(env: CompilerEnv, observation_space: str, reward_space: str): """Request every combination of observation and reward in a fresh environment.""" env.reward_space = None env.observation_space = None env.reset(benchmark="cbench-v1/crc32") observation = env.observation[observation_space] assert observation is not None reward = env.reward[reward_space] assert reward is not None
def test_benchmark_path_empty_file(env: CompilerEnv): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) (tmpdir / "test.bc").touch() benchmark = Benchmark(uri="benchmark://new", program=File(uri=f"file:///{tmpdir}/test.bc")) with pytest.raises(ValueError) as ctx: env.reset(benchmark=benchmark) assert str(ctx.value) == f'File is empty: "{tmpdir}/test.bc"'
def test_step(env: CompilerEnv, action_name: str): """Run each action on a single benchmark.""" env.reward_space = "IrInstructionCount" env.observation_space = "Autophase" env.reset(benchmark="cBench-v0/crc32") observation, reward, done, _ = env.step( env.action_space.from_string(action_name)) assert isinstance(observation, np.ndarray) assert observation.shape == (AUTOPHASE_FEATURE_DIM, ) assert isinstance(reward, float) assert isinstance(done, bool)
def test_inst2vec_embedding_indices_observation_space( env: CompilerEnv, cbench_crc32_inst2vec_embedding_indices: List[int]): env.reset("cBench-v0/crc32") space = "Inst2vecEmbeddingIndices" assert isinstance(env.observation.spaces[space], Sequence) value: List[int] = env.observation[space] print(value) assert isinstance(value, list) for item in value: assert isinstance(item, int) assert value == cbench_crc32_inst2vec_embedding_indices
def test_invalid_benchmark_path_contents(env: CompilerEnv): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) with open(str(tmpdir / "test.bc"), "w") as f: f.write("Invalid bitcode") benchmark = Benchmark(uri="benchmark://new", program=File(uri=f"file:///{tmpdir}/test.bc")) with pytest.raises(ValueError) as ctx: env.reset(benchmark=benchmark) assert str(ctx.value) == 'Failed to parse LLVM bitcode: "benchmark://new"'