def benchmark(self, uri: Optional[str] = None) -> Benchmark:
self.install()
if uri is None or len(uri) <= len(self.name) + 1:
return self._get_benchmark_by_index(self.random.integers(
self.size))
# The absolute path of the file, without an extension.
path_stem = self.dataset_root / uri[len(self.name) + 1:]
# If the file does not exist, compile it on-demand.
bitcode_path = Path(f"{path_stem}.bc")
cc_file_path = Path(f"{path_stem}.txt")
if not bitcode_path.is_file():
if not cc_file_path.is_file():
raise LookupError(
f"Benchmark not found: {uri} (file not found: {cc_file_path})"
)
# Load the C++ source into memory and pre-process it.
with open(cc_file_path) as f:
src = self.preprocess_poj104_source(f.read())
# Compile the C++ source into a bitcode file.
with atomic_file_write(bitcode_path) as tmp_bitcode_path:
compile_cmd = ClangInvocation.from_c_file(
"-",
copt=[
"-xc++",
"-ferror-limit=1", # Stop on first error.
"-w", # No warnings.
# Some of the programs use the gets() function that was
# deprecated in C++11 and removed in C++14.
"-std=c++11",
],
).command(outpath=tmp_bitcode_path)
logger.debug("Exec %s", compile_cmd)
clang = subprocess.Popen(
compile_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
_, stderr = clang.communicate(src.encode("utf-8"), timeout=300)
if clang.returncode:
compile_cmd = " ".join(compile_cmd)
error = truncate(stderr.decode("utf-8"),
max_lines=20,
max_line_len=100)
raise BenchmarkInitError(f"Compilation job failed!\n"
f"Command: {compile_cmd}\n"
f"Error: {error}")
if not bitcode_path.is_file():
raise BenchmarkInitError(
f"Compilation job failed to produce output file!\nCommand: {compile_cmd}"
)
return BenchmarkWithSource.create(uri, bitcode_path, "source.cc",
cc_file_path)
def _download(urls: List[str], sha256: Optional[str],
max_retries: int) -> bytes:
if not urls:
raise ValueError("No URLs to download")
# Cache hit.
if sha256 and cache_path(f"downloads/{sha256}").is_file():
with open(str(cache_path(f"downloads/{sha256}")), "rb") as f:
return f.read()
# A retry loop, and loop over all urls provided.
last_exception = None
wait_time = 10
for _ in range(max(max_retries, 1)):
for url in urls:
try:
return _do_download_attempt(url, sha256)
except TooManyRequests as e:
last_exception = e
logger.info(
"Download attempt failed with Too Many Requests error. "
"Watiting %.1f seconds",
wait_time,
)
sleep(wait_time)
wait_time *= 1.5
except DownloadFailed as e:
logger.info("Download attempt failed: %s", truncate(e))
last_exception = e
raise last_exception
def benchmark_from_parsed_uri(self, uri: BenchmarkUri) -> Benchmark:
self.install()
benchmark_name = uri.path[1:]
if not benchmark_name:
raise LookupError(f"No benchmark specified: {uri}")
# The absolute path of the file, without an extension.
path_stem = os.path.normpath(f"{self.dataset_root}/{uri.path}")
bc_path, cl_path = Path(f"{path_stem}.bc"), Path(f"{path_stem}.cl")
# If the file does not exist, compile it on-demand.
if not bc_path.is_file():
if not cl_path.is_file():
raise LookupError(
f"Benchmark not found: {uri} (file not found: {cl_path}, path_stem {path_stem})"
)
# Compile the OpenCL kernel into a bitcode file.
with atomic_file_write(bc_path) as tmp_bc_path:
compile_command: List[str] = ClangInvocation.from_c_file(
cl_path,
copt=[
"-isystem",
str(self.libclc_dir),
"-include",
str(self.opencl_h_path),
"-target",
"nvptx64-nvidia-nvcl",
"-ferror-limit=1", # Stop on first error.
"-w", # No warnings.
],
).command(outpath=tmp_bc_path)
logger.debug("Exec %s", compile_command)
try:
with Popen(
compile_command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as clang:
_, stderr = communicate(clang, timeout=300)
except subprocess.TimeoutExpired:
raise BenchmarkInitError(
f"Benchmark compilation timed out: {uri}")
if clang.returncode:
compile_command = " ".join(compile_command)
error = truncate(stderr.decode("utf-8"),
max_lines=20,
max_line_len=20000)
raise BenchmarkInitError(f"Compilation job failed!\n"
f"Command: {compile_command}\n"
f"Error: {error}")
return BenchmarkWithSource.create(uri, bc_path, "kernel.cl", cl_path)
def summarize_datasets(datasets: Iterable[Dataset]) -> str:
rows = []
# Override the default iteration order of datasets.
for dataset in sorted(datasets, key=lambda d: d.name):
# Raw numeric values here, formatted below.
description = truncate(dataset.description, max_line_len=60)
links = ", ".join(f"`{name} <{url}>`__"
for name, url in sorted(dataset.references.items()))
if links:
description = f"{description} [{links}]"
rows.append((
dataset.name,
dataset.size,
description,
dataset.validatable,
))
rows.append(("Total", sum(r[1] for r in rows), "", ""))
return (tabulate(
[
(
n,
# A size of zero means infinite.
f"{f:,d}" if f > 0 else "∞",
l,
v,
) for n, f, l, v in rows
],
headers=(
"Dataset",
"Num. Benchmarks [#f1]_",
"Description",
"Validatable [#f2]_",
),
) + f"""
.. [#f1] Values obtained on {sys.platform}. Datasets are platform-specific.
.. [#f2] A **validatable** dataset is one where the behavior of the benchmarks
can be checked by compiling the programs to binaries and executing
them. If the benchmarks crash, or are found to have different behavior,
then validation fails. This type of validation is used to check that
the compiler has not broken the semantics of the program.
See :mod:`compiler_gym.bin.validate`.
""")
def environment_validation_fails(env: "CompilerEnv") -> bool: # noqa: F821
"""A hypothesis that holds true if environment validation fails."""
validation_result = env.validate()
logger.debug(
truncate(str(validation_result), max_lines=1, max_line_len=120))
return not validation_result.okay()
def _step(request: StepRequest) -> StepReply:
"""Run the actual step with parsed arguments."""
states: List[StateToVisualize] = []
with env_lock:
env.reward_space = request.reward
env.reset(benchmark=request.benchmark)
# Replay all actions except the last one.
if request.all_states:
# Replay actions one at a time to receive incremental rewards. The
# first item represents the state prior to any actions.
(instcount, autophase), _, done, info = env.step(
action=[],
observations=[
env.observation.spaces["InstCountDict"],
env.observation.spaces["AutophaseDict"],
],
)
if done:
raise ValueError(
f"Failed to compute initial state: {info['error_details']}"
)
states.append(
StateToVisualize(
instcount=instcount,
autophase=autophase,
reward=0,
))
for action in request.actions[:-1]:
(instcount, autophase), reward, done, info = env.step(
action,
observations=[
env.observation.spaces["InstCountDict"],
env.observation.spaces["AutophaseDict"],
],
)
states.append(
StateToVisualize(
instcount=instcount,
autophase=autophase,
reward=reward,
))
if done:
raise ValueError(
f"Failed to apply action {action}: {info['error_details']}"
)
else:
# Replay actions in a single batch.
_, _, done, info = env.step(request.actions[:-1])
if done:
raise ValueError(
f"Failed to apply actions {request.actions}: {info['error_details']}"
)
# Perform the final action.
(ir, instcount, autophase), (reward, ), done, _ = env.raw_step(
actions=request.actions[-1:],
observations=[
env.observation.spaces["Ir"],
env.observation.spaces["InstCountDict"],
env.observation.spaces["AutophaseDict"],
],
rewards=[env.reward_space],
)
states.append(
StateToVisualize(
instcount=instcount,
autophase=autophase,
reward=reward,
))
return StepReply(
commandline=env.commandline(),
done=done,
ir=truncate(ir, max_line_len=250, max_lines=1024),
states=states,
)
def benchmark_from_parsed_uri(self, uri: BenchmarkUri) -> Benchmark:
self.install()
# The absolute path of the file, without an extension.
path_stem = os.path.normpath(f"{self.dataset_root}/{uri.path}")
# If the file does not exist, compile it on-demand.
bitcode_path = Path(f"{path_stem}.bc")
cc_file_path = Path(f"{path_stem}.txt")
if not bitcode_path.is_file():
if not cc_file_path.is_file():
raise LookupError(
f"Benchmark not found: {uri} (file not found: {cc_file_path})"
)
# Load the C++ source into memory and pre-process it.
with open(cc_file_path) as f:
src = self.preprocess_poj104_source(f.read())
# Compile the C++ source into a bitcode file.
with atomic_file_write(bitcode_path) as tmp_bitcode_path:
compile_cmd = ClangInvocation.from_c_file(
"-",
copt=[
"-xc++",
"-ferror-limit=1", # Stop on first error.
"-w", # No warnings.
# Some of the programs use the gets() function that was
# deprecated in C++11 and removed in C++14.
"-std=c++11",
],
).command(outpath=tmp_bitcode_path)
logger.debug("Exec %s", compile_cmd)
try:
with Popen(
compile_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as clang:
_, stderr = clang.communicate(
input=src.encode("utf-8"), timeout=300
)
except subprocess.TimeoutExpired:
raise BenchmarkInitError(f"Benchmark compilation timed out: {uri}")
if clang.returncode:
compile_cmd = " ".join(compile_cmd)
error = truncate(stderr.decode("utf-8"), max_lines=20, max_line_len=100)
if tmp_bitcode_path.is_file():
tmp_bitcode_path.unlink()
raise BenchmarkInitError(
f"Compilation job failed!\n"
f"Command: {compile_cmd}\n"
f"Error: {error}"
)
if not bitcode_path.is_file():
raise BenchmarkInitError(
f"Compilation job failed to produce output file!\nCommand: {compile_cmd}"
)
return BenchmarkWithSource.create(uri, bitcode_path, "source.cc", cc_file_path)
def test_truncate_no_truncation():
assert truncate("abc") == "abc"
assert truncate("abcdef\nabcdef", max_line_len=7, max_lines=2) == "abcdef\nabcdef"
def test_truncate_final_line():
assert truncate("abc\ndef\n123", max_line_len=5, max_lines=2) == "abc\nde..."
assert truncate("abc\ndef\n123", max_line_len=10, max_lines=2) == "abc\ndef..."
def test_truncate_dual_lines():
assert (
truncate("abcdefghijklmnop\nbcdefghijklmnop", max_line_len=5, max_lines=3)
== "ab...\nbc..."
)
def test_truncate_single_line():
assert truncate("abcdefghijklmnop", max_line_len=5) == "ab..."
def benchmark_from_seed(
self, seed: int, max_retries: int = 3, retry_count: int = 0
) -> CsmithBenchmark:
"""Get a benchmark from a uint32 seed.
:param seed: A number in the range 0 <= n < 2^32.
:return: A benchmark instance.
:raises OSError: If Csmith fails.
:raises BenchmarkInitError: If the C program generated by Csmith cannot
be lowered to LLVM-IR.
"""
if retry_count >= max_retries:
raise OSError(
f"Csmith failed after {retry_count} {plural(retry_count, 'attempt', 'attempts')} "
f"with seed {seed}"
)
self.install()
# Run csmith with the given seed and pipe the output to clang to
# assemble a bitcode.
logger.debug("Exec csmith --seed %d", seed)
with Popen(
[str(self.csmith_bin_path), "--seed", str(seed)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as csmith:
# Generate the C source.
src, stderr = csmith.communicate(timeout=300)
if csmith.returncode:
try:
stderr = "\n".join(
truncate(stderr.decode("utf-8"), max_line_len=200, max_lines=20)
)
logger.warning("Csmith failed with seed %d: %s", seed, stderr)
except UnicodeDecodeError:
# Failed to interpret the stderr output, generate a generic
# error message.
logger.warning("Csmith failed with seed %d", seed)
return self.benchmark_from_seed(
seed, max_retries=max_retries, retry_count=retry_count + 1
)
# Pre-process the source.
with tempfile.TemporaryDirectory() as tmpdir:
src_file = f"{tmpdir}/src.c"
with open(src_file, "wb") as f:
f.write(src)
preprocessed_src = self.gcc(
"-E",
"-I",
str(self.site_data_path / "includes"),
"-o",
"-",
src_file,
cwd=tmpdir,
timeout=60,
volumes={
str(self.site_data_path / "includes"): {
"bind": str(self.site_data_path / "includes"),
"mode": "ro",
}
},
)
return self.benchmark_class.create(
f"{self.name}/{seed}", preprocessed_src.encode("utf-8"), src
)
def benchmark_from_seed(self,
seed: int,
max_retries: int = 3,
retry_count: int = 0) -> CsmithBenchmark:
"""Get a benchmark from a uint32 seed.
:param seed: A number in the range 0 <= n < 2^32.
:return: A benchmark instance.
:raises OSError: If Csmith fails.
:raises BenchmarkInitError: If the C program generated by Csmith cannot
be lowered to LLVM-IR.
"""
if retry_count >= max_retries:
raise OSError(
f"Csmith failed after {retry_count} {plural(retry_count, 'attempt', 'attempts')} "
f"with seed {seed}")
self.install()
# Run csmith with the given seed and pipe the output to clang to
# assemble a bitcode.
logger.debug("Exec csmith --seed %d", seed)
try:
with Popen(
[str(self.csmith_bin_path), "--seed",
str(seed)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as csmith:
# Generate the C source.
src, stderr = communicate(csmith, timeout=300)
if csmith.returncode:
try:
stderr = "\n".join(
truncate(stderr.decode("utf-8"),
max_line_len=200,
max_lines=20))
logger.warning("Csmith failed with seed %d: %s", seed,
stderr)
except UnicodeDecodeError:
# Failed to interpret the stderr output, generate a generic
# error message.
logger.warning("Csmith failed with seed %d", seed)
return self.benchmark_from_seed(seed,
max_retries=max_retries,
retry_count=retry_count +
1)
# Compile to IR.
with Popen(
self.clang_compile_command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
) as clang:
stdout, _ = communicate(clang, input=src, timeout=300)
if clang.returncode:
compile_cmd = " ".join(self.clang_compile_command)
raise BenchmarkInitError(f"Compilation job failed!\n"
f"Csmith seed: {seed}\n"
f"Command: {compile_cmd}\n")
except subprocess.TimeoutExpired:
raise BenchmarkInitError(
f"Benchmark generation using seed {seed} timed out")
return self.benchmark_class.create(f"{self.name}/{seed}", stdout, src)