def random(env: CompilerEnv,
optimization_target: OptimizationTarget,
search_time_seconds: int,
patience: int = 350,
**kwargs) -> None:
"""Run a random search on the environment.
:param env: The environment to optimize.
:param optimization_target: The target to optimize for.
:param search_time_seconds: The total search time.
:param patience: The number of steps to search without an improvement before
resetting to a new trajectory.
"""
with TemporaryDirectory(dir=transient_cache_path("."),
prefix="autotune-") as tmpdir:
final_env = lib_random_search(
make_env=lambda: optimization_target.make_env(env.benchmark),
outdir=tmpdir,
total_runtime=search_time_seconds,
patience=patience,
nproc=1,
)
env.apply(final_env.state)
final_env.close()
def opentuner_ga(
env: ClientServiceCompilerEnv,
optimization_target: OptimizationTarget,
search_time_seconds: int,
seed: int,
max_copies_of_pass: int = 4,
population: int = 200,
tournament: int = 5,
mutate: int = 2,
sharing: int = 1,
**kwargs,
) -> None:
"""Optimize an environment using opentuner.
OpenTuner is an extensible framework for program autotuning:
https://opentuner.org/
"""
cache_dir = transient_cache_path("llvm_autotuning")
cache_dir.mkdir(exist_ok=True, parents=True)
with tempfile.TemporaryDirectory(dir=cache_dir,
prefix="opentuner-") as tmpdir:
argparser = ot.default_argparser()
args = argparser.parse_args(args=[
f"--stop-after={search_time_seconds}",
f"--database={tmpdir}/opentuner.db",
"--no-dups",
"--technique=custom",
f"--seed={seed}",
"--parallelism=1",
])
ot.search.technique.register(
BinaryGA(
population=population,
tournament=tournament,
mutate=mutate,
sharing=sharing,
name="custom",
))
manipulator = LlvmOptFlagsTuner(
args,
target=optimization_target,
benchmark=env.benchmark,
max_copies_of_pass=max_copies_of_pass,
)
tuner = TuningRunMain(manipulator, args)
tuner.main()
class DesiredResult:
def __init__(self, configuration) -> None:
self.configuration = configuration
class Configuration:
def __init__(self, data) -> None:
self.data = data
wrapped = DesiredResult(Configuration(manipulator.best_config))
manipulator.run(wrapped, None, None)
env.reset()
env.multistep(manipulator.serialize_actions(manipulator.best_config))
def compile(self, env, timeout: int = 60) -> None:
"""This completes the compilation and linking of the final executable
specified by the original command line.
"""
with tempfile.NamedTemporaryFile(
dir=transient_cache_path("."), prefix="benchmark-", suffix=".bc"
) as f:
bitcode_path = f.name
env.write_bitcode(bitcode_path)
# Set the placeholder for input path.
cmd = list(self.proto.dynamic_config.build_cmd.argument).copy()
cmd = [bitcode_path if c == "$IN" else c for c in cmd]
logger.debug(f"$ {join_cmd(cmd)}")
with Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) as lower:
stdout, _ = lower.communicate(timeout=timeout)
if lower.returncode:
raise BenchmarkInitError(
f"Failed to lower LLVM bitcode with error:\n"
f"{stdout.decode('utf-8').rstrip()}\n"
f"Running command: {join_cmd(cmd)}"
)
def download_and_unpack_database(db: str, sha256: str) -> Path:
"""Download the given database, unpack it to the local filesystem, and
return the path.
"""
local_dir = cache_path(f"state_transition_dataset/{sha256}")
with _DB_DOWNLOAD_LOCK, InterProcessLock(
transient_cache_path(".state_transition_database_download.LOCK")
):
if not (local_dir / ".installed").is_file():
tar_data = io.BytesIO(download(db, sha256))
local_dir.mkdir(parents=True, exist_ok=True)
logger.info("Unpacking database to %s ...", local_dir)
with tarfile.open(fileobj=tar_data, mode="r:bz2") as arc:
arc.extractall(str(local_dir))
(local_dir / ".installed").touch()
unpacked = [f for f in local_dir.iterdir() if f.name != ".installed"]
if len(unpacked) != 1:
print(
f"fatal: Archive {db} expected to contain one file, contains: {len(unpacked)}",
file=sys.stderr,
)
return unpacked[0]
def make_working_dir():
"""Make a working directory for a service. The calling code is responsible for
removing this directory when done.
"""
service_directory = transient_cache_path("service")
timestamp = datetime.now().isoformat()
random_hash = random.getrandbits(32)
working_dir = Path(service_directory / f"{timestamp}-{random_hash:08x}")
(working_dir / "logs").mkdir(parents=True, exist_ok=False)
return working_dir
def get_storage_paths() -> List[Path]:
"""Return the list of paths used by CompilerGym for filesystem storage.
:return: A list of filesystem paths that CompilerGym uses to store files.
"""
return sorted({
runfiles_path.cache_path("."),
runfiles_path.transient_cache_path("."),
runfiles_path.site_data_path("."),
})
def __init__(self):
self.path = _create_timestamped_unique_service_dir(
transient_cache_path("."))
(self.path / "logs").mkdir()
self._directories_to_remove = [self.path]
if is_in_memory(self.path):
disk = _create_timestamped_unique_service_dir(cache_path("."))
self._directories_to_remove.append(disk)
os.symlink(disk, self.path / "disk")
else:
(self.path / "disk").mkdir()
def make_working_dir() -> Path:
"""Make a working directory for a service. The calling code is responsible
for removing this directory when done.
"""
while True:
random_hash = random.getrandbits(16)
service_name = datetime.now().strftime(
f"s/%m%dT%H%M%S-%f-{random_hash:04x}")
working_dir = transient_cache_path(service_name)
# Guard against the unlike scenario that there is a collision between
# the randomly generated working directories of multiple
# make_working_dir() calls.
try:
(working_dir / "logs").mkdir(parents=True, exist_ok=False)
break
except FileExistsError:
pass
return working_dir
def __call__(self, env: CompilerEnv, seed: int = 0xCC) -> CompilerEnvState:
"""Autotune the given environment.
:param env: The environment to autotune.
:param seed: The random seed for the autotuner.
:returns: A CompilerEnvState tuple describing the autotuning result.
"""
# Run the autotuner in a temporary working directory and capture the
# stdout/stderr.
with tempfile.TemporaryDirectory(dir=transient_cache_path("."),
prefix="autotune-") as tmpdir:
with temporary_working_directory(Path(tmpdir)):
with capture_output():
with Timer() as timer:
self.autotune(env, seed=seed, **self.autotune_kwargs)
return CompilerEnvState(
benchmark=env.benchmark.uri,
commandline=env.commandline(),
walltime=timer.time,
reward=self.optimization_target.final_reward(env),
)
def make_benchmark_from_command_line(
self,
cmd: Union[str, List[str]],
replace_driver: bool = True,
system_includes: bool = True,
timeout: int = 600,
) -> Benchmark:
"""Create a benchmark for use with this environment.
This function takes a command line compiler invocation as input,
modifies it to produce an unoptimized LLVM-IR bitcode, and then runs the
modified command line to produce a bitcode benchmark.
For example, the command line:
>>> benchmark = env.make_benchmark_from_command_line(
... ["gcc", "-DNDEBUG", "a.c", "b.c", "-o", "foo", "-lm"]
... )
Will compile a.c and b.c to an unoptimized benchmark that can be then
passed to :meth:`reset() <compiler_env.envs.CompilerEnv.reset>`.
The way this works is to change the first argument of the command line
invocation to the version of clang shipped with CompilerGym, and to then
append command line flags that causes the compiler to produce LLVM-IR
with optimizations disabled. For example the input command line:
.. code-block::
gcc -DNDEBUG a.c b.c -o foo -lm
Will be rewritten to be roughly equivalent to:
.. code-block::
/path/to/compiler_gym/clang -DNDEG a.c b.c \\
-Xclang -disable-llvm-passes -Xclang -disable-llvm-optzns \\ -c
-emit-llvm -o -
The generated benchmark then has a method :meth:`compile()
<compiler_env.envs.llvm.BenchmarkFromCommandLine.compile>` which
completes the linking and compilatilion to executable. For the above
example, this would be roughly equivalent to:
.. code-block::
/path/to/compiler_gym/clang environment-bitcode.bc -o foo -lm
:param cmd: A command line compiler invocation, either as a list of
arguments (e.g. :code:`["clang", "in.c"]`) or as a single shell
string (e.g. :code:`"clang in.c"`).
:param replace_driver: Whether to replace the first argument of the
command with the clang driver used by this environment.
:param system_includes: Whether to include the system standard libraries
during compilation jobs. This requires a system toolchain. See
:func:`get_system_library_flags`.
:param timeout: The maximum number of seconds to allow the compilation
job to run before terminating.
:return: A :class:`BenchmarkFromCommandLine
<compiler_gym.envs.llvm.BenchmarkFromCommandLine>` instance.
:raises ValueError: If no command line is provided.
:raises BenchmarkInitError: If executing the command line fails.
:raises TimeoutExpired: If a compilation job exceeds :code:`timeout`
seconds.
"""
if not cmd:
raise ValueError("Input command line is empty")
# Split the command line if passed a single string.
if isinstance(cmd, str):
cmd = shlex.split(cmd)
rewritten_cmd: List[str] = cmd.copy()
if len(cmd) < 2:
raise ValueError(
f"Input command line '{join_cmd(cmd)}' is too short")
# Append include flags for the system headers if requested.
if system_includes:
rewritten_cmd += get_system_library_flags()
# Use the CompilerGym clang binary in place of the original driver.
if replace_driver:
rewritten_cmd[0] = str(clang_path())
# Strip the -S flag, if present, as that changes the output format.
rewritten_cmd = [c for c in rewritten_cmd if c != "-S"]
invocation = GccInvocation(rewritten_cmd)
# Strip the output specifier(s). This is not strictly required since we
# override it later, but makes the generated command easier to
# understand.
for i in range(len(rewritten_cmd) - 2, -1, -1):
if rewritten_cmd[i] == "-o":
del rewritten_cmd[i + 1]
del rewritten_cmd[i]
# Fail early.
if "-" in invocation.sources:
raise ValueError("Input command line reads from stdin, "
f"which is not supported: '{join_cmd(cmd)}'")
# Convert all of the C/C++ sources to bitcodes which can then be linked
# into a single bitcode. We must process them individually because the
# '-c' flag does not support multiple sources when we are specifying the
# output path using '-o'.
sources = set(s for s in invocation.sources if not s.endswith(".o"))
if not sources:
raise ValueError(
f"Input command line has no source file inputs: '{join_cmd(cmd)}'"
)
bitcodes: List[bytes] = []
for source in sources:
# Adapt and execute the command line so that it will generate an
# unoptimized bitecode file.
emit_bitcode_command = rewritten_cmd.copy()
# Strip the name of other sources:
if len(sources) > 1:
emit_bitcode_command = [
c for c in emit_bitcode_command
if c == source or c not in sources
]
# Append the flags to emit the bitcode and disable the optimization
# passes.
emit_bitcode_command += [
"-c",
"-emit-llvm",
"-o",
"-",
"-Xclang",
"-disable-llvm-passes",
"-Xclang",
"-disable-llvm-optzns",
]
with Popen(emit_bitcode_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as clang:
logger.debug(
f"Generating LLVM bitcode benchmark: {join_cmd(emit_bitcode_command)}"
)
bitcode, stderr = clang.communicate(timeout=timeout)
if clang.returncode:
raise BenchmarkInitError(
f"Failed to generate LLVM bitcode with error:\n"
f"{stderr.decode('utf-8').rstrip()}\n"
f"Running command: {join_cmd(emit_bitcode_command)}\n"
f"From original commandline: {join_cmd(cmd)}")
bitcodes.append(bitcode)
# If there were multiple sources then link the bitcodes together.
if len(bitcodes) > 1:
with TemporaryDirectory(dir=transient_cache_path("."),
prefix="llvm-benchmark-") as dir:
# Write the bitcodes to files.
for i, bitcode in enumerate(bitcodes):
with open(os.path.join(dir, f"{i}.bc"), "wb") as f:
f.write(bitcode)
# Link the bitcode files.
llvm_link_cmd = [str(llvm_link_path()), "-o", "-"] + [
os.path.join(dir, f"{i}.bc") for i in range(len(bitcodes))
]
with Popen(llvm_link_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as llvm_link:
bitcode, stderr = llvm_link.communicate(timeout=timeout)
if llvm_link.returncode:
raise BenchmarkInitError(
f"Failed to link LLVM bitcodes with error: {stderr.decode('utf-8')}"
)
return BenchmarkFromCommandLine(invocation, bitcode, timeout)
def _get_system_library_flags(compiler: str) -> Iterable[str]:
"""Private implementation function."""
# Create a temporary file to write the compiled binary to, since GNU
# assembler does not support piping to stdout.
transient_cache = transient_cache_path(".")
transient_cache.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=transient_cache) as f:
cmd = [compiler, "-xc++", "-v", "-", "-o", f.name]
# On macOS we need to compile a binary to invoke the linker.
if sys.platform != "darwin":
cmd.append("-c")
# Retry loop to permit timeouts, though unlikely, in case of a
# heavily overloaded system (I have observed CI failures because
# of this).
for _ in range(3):
try:
with Popen(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
universal_newlines=True,
) as process:
_, stderr = communicate(
process=process, input="int main(){return 0;}", timeout=30
)
if process.returncode:
raise HostCompilerFailure(
f"Failed to invoke '{compiler}'. "
f"Is there a working system compiler?\n"
f"Error: {stderr.strip()}"
)
break
except subprocess.TimeoutExpired:
continue
except FileNotFoundError as e:
raise HostCompilerFailure(
f"Failed to invoke '{compiler}'. "
f"Is there a working system compiler?\n"
f"Error: {e}"
) from e
else:
raise HostCompilerFailure(
f"Compiler invocation '{join_cmd(cmd)}' timed out after 3 attempts."
)
# Parse the compiler output that matches the conventional output format
# used by clang and GCC:
#
# #include <...> search starts here:
# /path/1
# /path/2
# End of search list
in_search_list = False
lines = stderr.split("\n")
for line in lines:
if in_search_list and line.startswith("End of search list"):
break
elif in_search_list:
# We have an include path to return.
path = Path(line.strip())
yield "-isystem"
yield str(path)
# Compatibility fix for compiling benchmark sources which use the
# '#include <endian.h>' header, which on macOS is located in a
# 'machine/endian.h' directory.
if (path / "machine").is_dir():
yield "-isystem"
yield str(path / "machine")
elif line.startswith("#include <...> search starts here:"):
in_search_list = True
else:
msg = f"Failed to parse '#include <...>' search paths from '{compiler}'"
stderr = stderr.strip()
if stderr:
msg += f":\n{stderr}"
raise UnableToParseHostCompilerOutput(msg)
if sys.platform == "darwin":
yield "-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib"
def make_benchmark(
inputs: Union[str, Path, ClangInvocation, List[Union[str, Path, ClangInvocation]]],
copt: Optional[List[str]] = None,
system_includes: bool = True,
timeout: int = 600,
) -> Benchmark:
"""Create a benchmark for use by LLVM environments.
This function takes one or more inputs and uses them to create an LLVM
bitcode benchmark that can be passed to
:meth:`compiler_gym.envs.LlvmEnv.reset`.
The following input types are supported:
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
| **File Suffix** | **Treated as** | **Converted using** |
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
| :code:`.bc` | LLVM IR bitcode | No conversion required. |
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
| :code:`.ll` | LLVM IR text format | Assembled to bitcode using llvm-as. |
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
| :code:`.c`, :code:`.cc`, :code:`.cpp`, :code:`.cxx` | C / C++ source | Compiled to bitcode using clang and the given :code:`copt`. |
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
.. note::
The LLVM IR format has no compatability guarantees between versions (see
`LLVM docs
<https://llvm.org/docs/DeveloperPolicy.html#ir-backwards-compatibility>`_).
You must ensure that any :code:`.bc` and :code:`.ll` files are
compatible with the LLVM version used by CompilerGym, which can be
reported using :func:`env.compiler_version
<compiler_gym.envs.CompilerEnv.compiler_version>`.
E.g. for single-source C/C++ programs, you can pass the path of the source
file:
>>> benchmark = make_benchmark('my_app.c')
>>> env = gym.make("llvm-v0")
>>> env.reset(benchmark=benchmark)
The clang invocation used is roughly equivalent to:
.. code-block::
$ clang my_app.c -O0 -c -emit-llvm -o benchmark.bc
Additional compile-time arguments to clang can be provided using the
:code:`copt` argument:
>>> benchmark = make_benchmark('/path/to/my_app.cpp', copt=['-O2'])
If you need more fine-grained control over the options, you can directly
construct a :class:`ClangInvocation
<compiler_gym.envs.llvm.ClangInvocation>` to pass a list of arguments to
clang:
>>> benchmark = make_benchmark(
ClangInvocation(['/path/to/my_app.c'], system_includes=False, timeout=10)
)
For multi-file programs, pass a list of inputs that will be compiled
separately and then linked to a single module:
>>> benchmark = make_benchmark([
'main.c',
'lib.cpp',
'lib2.bc',
'foo/input.bc'
])
:param inputs: An input, or list of inputs.
:param copt: A list of command line options to pass to clang when compiling
source files.
:param system_includes: Whether to include the system standard libraries
during compilation jobs. This requires a system toolchain. See
:func:`get_system_library_flags`.
:param timeout: The maximum number of seconds to allow clang to run before
terminating.
:return: A :code:`Benchmark` instance.
:raises FileNotFoundError: If any input sources are not found.
:raises TypeError: If the inputs are of unsupported types.
:raises OSError: If a suitable compiler cannot be found.
:raises BenchmarkInitError: If a compilation job fails.
:raises TimeoutExpired: If a compilation job exceeds :code:`timeout`
seconds.
"""
copt = copt or []
bitcodes: List[Path] = []
clang_jobs: List[ClangInvocation] = []
ll_paths: List[Path] = []
def _add_path(path: Path):
if not path.is_file():
raise FileNotFoundError(path)
if path.suffix == ".bc":
bitcodes.append(path.absolute())
elif path.suffix in {".c", ".cc", ".cpp", ".cxx"}:
clang_jobs.append(
ClangInvocation.from_c_file(
path, copt=copt, system_includes=system_includes, timeout=timeout
)
)
elif path.suffix == ".ll":
ll_paths.append(path)
else:
raise ValueError(f"Unrecognized file type: {path.name}")
# Determine from inputs the list of pre-compiled bitcodes and the clang
# invocations required to compile the bitcodes.
if isinstance(inputs, str) or isinstance(inputs, Path):
_add_path(Path(inputs))
elif isinstance(inputs, ClangInvocation):
clang_jobs.append(inputs)
else:
for input in inputs:
if isinstance(input, str) or isinstance(input, Path):
_add_path(Path(input))
elif isinstance(input, ClangInvocation):
clang_jobs.append(input)
else:
raise TypeError(f"Invalid input type: {type(input).__name__}")
# Shortcut if we only have a single pre-compiled bitcode.
if len(bitcodes) == 1 and not clang_jobs and not ll_paths:
bitcode = bitcodes[0]
return Benchmark.from_file(uri=f"benchmark://file-v0{bitcode}", path=bitcode)
tmpdir_root = transient_cache_path(".")
tmpdir_root.mkdir(exist_ok=True, parents=True)
with tempfile.TemporaryDirectory(
dir=tmpdir_root, prefix="llvm-make_benchmark-"
) as d:
working_dir = Path(d)
clang_outs = [
working_dir / f"clang-out-{i}.bc" for i in range(1, len(clang_jobs) + 1)
]
llvm_as_outs = [
working_dir / f"llvm-as-out-{i}.bc" for i in range(1, len(ll_paths) + 1)
]
# Run the clang and llvm-as invocations in parallel. Avoid running this
# code path if possible as get_thread_pool_executor() requires locking.
if clang_jobs or ll_paths:
llvm_as_path = str(llvm.llvm_as_path())
executor = get_thread_pool_executor()
llvm_as_commands = [
[llvm_as_path, str(ll_path), "-o", bc_path]
for ll_path, bc_path in zip(ll_paths, llvm_as_outs)
]
# Fire off the clang and llvm-as jobs.
futures = [
executor.submit(run_command, job.command(out), job.timeout)
for job, out in zip(clang_jobs, clang_outs)
] + [
executor.submit(run_command, command, timeout)
for command in llvm_as_commands
]
# Block until finished.
list(future.result() for future in as_completed(futures))
# Check that the expected files were generated.
for clang_job, bc_path in zip(clang_jobs, clang_outs):
if not bc_path.is_file():
raise BenchmarkInitError(
f"clang failed: {' '.join(clang_job.command(bc_path))}"
)
for command, bc_path in zip(llvm_as_commands, llvm_as_outs):
if not bc_path.is_file():
raise BenchmarkInitError(f"llvm-as failed: {command}")
all_outs = bitcodes + clang_outs + llvm_as_outs
if not all_outs:
raise ValueError("No inputs")
elif len(all_outs) == 1:
# We only have a single bitcode so read it.
with open(str(all_outs[0]), "rb") as f:
bitcode = f.read()
else:
# Link all of the bitcodes into a single module.
llvm_link_cmd = [str(llvm.llvm_link_path()), "-o", "-"] + [
str(path) for path in bitcodes + clang_outs
]
with Popen(
llvm_link_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
) as llvm_link:
bitcode, stderr = llvm_link.communicate(timeout=timeout)
if llvm_link.returncode:
raise BenchmarkInitError(
f"Failed to link LLVM bitcodes with error: {stderr.decode('utf-8')}"
)
timestamp = datetime.now().strftime("%Y%m%HT%H%M%S")
uri = f"benchmark://user-v0/{timestamp}-{random.randrange(16**4):04x}"
return Benchmark.from_file_contents(uri, bitcode)
