def test_custom_benchmark_runtimes_differ(env: LlvmEnv, tmpdir):
"""Same as above, but test that runtimes differ from run to run."""
env.reset()
env.runtime_observation_count = 10
with open(tmpdir / "program.c", "w") as f:
f.write("""
#include <stdio.h>
int main(int argc, char** argv) {
printf("Hello\\n");
for (int i = 0; i < 10; ++i) {
argc += 2;
}
return argc - argc;
}
""")
benchmark = env.make_benchmark(Path(tmpdir) / "program.c")
benchmark.proto.dynamic_config.build_cmd.argument.extend(
["$CC", "$IN"] + llvm_benchmark.get_system_library_flags())
benchmark.proto.dynamic_config.build_cmd.outfile.extend(["a.out"])
benchmark.proto.dynamic_config.build_cmd.timeout_seconds = 10
benchmark.proto.dynamic_config.run_cmd.argument.extend(["./a.out"])
benchmark.proto.dynamic_config.run_cmd.timeout_seconds = 10
env.reset(benchmark=benchmark)
runtimes_a = env.observation.Runtime()
runtimes_b = env.observation.Runtime()
assert not np.all(runtimes_a == runtimes_b)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._src = None
self.proto.dynamic_config.MergeFrom(
BenchmarkDynamicConfig(
build_cmd=Command(
argument=["$CC", "$IN"] +
llvm_benchmark.get_system_library_flags(),
outfile=["a.out"],
timeout_seconds=60,
),
run_cmd=Command(
argument=["./a.out"],
timeout_seconds=300,
),
))
def preprocess(src: Path) -> bytes:
"""Front a C source through the compiler frontend."""
# TODO(github.com/facebookresearch/CompilerGym/issues/325): We can skip
# this pre-processing, or do it on the service side, once support for
# multi-file benchmarks lands.
cmd = [
str(llvm.clang_path()),
"-E",
"-o",
"-",
"-I",
str(NEURO_VECTORIZER_HEADER.parent),
src,
] + get_system_library_flags()
return subprocess.check_output(
cmd,
timeout=300,
)
def make_benchmark_from_command_line(
self,
cmd: Union[str, List[str]],
replace_driver: bool = True,
system_includes: bool = True,
timeout: int = 600,
) -> Benchmark:
"""Create a benchmark for use with this environment.
This function takes a command line compiler invocation as input,
modifies it to produce an unoptimized LLVM-IR bitcode, and then runs the
modified command line to produce a bitcode benchmark.
For example, the command line:
>>> benchmark = env.make_benchmark_from_command_line(
... ["gcc", "-DNDEBUG", "a.c", "b.c", "-o", "foo", "-lm"]
... )
Will compile a.c and b.c to an unoptimized benchmark that can be then
passed to :meth:`reset() <compiler_env.envs.CompilerEnv.reset>`.
The way this works is to change the first argument of the command line
invocation to the version of clang shipped with CompilerGym, and to then
append command line flags that causes the compiler to produce LLVM-IR
with optimizations disabled. For example the input command line:
.. code-block::
gcc -DNDEBUG a.c b.c -o foo -lm
Will be rewritten to be roughly equivalent to:
.. code-block::
/path/to/compiler_gym/clang -DNDEG a.c b.c \\
-Xclang -disable-llvm-passes -Xclang -disable-llvm-optzns \\ -c
-emit-llvm -o -
The generated benchmark then has a method :meth:`compile()
<compiler_env.envs.llvm.BenchmarkFromCommandLine.compile>` which
completes the linking and compilatilion to executable. For the above
example, this would be roughly equivalent to:
.. code-block::
/path/to/compiler_gym/clang environment-bitcode.bc -o foo -lm
:param cmd: A command line compiler invocation, either as a list of
arguments (e.g. :code:`["clang", "in.c"]`) or as a single shell
string (e.g. :code:`"clang in.c"`).
:param replace_driver: Whether to replace the first argument of the
command with the clang driver used by this environment.
:param system_includes: Whether to include the system standard libraries
during compilation jobs. This requires a system toolchain. See
:func:`get_system_library_flags`.
:param timeout: The maximum number of seconds to allow the compilation
job to run before terminating.
:return: A :class:`BenchmarkFromCommandLine
<compiler_gym.envs.llvm.BenchmarkFromCommandLine>` instance.
:raises ValueError: If no command line is provided.
:raises BenchmarkInitError: If executing the command line fails.
:raises TimeoutExpired: If a compilation job exceeds :code:`timeout`
seconds.
"""
if not cmd:
raise ValueError("Input command line is empty")
# Split the command line if passed a single string.
if isinstance(cmd, str):
cmd = shlex.split(cmd)
rewritten_cmd: List[str] = cmd.copy()
if len(cmd) < 2:
raise ValueError(
f"Input command line '{join_cmd(cmd)}' is too short")
# Append include flags for the system headers if requested.
if system_includes:
rewritten_cmd += get_system_library_flags()
# Use the CompilerGym clang binary in place of the original driver.
if replace_driver:
rewritten_cmd[0] = str(clang_path())
# Strip the -S flag, if present, as that changes the output format.
rewritten_cmd = [c for c in rewritten_cmd if c != "-S"]
invocation = GccInvocation(rewritten_cmd)
# Strip the output specifier(s). This is not strictly required since we
# override it later, but makes the generated command easier to
# understand.
for i in range(len(rewritten_cmd) - 2, -1, -1):
if rewritten_cmd[i] == "-o":
del rewritten_cmd[i + 1]
del rewritten_cmd[i]
# Fail early.
if "-" in invocation.sources:
raise ValueError("Input command line reads from stdin, "
f"which is not supported: '{join_cmd(cmd)}'")
# Convert all of the C/C++ sources to bitcodes which can then be linked
# into a single bitcode. We must process them individually because the
# '-c' flag does not support multiple sources when we are specifying the
# output path using '-o'.
sources = set(s for s in invocation.sources if not s.endswith(".o"))
if not sources:
raise ValueError(
f"Input command line has no source file inputs: '{join_cmd(cmd)}'"
)
bitcodes: List[bytes] = []
for source in sources:
# Adapt and execute the command line so that it will generate an
# unoptimized bitecode file.
emit_bitcode_command = rewritten_cmd.copy()
# Strip the name of other sources:
if len(sources) > 1:
emit_bitcode_command = [
c for c in emit_bitcode_command
if c == source or c not in sources
]
# Append the flags to emit the bitcode and disable the optimization
# passes.
emit_bitcode_command += [
"-c",
"-emit-llvm",
"-o",
"-",
"-Xclang",
"-disable-llvm-passes",
"-Xclang",
"-disable-llvm-optzns",
]
with Popen(emit_bitcode_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as clang:
logger.debug(
f"Generating LLVM bitcode benchmark: {join_cmd(emit_bitcode_command)}"
)
bitcode, stderr = clang.communicate(timeout=timeout)
if clang.returncode:
raise BenchmarkInitError(
f"Failed to generate LLVM bitcode with error:\n"
f"{stderr.decode('utf-8').rstrip()}\n"
f"Running command: {join_cmd(emit_bitcode_command)}\n"
f"From original commandline: {join_cmd(cmd)}")
bitcodes.append(bitcode)
# If there were multiple sources then link the bitcodes together.
if len(bitcodes) > 1:
with TemporaryDirectory(dir=transient_cache_path("."),
prefix="llvm-benchmark-") as dir:
# Write the bitcodes to files.
for i, bitcode in enumerate(bitcodes):
with open(os.path.join(dir, f"{i}.bc"), "wb") as f:
f.write(bitcode)
# Link the bitcode files.
llvm_link_cmd = [str(llvm_link_path()), "-o", "-"] + [
os.path.join(dir, f"{i}.bc") for i in range(len(bitcodes))
]
with Popen(llvm_link_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as llvm_link:
bitcode, stderr = llvm_link.communicate(timeout=timeout)
if llvm_link.returncode:
raise BenchmarkInitError(
f"Failed to link LLVM bitcodes with error: {stderr.decode('utf-8')}"
)
return BenchmarkFromCommandLine(invocation, bitcode, timeout)
def validator(
benchmark: str,
cmd: str,
data: Optional[List[str]] = None,
outs: Optional[List[str]] = None,
platforms: Optional[List[str]] = None,
compare_output: bool = True,
validate_result: Optional[Callable[[BenchmarkExecutionResult],
Optional[str]]] = None,
linkopts: Optional[List[str]] = None,
env: Optional[Dict[str, str]] = None,
pre_execution_callback: Optional[Callable[[], None]] = None,
sanitizers: Optional[List[LlvmSanitizer]] = None,
) -> bool:
"""Declare a new benchmark validator.
TODO(cummins): Pull this out into a public API.
:param benchmark: The name of the benchmark that this validator supports.
:cmd: The shell command to run the validation. Variable substitution is
applied to this value as follows: :code:`$BIN` is replaced by the path
of the compiled binary and :code:`$D` is replaced with the path to the
benchmark's runtime data directory.
:data: A list of paths to input files.
:outs: A list of paths to output files.
:return: :code:`True` if the new validator was registered, else :code:`False`.
"""
platforms = platforms or ["linux", "macos"]
if {"darwin": "macos"}.get(sys.platform, sys.platform) not in platforms:
return False
infiles = data or []
outfiles = [Path(p) for p in outs or []]
linkopts = linkopts or []
env = env or {}
if sanitizers is None:
sanitizers = LlvmSanitizer
VALIDATORS[benchmark].append(
_make_cBench_validator(
cmd=cmd,
input_files=infiles,
output_files=outfiles,
compare_output=compare_output,
validate_result=validate_result,
linkopts=linkopts,
os_env=env,
pre_execution_callback=pre_execution_callback,
))
# Register additional validators using the sanitizers.
if sys.platform.startswith("linux"):
for sanitizer in sanitizers:
VALIDATORS[benchmark].append(
_make_cBench_validator(
cmd=cmd,
input_files=infiles,
output_files=outfiles,
compare_output=compare_output,
validate_result=validate_result,
linkopts=linkopts,
os_env=env,
pre_execution_callback=pre_execution_callback,
sanitizer=sanitizer,
))
# Create the BenchmarkDynamicConfig object.
cbench_data = site_data_path("llvm-v0/cbench-v1-runtime-data/runtime_data")
uri = BenchmarkUri.from_string(benchmark)
DYNAMIC_CONFIGS[uri.path].append(
BenchmarkDynamicConfig(
build_cmd=Command(
argument=["$CC", "$IN"] +
llvm_benchmark.get_system_library_flags() + linkopts,
timeout_seconds=60,
outfile=["a.out"],
),
run_cmd=Command(
argument=cmd.replace("$BIN", "./a.out").replace(
"$D", str(cbench_data)).split(),
timeout_seconds=300,
infile=["a.out", "_finfo_dataset"],
outfile=[str(s) for s in outfiles],
),
pre_run_cmd=[
Command(argument=["echo", "1", ">_finfo_dataset"],
timeout_seconds=30),
],
))
return True