def test_benchmark_immutable():
"""Test that benchmark properties are immutable."""
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
with pytest.raises(AttributeError):
benchmark.uri = 123
with pytest.raises(AttributeError):
benchmark.proto = 123
def test_validation_callback_error_iter():
"""Test error propagation from custom validation callback using iterable."""
def a(env):
yield ValidationError(type="Compilation Error")
yield ValidationError(type="Runtime Error")
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.ivalidate(env=None)
next(errors) == ValidationError(type="Compilation Error")
next(errors) == ValidationError(type="Runtime Error")
def __init__(self, *args, **kwargs):
super().__init__(
name="benchmark://example-v0",
license="MIT",
description="An example dataset",
)
self._benchmarks = {
"/foo":
Benchmark.from_file_contents("benchmark://example-v0/foo",
"Ir data".encode("utf-8")),
"/bar":
Benchmark.from_file_contents("benchmark://example-v0/bar",
"Ir data".encode("utf-8")),
}
def test_validation_callback_error():
"""Test error propagation from custom validation callback."""
def a(env):
yield ValidationError(type="Compilation Error")
yield ValidationError(type="Runtime Error")
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == [
ValidationError(type="Compilation Error"),
ValidationError(type="Runtime Error"),
]
def test_add_validation_callbacks_call_count():
"""Test that custom validation callbacks are called on validate()."""
a_call_count = 0
b_call_count = 0
def a(env):
nonlocal a_call_count
a_call_count += 1
def b(env):
nonlocal b_call_count
b_call_count += 1
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == []
assert a_call_count == 1
assert b_call_count == 0
benchmark.add_validation_callback(b)
errors = benchmark.validate(env=None)
assert errors == []
assert a_call_count == 2
assert b_call_count == 1
def test_add_benchmark_invalid_path(env: CompilerEnv):
with tempfile.TemporaryDirectory() as d:
tmp = Path(d) / "not_a_file"
with pytest.raises(FileNotFoundError) as ctx:
env.reset(benchmark=Benchmark.from_file("benchmark://foo", tmp))
# Use endswith() because on macOS there may be a /private prefix.
assert str(ctx.value).endswith(str(tmp))
def benchmark_from_seed(self, seed: int) -> Benchmark:
"""Get a benchmark from a uint32 seed.
:param seed: A number in the range 0 <= n < 2^32.
:return: A benchmark instance.
"""
self.install()
# Run llvm-stress with the given seed and pipe the output to llvm-as to
# assemble a bitcode.
try:
with Popen(
[str(llvm.llvm_stress_path()), f"--seed={seed}"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as llvm_stress:
with Popen(
[str(llvm.llvm_as_path()), "-"],
stdin=llvm_stress.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as llvm_as:
stdout, _ = llvm_as.communicate(timeout=60)
llvm_stress.communicate(timeout=60)
if llvm_stress.returncode or llvm_as.returncode:
raise BenchmarkInitError(
"Failed to generate benchmark")
except subprocess.TimeoutExpired:
raise BenchmarkInitError("Benchmark generation timed out")
return Benchmark.from_file_contents(f"{self.name}/{seed}", stdout)
def test_invalid_benchmark_data(env: LlvmEnv):
benchmark = Benchmark.from_file_contents("benchmark://new",
"Invalid bitcode".encode("utf-8"))
with pytest.raises(
ValueError,
match='Failed to parse LLVM bitcode: "benchmark://new"'):
env.reset(benchmark=benchmark)
def test_add_benchmark_invalid_protocol(env: CompilerEnv):
with pytest.raises(ValueError) as ctx:
env.reset(benchmark=Benchmark(
BenchmarkProto(uri="benchmark://foo",
program=File(uri="https://invalid/protocol")), ))
assert str(ctx.value) == (
"Invalid benchmark data URI. "
'Only the file:/// protocol is supported: "https://invalid/protocol"')
def test_invalid_benchmark_missing_file(env: LlvmEnv):
benchmark = Benchmark(
BenchmarkProto(
uri="benchmark://new",
)
)
with pytest.raises(ValueError, match="No program set"):
env.reset(benchmark=benchmark)
def test_benchmark_path_empty_file(env: LlvmEnv):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
(tmpdir / "test.bc").touch()
benchmark = Benchmark.from_file("benchmark://new", tmpdir / "test.bc")
with pytest.raises(ValueError, match="Failed to parse LLVM bitcode"):
env.reset(benchmark=benchmark)
def test_invalid_benchmark_path_contents(env: LlvmEnv):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
with open(str(tmpdir / "test.bc"), "w") as f:
f.write("Invalid bitcode")
benchmark = Benchmark.from_file("benchmark://new", tmpdir / "test.bc")
with pytest.raises(ValueError, match="Failed to parse LLVM bitcode"):
env.reset(benchmark=benchmark)
def __init__(self, *args, **kwargs):
super().__init__(
name="benchmark://unrolling-v0",
license="MIT",
description="Unrolling example dataset",
)
self._benchmarks = {
"/offsets1":
Benchmark.from_file_contents(
"benchmark://unrolling-v0/offsets1",
self.preprocess(BENCHMARKS_PATH / "offsets1.c"),
),
"/conv2d":
Benchmark.from_file_contents(
"benchmark://unrolling-v0/conv2d",
self.preprocess(BENCHMARKS_PATH / "conv2d.c"),
),
}
def __init__(self, *args, **kwargs):
super().__init__(
name="benchmark://unrolling-v0",
license="MIT",
description="Unrolling example dataset",
site_data_base=site_data_path(
"example_dataset"
), # TODO: what should we set this to? we are not using it
)
self._benchmarks = {
"/offsets1": Benchmark.from_file_contents(
"benchmark://unrolling-v0/offsets1",
self.preprocess(BENCHMARKS_PATH / "offsets1.c"),
),
"/conv2d": Benchmark.from_file_contents(
"benchmark://unrolling-v0/conv2d",
self.preprocess(BENCHMARKS_PATH / "conv2d.c"),
),
}
def benchmark_from_flags() -> Optional[Union[Benchmark, str]]:
"""Returns either the name of the benchmark, or a Benchmark message."""
if FLAGS.benchmark:
if FLAGS.benchmark.startswith("file:///"):
path = Path(FLAGS.benchmark[len("file:///") :])
uri = f"benchmark://user-v0/{path}"
return Benchmark.from_file(uri=uri, path=path)
else:
return FLAGS.benchmark
else:
# No benchmark was specified.
return None
def test_benchmark_path_invalid_protocol(env: LlvmEnv):
benchmark = Benchmark(
BenchmarkProto(uri="benchmark://new",
program=File(uri="invalid_protocol://test")), )
with pytest.raises(
ValueError,
match=
("Invalid benchmark data URI. "
'Only the file:/// protocol is supported: "invalid_protocol://test"'),
):
env.reset(benchmark=benchmark)
def __init__(self, *args, **kwargs):
super().__init__(
name="benchmark://loops-opt-v0",
license="MIT",
description="Loops optimization dataset",
)
self._benchmarks = {
"benchmark://loops-opt-v0/add": Benchmark.from_file_contents(
"benchmark://loops-opt-v0/add",
self.preprocess(BENCHMARKS_PATH / "add.c"),
),
"benchmark://loops-opt-v0/offsets1": Benchmark.from_file_contents(
"benchmark://loops-opt-v0/offsets1",
self.preprocess(BENCHMARKS_PATH / "offsets1.c"),
),
"benchmark://loops-opt-v0/conv2d": Benchmark.from_file_contents(
"benchmark://loops-opt-v0/conv2d",
self.preprocess(BENCHMARKS_PATH / "conv2d.c"),
),
}
def test_dataset_equality_and_sorting():
"""Test comparison operators between datasets."""
a = Benchmark(BenchmarkProto(uri="benchmark://example-v0/a"))
a2 = Benchmark(BenchmarkProto(uri="benchmark://example-v0/a"))
b = Benchmark(BenchmarkProto(uri="benchmark://example-v0/b"))
assert a == a2
assert a != b
assert a < b
assert a <= b
assert b > a
assert b >= a
# String comparisons
assert a == "benchmark://example-v0/a"
assert a != "benchmark://example-v0/b"
assert a < "benchmark://example-v0/b"
# Sorting
assert sorted([a2, b, a]) == [
"benchmark://example-v0/a",
"benchmark://example-v0/a",
"benchmark://example-v0/b",
]
def benchmark(self, uri: str) -> Benchmark:
self.install()
benchmark_name = uri[len(self.name) + 1:]
if not benchmark_name:
raise LookupError(f"No benchmark specified: {uri}")
# Most of the source files are named after the parent directory, but not
# all.
c_file_name = {
"blowfish": "bf.c",
"motion": "mpeg2.c",
"sha": "sha_driver.c",
"jpeg": "main.c",
}.get(benchmark_name, f"{benchmark_name}.c")
source_dir_path = self.dataset_root / benchmark_name
source_path = source_dir_path / c_file_name
preprocessed_path = source_dir_path / "src.c"
# If the file does not exist, preprocess it on-demand.
if not preprocessed_path.is_file():
if not source_path.is_file():
raise LookupError(
f"Benchmark not found: {uri} (file not found: {source_path})"
)
with atomic_file_write(preprocessed_path) as tmp_path:
# TODO(github.com/facebookresearch/CompilerGym/issues/325): Send
# over the unprocessed code to the service, have the service
# preprocess. Until then, we do it client side with GCC having
# to fixed by an environment variable
self.gcc(
"-E",
"-o",
tmp_path.name,
c_file_name,
cwd=source_dir_path,
timeout=300,
)
return Benchmark.from_file(uri, preprocessed_path)
def test_benchmark_sources(tmpwd: Path):
a = Benchmark(
BenchmarkProto(uri="benchmark://example-v0/foo"),
sources=[("example.py", "Hello, world!".encode("utf-8"))],
)
a.add_source(BenchmarkSource(filename="foo.py", contents="Hi".encode("utf-8")))
assert list(a.sources) == [
BenchmarkSource("example.py", "Hello, world!".encode("utf-8")),
BenchmarkSource(filename="foo.py", contents="Hi".encode("utf-8")),
]
a.write_sources_to_directory("benchmark_sources")
with open(tmpwd / "benchmark_sources" / "example.py") as f:
assert f.read() == "Hello, world!"
with open(tmpwd / "benchmark_sources" / "foo.py") as f:
assert f.read() == "Hi"
def test_validation_callback_flaky():
"""Test error propagation on callback which *may* fail."""
flaky = False
def a(env):
nonlocal flaky
del env
if flaky:
yield ValidationError(type="Runtime Error")
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == []
flaky = True
errors = benchmark.validate(env=None)
assert errors == [
ValidationError(type="Runtime Error"),
]
def test_ne_benchmarks():
a = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foo"))
b = Benchmark(BenchmarkProto(uri="benchmark://example-v0/bar"))
assert a != b
def benchmark_from_parsed_uri(self, uri: BenchmarkUri) -> Benchmark:
return Benchmark(proto=benchmark.BenchmarkProto(uri=str(uri)))
def test_benchmark_attribute_outside_init():
"""Test that new attributes cannot be added to Benchmark."""
benchmark = Benchmark(None)
with pytest.raises(AttributeError):
# pylint: disable=assigning-non-slot
benchmark.foobar = 123 # noqa
def make_benchmark(
inputs: Union[str, Path, ClangInvocation, List[Union[str, Path, ClangInvocation]]],
copt: Optional[List[str]] = None,
system_includes: bool = True,
timeout: int = 600,
) -> Benchmark:
"""Create a benchmark for use by LLVM environments.
This function takes one or more inputs and uses them to create an LLVM
bitcode benchmark that can be passed to
:meth:`compiler_gym.envs.LlvmEnv.reset`.
The following input types are supported:
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
| **File Suffix** | **Treated as** | **Converted using** |
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
| :code:`.bc` | LLVM IR bitcode | No conversion required. |
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
| :code:`.ll` | LLVM IR text format | Assembled to bitcode using llvm-as. |
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
| :code:`.c`, :code:`.cc`, :code:`.cpp`, :code:`.cxx` | C / C++ source | Compiled to bitcode using clang and the given :code:`copt`. |
+-----------------------------------------------------+---------------------+-------------------------------------------------------------+
.. note::
The LLVM IR format has no compatability guarantees between versions (see
`LLVM docs
<https://llvm.org/docs/DeveloperPolicy.html#ir-backwards-compatibility>`_).
You must ensure that any :code:`.bc` and :code:`.ll` files are
compatible with the LLVM version used by CompilerGym, which can be
reported using :func:`env.compiler_version
<compiler_gym.envs.CompilerEnv.compiler_version>`.
E.g. for single-source C/C++ programs, you can pass the path of the source
file:
>>> benchmark = make_benchmark('my_app.c')
>>> env = gym.make("llvm-v0")
>>> env.reset(benchmark=benchmark)
The clang invocation used is roughly equivalent to:
.. code-block::
$ clang my_app.c -O0 -c -emit-llvm -o benchmark.bc
Additional compile-time arguments to clang can be provided using the
:code:`copt` argument:
>>> benchmark = make_benchmark('/path/to/my_app.cpp', copt=['-O2'])
If you need more fine-grained control over the options, you can directly
construct a :class:`ClangInvocation
<compiler_gym.envs.llvm.ClangInvocation>` to pass a list of arguments to
clang:
>>> benchmark = make_benchmark(
ClangInvocation(['/path/to/my_app.c'], system_includes=False, timeout=10)
)
For multi-file programs, pass a list of inputs that will be compiled
separately and then linked to a single module:
>>> benchmark = make_benchmark([
'main.c',
'lib.cpp',
'lib2.bc',
'foo/input.bc'
])
:param inputs: An input, or list of inputs.
:param copt: A list of command line options to pass to clang when compiling
source files.
:param system_includes: Whether to include the system standard libraries
during compilation jobs. This requires a system toolchain. See
:func:`get_system_library_flags`.
:param timeout: The maximum number of seconds to allow clang to run before
terminating.
:return: A :code:`Benchmark` instance.
:raises FileNotFoundError: If any input sources are not found.
:raises TypeError: If the inputs are of unsupported types.
:raises OSError: If a suitable compiler cannot be found.
:raises BenchmarkInitError: If a compilation job fails.
:raises TimeoutExpired: If a compilation job exceeds :code:`timeout`
seconds.
"""
copt = copt or []
bitcodes: List[Path] = []
clang_jobs: List[ClangInvocation] = []
ll_paths: List[Path] = []
def _add_path(path: Path):
if not path.is_file():
raise FileNotFoundError(path)
if path.suffix == ".bc":
bitcodes.append(path.absolute())
elif path.suffix in {".c", ".cc", ".cpp", ".cxx"}:
clang_jobs.append(
ClangInvocation.from_c_file(
path, copt=copt, system_includes=system_includes, timeout=timeout
)
)
elif path.suffix == ".ll":
ll_paths.append(path)
else:
raise ValueError(f"Unrecognized file type: {path.name}")
# Determine from inputs the list of pre-compiled bitcodes and the clang
# invocations required to compile the bitcodes.
if isinstance(inputs, str) or isinstance(inputs, Path):
_add_path(Path(inputs))
elif isinstance(inputs, ClangInvocation):
clang_jobs.append(inputs)
else:
for input in inputs:
if isinstance(input, str) or isinstance(input, Path):
_add_path(Path(input))
elif isinstance(input, ClangInvocation):
clang_jobs.append(input)
else:
raise TypeError(f"Invalid input type: {type(input).__name__}")
# Shortcut if we only have a single pre-compiled bitcode.
if len(bitcodes) == 1 and not clang_jobs and not ll_paths:
bitcode = bitcodes[0]
return Benchmark.from_file(uri=f"benchmark://file-v0{bitcode}", path=bitcode)
tmpdir_root = transient_cache_path(".")
tmpdir_root.mkdir(exist_ok=True, parents=True)
with tempfile.TemporaryDirectory(
dir=tmpdir_root, prefix="llvm-make_benchmark-"
) as d:
working_dir = Path(d)
clang_outs = [
working_dir / f"clang-out-{i}.bc" for i in range(1, len(clang_jobs) + 1)
]
llvm_as_outs = [
working_dir / f"llvm-as-out-{i}.bc" for i in range(1, len(ll_paths) + 1)
]
# Run the clang and llvm-as invocations in parallel. Avoid running this
# code path if possible as get_thread_pool_executor() requires locking.
if clang_jobs or ll_paths:
llvm_as_path = str(llvm.llvm_as_path())
executor = get_thread_pool_executor()
llvm_as_commands = [
[llvm_as_path, str(ll_path), "-o", bc_path]
for ll_path, bc_path in zip(ll_paths, llvm_as_outs)
]
# Fire off the clang and llvm-as jobs.
futures = [
executor.submit(run_command, job.command(out), job.timeout)
for job, out in zip(clang_jobs, clang_outs)
] + [
executor.submit(run_command, command, timeout)
for command in llvm_as_commands
]
# Block until finished.
list(future.result() for future in as_completed(futures))
# Check that the expected files were generated.
for clang_job, bc_path in zip(clang_jobs, clang_outs):
if not bc_path.is_file():
raise BenchmarkInitError(
f"clang failed: {' '.join(clang_job.command(bc_path))}"
)
for command, bc_path in zip(llvm_as_commands, llvm_as_outs):
if not bc_path.is_file():
raise BenchmarkInitError(f"llvm-as failed: {command}")
all_outs = bitcodes + clang_outs + llvm_as_outs
if not all_outs:
raise ValueError("No inputs")
elif len(all_outs) == 1:
# We only have a single bitcode so read it.
with open(str(all_outs[0]), "rb") as f:
bitcode = f.read()
else:
# Link all of the bitcodes into a single module.
llvm_link_cmd = [str(llvm.llvm_link_path()), "-o", "-"] + [
str(path) for path in bitcodes + clang_outs
]
with Popen(
llvm_link_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
) as llvm_link:
bitcode, stderr = llvm_link.communicate(timeout=timeout)
if llvm_link.returncode:
raise BenchmarkInitError(
f"Failed to link LLVM bitcodes with error: {stderr.decode('utf-8')}"
)
timestamp = datetime.now().strftime("%Y%m%HT%H%M%S")
uri = f"benchmark://user-v0/{timestamp}-{random.randrange(16**4):04x}"
return Benchmark.from_file_contents(uri, bitcode)
def test_ne_strings():
a = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foo"))
b = "benchmark://example-v0/bar"
assert a != b
def test_add_validation_callbacks_values():
"""Test methods for adding and checking custom validation callbacks."""
def a(env):
pass
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
assert benchmark.validation_callbacks() == []
assert not benchmark.is_validatable()
benchmark.add_validation_callback(a)
assert benchmark.validation_callbacks() == [a]
assert benchmark.is_validatable()
benchmark.add_validation_callback(a)
assert benchmark.validation_callbacks() == [a, a]
def test_benchmark_from_file(tmpwd: Path):
path = tmpwd / "foo.txt"
with open(path, "w") as f:
f.write("Hello, world!")
benchmark = Benchmark.from_file("benchmark://example-v0/foo", path)
assert benchmark.proto.program.contents.decode("utf-8") == "Hello, world!"
def test_benchmark_properties():
"""Test benchmark properties."""
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
assert benchmark.uri == "benchmark://example-v0/foobar"
assert benchmark.proto == BenchmarkProto(
uri="benchmark://example-v0/foobar")
def test_benchmark_from_file_not_found(tmpwd: Path):
path = tmpwd / "foo.txt"
with pytest.raises(FileNotFoundError, match=str(path)):
Benchmark.from_file("benchmark://example-v0/foo", path)
