def test_benchmark_immutable(): """Test that benchmark properties are immutable.""" benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar")) with pytest.raises(AttributeError): benchmark.uri = 123 with pytest.raises(AttributeError): benchmark.proto = 123
def test_validation_callback_error_iter(): """Test error propagation from custom validation callback using iterable.""" def a(env): yield ValidationError(type="Compilation Error") yield ValidationError(type="Runtime Error") benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar")) benchmark.add_validation_callback(a) errors = benchmark.ivalidate(env=None) next(errors) == ValidationError(type="Compilation Error") next(errors) == ValidationError(type="Runtime Error")
def __init__(self, *args, **kwargs): super().__init__( name="benchmark://example-v0", license="MIT", description="An example dataset", ) self._benchmarks = { "/foo": Benchmark.from_file_contents("benchmark://example-v0/foo", "Ir data".encode("utf-8")), "/bar": Benchmark.from_file_contents("benchmark://example-v0/bar", "Ir data".encode("utf-8")), }
def test_validation_callback_error(): """Test error propagation from custom validation callback.""" def a(env): yield ValidationError(type="Compilation Error") yield ValidationError(type="Runtime Error") benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar")) benchmark.add_validation_callback(a) errors = benchmark.validate(env=None) assert errors == [ ValidationError(type="Compilation Error"), ValidationError(type="Runtime Error"), ]
def test_add_validation_callbacks_call_count(): """Test that custom validation callbacks are called on validate().""" a_call_count = 0 b_call_count = 0 def a(env): nonlocal a_call_count a_call_count += 1 def b(env): nonlocal b_call_count b_call_count += 1 benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar")) benchmark.add_validation_callback(a) errors = benchmark.validate(env=None) assert errors == [] assert a_call_count == 1 assert b_call_count == 0 benchmark.add_validation_callback(b) errors = benchmark.validate(env=None) assert errors == [] assert a_call_count == 2 assert b_call_count == 1
def test_add_benchmark_invalid_path(env: CompilerEnv): with tempfile.TemporaryDirectory() as d: tmp = Path(d) / "not_a_file" with pytest.raises(FileNotFoundError) as ctx: env.reset(benchmark=Benchmark.from_file("benchmark://foo", tmp)) # Use endswith() because on macOS there may be a /private prefix. assert str(ctx.value).endswith(str(tmp))
def benchmark_from_seed(self, seed: int) -> Benchmark: """Get a benchmark from a uint32 seed. :param seed: A number in the range 0 <= n < 2^32. :return: A benchmark instance. """ self.install() # Run llvm-stress with the given seed and pipe the output to llvm-as to # assemble a bitcode. try: with Popen( [str(llvm.llvm_stress_path()), f"--seed={seed}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) as llvm_stress: with Popen( [str(llvm.llvm_as_path()), "-"], stdin=llvm_stress.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) as llvm_as: stdout, _ = llvm_as.communicate(timeout=60) llvm_stress.communicate(timeout=60) if llvm_stress.returncode or llvm_as.returncode: raise BenchmarkInitError( "Failed to generate benchmark") except subprocess.TimeoutExpired: raise BenchmarkInitError("Benchmark generation timed out") return Benchmark.from_file_contents(f"{self.name}/{seed}", stdout)
def test_invalid_benchmark_data(env: LlvmEnv): benchmark = Benchmark.from_file_contents("benchmark://new", "Invalid bitcode".encode("utf-8")) with pytest.raises( ValueError, match='Failed to parse LLVM bitcode: "benchmark://new"'): env.reset(benchmark=benchmark)
def test_add_benchmark_invalid_protocol(env: CompilerEnv): with pytest.raises(ValueError) as ctx: env.reset(benchmark=Benchmark( BenchmarkProto(uri="benchmark://foo", program=File(uri="https://invalid/protocol")), )) assert str(ctx.value) == ( "Invalid benchmark data URI. " 'Only the file:/// protocol is supported: "https://invalid/protocol"')
def test_invalid_benchmark_missing_file(env: LlvmEnv): benchmark = Benchmark( BenchmarkProto( uri="benchmark://new", ) ) with pytest.raises(ValueError, match="No program set"): env.reset(benchmark=benchmark)
def test_benchmark_path_empty_file(env: LlvmEnv): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) (tmpdir / "test.bc").touch() benchmark = Benchmark.from_file("benchmark://new", tmpdir / "test.bc") with pytest.raises(ValueError, match="Failed to parse LLVM bitcode"): env.reset(benchmark=benchmark)
def test_invalid_benchmark_path_contents(env: LlvmEnv): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) with open(str(tmpdir / "test.bc"), "w") as f: f.write("Invalid bitcode") benchmark = Benchmark.from_file("benchmark://new", tmpdir / "test.bc") with pytest.raises(ValueError, match="Failed to parse LLVM bitcode"): env.reset(benchmark=benchmark)
def __init__(self, *args, **kwargs): super().__init__( name="benchmark://unrolling-v0", license="MIT", description="Unrolling example dataset", ) self._benchmarks = { "/offsets1": Benchmark.from_file_contents( "benchmark://unrolling-v0/offsets1", self.preprocess(BENCHMARKS_PATH / "offsets1.c"), ), "/conv2d": Benchmark.from_file_contents( "benchmark://unrolling-v0/conv2d", self.preprocess(BENCHMARKS_PATH / "conv2d.c"), ), }
def __init__(self, *args, **kwargs): super().__init__( name="benchmark://unrolling-v0", license="MIT", description="Unrolling example dataset", site_data_base=site_data_path( "example_dataset" ), # TODO: what should we set this to? we are not using it ) self._benchmarks = { "/offsets1": Benchmark.from_file_contents( "benchmark://unrolling-v0/offsets1", self.preprocess(BENCHMARKS_PATH / "offsets1.c"), ), "/conv2d": Benchmark.from_file_contents( "benchmark://unrolling-v0/conv2d", self.preprocess(BENCHMARKS_PATH / "conv2d.c"), ), }
def benchmark_from_flags() -> Optional[Union[Benchmark, str]]: """Returns either the name of the benchmark, or a Benchmark message.""" if FLAGS.benchmark: if FLAGS.benchmark.startswith("file:///"): path = Path(FLAGS.benchmark[len("file:///") :]) uri = f"benchmark://user-v0/{path}" return Benchmark.from_file(uri=uri, path=path) else: return FLAGS.benchmark else: # No benchmark was specified. return None
def test_benchmark_path_invalid_protocol(env: LlvmEnv): benchmark = Benchmark( BenchmarkProto(uri="benchmark://new", program=File(uri="invalid_protocol://test")), ) with pytest.raises( ValueError, match= ("Invalid benchmark data URI. " 'Only the file:/// protocol is supported: "invalid_protocol://test"'), ): env.reset(benchmark=benchmark)
def __init__(self, *args, **kwargs): super().__init__( name="benchmark://loops-opt-v0", license="MIT", description="Loops optimization dataset", ) self._benchmarks = { "benchmark://loops-opt-v0/add": Benchmark.from_file_contents( "benchmark://loops-opt-v0/add", self.preprocess(BENCHMARKS_PATH / "add.c"), ), "benchmark://loops-opt-v0/offsets1": Benchmark.from_file_contents( "benchmark://loops-opt-v0/offsets1", self.preprocess(BENCHMARKS_PATH / "offsets1.c"), ), "benchmark://loops-opt-v0/conv2d": Benchmark.from_file_contents( "benchmark://loops-opt-v0/conv2d", self.preprocess(BENCHMARKS_PATH / "conv2d.c"), ), }
def test_dataset_equality_and_sorting(): """Test comparison operators between datasets.""" a = Benchmark(BenchmarkProto(uri="benchmark://example-v0/a")) a2 = Benchmark(BenchmarkProto(uri="benchmark://example-v0/a")) b = Benchmark(BenchmarkProto(uri="benchmark://example-v0/b")) assert a == a2 assert a != b assert a < b assert a <= b assert b > a assert b >= a # String comparisons assert a == "benchmark://example-v0/a" assert a != "benchmark://example-v0/b" assert a < "benchmark://example-v0/b" # Sorting assert sorted([a2, b, a]) == [ "benchmark://example-v0/a", "benchmark://example-v0/a", "benchmark://example-v0/b", ]
def benchmark(self, uri: str) -> Benchmark: self.install() benchmark_name = uri[len(self.name) + 1:] if not benchmark_name: raise LookupError(f"No benchmark specified: {uri}") # Most of the source files are named after the parent directory, but not # all. c_file_name = { "blowfish": "bf.c", "motion": "mpeg2.c", "sha": "sha_driver.c", "jpeg": "main.c", }.get(benchmark_name, f"{benchmark_name}.c") source_dir_path = self.dataset_root / benchmark_name source_path = source_dir_path / c_file_name preprocessed_path = source_dir_path / "src.c" # If the file does not exist, preprocess it on-demand. if not preprocessed_path.is_file(): if not source_path.is_file(): raise LookupError( f"Benchmark not found: {uri} (file not found: {source_path})" ) with atomic_file_write(preprocessed_path) as tmp_path: # TODO(github.com/facebookresearch/CompilerGym/issues/325): Send # over the unprocessed code to the service, have the service # preprocess. Until then, we do it client side with GCC having # to fixed by an environment variable self.gcc( "-E", "-o", tmp_path.name, c_file_name, cwd=source_dir_path, timeout=300, ) return Benchmark.from_file(uri, preprocessed_path)
def test_benchmark_sources(tmpwd: Path): a = Benchmark( BenchmarkProto(uri="benchmark://example-v0/foo"), sources=[("example.py", "Hello, world!".encode("utf-8"))], ) a.add_source(BenchmarkSource(filename="foo.py", contents="Hi".encode("utf-8"))) assert list(a.sources) == [ BenchmarkSource("example.py", "Hello, world!".encode("utf-8")), BenchmarkSource(filename="foo.py", contents="Hi".encode("utf-8")), ] a.write_sources_to_directory("benchmark_sources") with open(tmpwd / "benchmark_sources" / "example.py") as f: assert f.read() == "Hello, world!" with open(tmpwd / "benchmark_sources" / "foo.py") as f: assert f.read() == "Hi"
def test_validation_callback_flaky(): """Test error propagation on callback which *may* fail.""" flaky = False def a(env): nonlocal flaky del env if flaky: yield ValidationError(type="Runtime Error") benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar")) benchmark.add_validation_callback(a) errors = benchmark.validate(env=None) assert errors == [] flaky = True errors = benchmark.validate(env=None) assert errors == [ ValidationError(type="Runtime Error"), ]
def test_ne_benchmarks(): a = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foo")) b = Benchmark(BenchmarkProto(uri="benchmark://example-v0/bar")) assert a != b
def benchmark_from_parsed_uri(self, uri: BenchmarkUri) -> Benchmark: return Benchmark(proto=benchmark.BenchmarkProto(uri=str(uri)))
def test_benchmark_attribute_outside_init(): """Test that new attributes cannot be added to Benchmark.""" benchmark = Benchmark(None) with pytest.raises(AttributeError): # pylint: disable=assigning-non-slot benchmark.foobar = 123 # noqa
def make_benchmark( inputs: Union[str, Path, ClangInvocation, List[Union[str, Path, ClangInvocation]]], copt: Optional[List[str]] = None, system_includes: bool = True, timeout: int = 600, ) -> Benchmark: """Create a benchmark for use by LLVM environments. This function takes one or more inputs and uses them to create an LLVM bitcode benchmark that can be passed to :meth:`compiler_gym.envs.LlvmEnv.reset`. The following input types are supported: +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ | **File Suffix** | **Treated as** | **Converted using** | +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ | :code:`.bc` | LLVM IR bitcode | No conversion required. | +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ | :code:`.ll` | LLVM IR text format | Assembled to bitcode using llvm-as. | +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ | :code:`.c`, :code:`.cc`, :code:`.cpp`, :code:`.cxx` | C / C++ source | Compiled to bitcode using clang and the given :code:`copt`. | +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ .. note:: The LLVM IR format has no compatability guarantees between versions (see `LLVM docs <https://llvm.org/docs/DeveloperPolicy.html#ir-backwards-compatibility>`_). You must ensure that any :code:`.bc` and :code:`.ll` files are compatible with the LLVM version used by CompilerGym, which can be reported using :func:`env.compiler_version <compiler_gym.envs.CompilerEnv.compiler_version>`. E.g. for single-source C/C++ programs, you can pass the path of the source file: >>> benchmark = make_benchmark('my_app.c') >>> env = gym.make("llvm-v0") >>> env.reset(benchmark=benchmark) The clang invocation used is roughly equivalent to: .. code-block:: $ clang my_app.c -O0 -c -emit-llvm -o benchmark.bc Additional compile-time arguments to clang can be provided using the :code:`copt` argument: >>> benchmark = make_benchmark('/path/to/my_app.cpp', copt=['-O2']) If you need more fine-grained control over the options, you can directly construct a :class:`ClangInvocation <compiler_gym.envs.llvm.ClangInvocation>` to pass a list of arguments to clang: >>> benchmark = make_benchmark( ClangInvocation(['/path/to/my_app.c'], system_includes=False, timeout=10) ) For multi-file programs, pass a list of inputs that will be compiled separately and then linked to a single module: >>> benchmark = make_benchmark([ 'main.c', 'lib.cpp', 'lib2.bc', 'foo/input.bc' ]) :param inputs: An input, or list of inputs. :param copt: A list of command line options to pass to clang when compiling source files. :param system_includes: Whether to include the system standard libraries during compilation jobs. This requires a system toolchain. See :func:`get_system_library_flags`. :param timeout: The maximum number of seconds to allow clang to run before terminating. :return: A :code:`Benchmark` instance. :raises FileNotFoundError: If any input sources are not found. :raises TypeError: If the inputs are of unsupported types. :raises OSError: If a suitable compiler cannot be found. :raises BenchmarkInitError: If a compilation job fails. :raises TimeoutExpired: If a compilation job exceeds :code:`timeout` seconds. """ copt = copt or [] bitcodes: List[Path] = [] clang_jobs: List[ClangInvocation] = [] ll_paths: List[Path] = [] def _add_path(path: Path): if not path.is_file(): raise FileNotFoundError(path) if path.suffix == ".bc": bitcodes.append(path.absolute()) elif path.suffix in {".c", ".cc", ".cpp", ".cxx"}: clang_jobs.append( ClangInvocation.from_c_file( path, copt=copt, system_includes=system_includes, timeout=timeout ) ) elif path.suffix == ".ll": ll_paths.append(path) else: raise ValueError(f"Unrecognized file type: {path.name}") # Determine from inputs the list of pre-compiled bitcodes and the clang # invocations required to compile the bitcodes. if isinstance(inputs, str) or isinstance(inputs, Path): _add_path(Path(inputs)) elif isinstance(inputs, ClangInvocation): clang_jobs.append(inputs) else: for input in inputs: if isinstance(input, str) or isinstance(input, Path): _add_path(Path(input)) elif isinstance(input, ClangInvocation): clang_jobs.append(input) else: raise TypeError(f"Invalid input type: {type(input).__name__}") # Shortcut if we only have a single pre-compiled bitcode. if len(bitcodes) == 1 and not clang_jobs and not ll_paths: bitcode = bitcodes[0] return Benchmark.from_file(uri=f"benchmark://file-v0{bitcode}", path=bitcode) tmpdir_root = transient_cache_path(".") tmpdir_root.mkdir(exist_ok=True, parents=True) with tempfile.TemporaryDirectory( dir=tmpdir_root, prefix="llvm-make_benchmark-" ) as d: working_dir = Path(d) clang_outs = [ working_dir / f"clang-out-{i}.bc" for i in range(1, len(clang_jobs) + 1) ] llvm_as_outs = [ working_dir / f"llvm-as-out-{i}.bc" for i in range(1, len(ll_paths) + 1) ] # Run the clang and llvm-as invocations in parallel. Avoid running this # code path if possible as get_thread_pool_executor() requires locking. if clang_jobs or ll_paths: llvm_as_path = str(llvm.llvm_as_path()) executor = get_thread_pool_executor() llvm_as_commands = [ [llvm_as_path, str(ll_path), "-o", bc_path] for ll_path, bc_path in zip(ll_paths, llvm_as_outs) ] # Fire off the clang and llvm-as jobs. futures = [ executor.submit(run_command, job.command(out), job.timeout) for job, out in zip(clang_jobs, clang_outs) ] + [ executor.submit(run_command, command, timeout) for command in llvm_as_commands ] # Block until finished. list(future.result() for future in as_completed(futures)) # Check that the expected files were generated. for clang_job, bc_path in zip(clang_jobs, clang_outs): if not bc_path.is_file(): raise BenchmarkInitError( f"clang failed: {' '.join(clang_job.command(bc_path))}" ) for command, bc_path in zip(llvm_as_commands, llvm_as_outs): if not bc_path.is_file(): raise BenchmarkInitError(f"llvm-as failed: {command}") all_outs = bitcodes + clang_outs + llvm_as_outs if not all_outs: raise ValueError("No inputs") elif len(all_outs) == 1: # We only have a single bitcode so read it. with open(str(all_outs[0]), "rb") as f: bitcode = f.read() else: # Link all of the bitcodes into a single module. llvm_link_cmd = [str(llvm.llvm_link_path()), "-o", "-"] + [ str(path) for path in bitcodes + clang_outs ] with Popen( llvm_link_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) as llvm_link: bitcode, stderr = llvm_link.communicate(timeout=timeout) if llvm_link.returncode: raise BenchmarkInitError( f"Failed to link LLVM bitcodes with error: {stderr.decode('utf-8')}" ) timestamp = datetime.now().strftime("%Y%m%HT%H%M%S") uri = f"benchmark://user-v0/{timestamp}-{random.randrange(16**4):04x}" return Benchmark.from_file_contents(uri, bitcode)
def test_ne_strings(): a = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foo")) b = "benchmark://example-v0/bar" assert a != b
def test_add_validation_callbacks_values(): """Test methods for adding and checking custom validation callbacks.""" def a(env): pass benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar")) assert benchmark.validation_callbacks() == [] assert not benchmark.is_validatable() benchmark.add_validation_callback(a) assert benchmark.validation_callbacks() == [a] assert benchmark.is_validatable() benchmark.add_validation_callback(a) assert benchmark.validation_callbacks() == [a, a]
def test_benchmark_from_file(tmpwd: Path): path = tmpwd / "foo.txt" with open(path, "w") as f: f.write("Hello, world!") benchmark = Benchmark.from_file("benchmark://example-v0/foo", path) assert benchmark.proto.program.contents.decode("utf-8") == "Hello, world!"
def test_benchmark_properties(): """Test benchmark properties.""" benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar")) assert benchmark.uri == "benchmark://example-v0/foobar" assert benchmark.proto == BenchmarkProto( uri="benchmark://example-v0/foobar")
def test_benchmark_from_file_not_found(tmpwd: Path): path = tmpwd / "foo.txt" with pytest.raises(FileNotFoundError, match=str(path)): Benchmark.from_file("benchmark://example-v0/foo", path)