def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._src = None
self.proto.dynamic_config.MergeFrom(
BenchmarkDynamicConfig(
build_cmd=Command(
argument=["$CC", "$IN"],
outfile=["benchmark_main"],
timeout_seconds=120,
),
run_cmd=Command(
argument=["./benchmark_main", "--benchmark_format=json"],
timeout_seconds=300,
),
))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._src = None
self.proto.dynamic_config.MergeFrom(
BenchmarkDynamicConfig(
build_cmd=Command(
argument=["$CC", "$IN"] +
llvm_benchmark.get_system_library_flags(),
outfile=["a.out"],
timeout_seconds=60,
),
run_cmd=Command(
argument=["./a.out"],
timeout_seconds=300,
),
))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
for val in VALIDATORS.get(self.uri, []):
self.add_validation_callback(val)
self.proto.dynamic_config.MergeFrom(
DYNAMIC_CONFIGS.get(self.uri, BenchmarkDynamicConfig()))
def validator(
benchmark: str,
cmd: str,
data: Optional[List[str]] = None,
outs: Optional[List[str]] = None,
platforms: Optional[List[str]] = None,
compare_output: bool = True,
validate_result: Optional[Callable[[BenchmarkExecutionResult],
Optional[str]]] = None,
linkopts: Optional[List[str]] = None,
env: Optional[Dict[str, str]] = None,
pre_execution_callback: Optional[Callable[[], None]] = None,
sanitizers: Optional[List[LlvmSanitizer]] = None,
) -> bool:
"""Declare a new benchmark validator.
TODO(cummins): Pull this out into a public API.
:param benchmark: The name of the benchmark that this validator supports.
:cmd: The shell command to run the validation. Variable substitution is
applied to this value as follows: :code:`$BIN` is replaced by the path
of the compiled binary and :code:`$D` is replaced with the path to the
benchmark's runtime data directory.
:data: A list of paths to input files.
:outs: A list of paths to output files.
:return: :code:`True` if the new validator was registered, else :code:`False`.
"""
platforms = platforms or ["linux", "macos"]
if {"darwin": "macos"}.get(sys.platform, sys.platform) not in platforms:
return False
infiles = data or []
outfiles = [Path(p) for p in outs or []]
linkopts = linkopts or []
env = env or {}
if sanitizers is None:
sanitizers = LlvmSanitizer
VALIDATORS[benchmark].append(
_make_cBench_validator(
cmd=cmd,
input_files=infiles,
output_files=outfiles,
compare_output=compare_output,
validate_result=validate_result,
linkopts=linkopts,
os_env=env,
pre_execution_callback=pre_execution_callback,
))
# Register additional validators using the sanitizers.
if sys.platform.startswith("linux"):
for sanitizer in sanitizers:
VALIDATORS[benchmark].append(
_make_cBench_validator(
cmd=cmd,
input_files=infiles,
output_files=outfiles,
compare_output=compare_output,
validate_result=validate_result,
linkopts=linkopts,
os_env=env,
pre_execution_callback=pre_execution_callback,
sanitizer=sanitizer,
))
# Create the BenchmarkDynamicConfig object.
cbench_data = site_data_path("llvm-v0/cbench-v1-runtime-data/runtime_data")
DYNAMIC_CONFIGS[benchmark] = BenchmarkDynamicConfig(
build_cmd=Command(
argument=["$CC", "$IN"] + linkopts,
timeout_seconds=60,
outfile=["a.out"],
),
run_cmd=Command(
argument=cmd.replace("$BIN",
"./a.out").replace("$D",
str(cbench_data)).split(),
timeout_seconds=300,
infile=["a.out", "_finfo_dataset"],
outfile=[str(s) for s in outfiles],
),
pre_run_cmd=[
Command(argument=["echo", "1", ">_finfo_dataset"],
timeout_seconds=30),
],
)
return True