def validate_benchmarks(benchmarks: List[str]):
"""Parses and validates list of benchmarks."""
benchmark_types = set()
for benchmark in set(benchmarks):
if benchmarks.count(benchmark) > 1:
raise Exception('Benchmark "%s" is included more than once.' %
benchmark)
# Validate benchmarks here. It's possible someone might run an
# experiment without going through presubmit. Better to catch an invalid
# benchmark than see it in production.
if not benchmark_utils.validate(benchmark):
raise Exception('Benchmark "%s" is invalid.' % benchmark)
benchmark_types.add(benchmark_utils.get_type(benchmark))
if (benchmark_utils.BenchmarkType.CODE.value in benchmark_types
and benchmark_utils.BenchmarkType.BUG.value in benchmark_types):
raise Exception(
'Cannot mix bug benchmarks with code coverage benchmarks.')
def validate_benchmark(self, path: Path):
"""Return True if |path| is part of a valid benchmark. Otherwise return
False and print an error."""
benchmark = get_benchmark(path)
if benchmark is None:
return True
if benchmark in self.invalid_benchmarks:
# We know this is invalid and have already complained about it.
return False
valid = benchmark_utils.validate(benchmark)
if valid:
return True
self.invalid_benchmarks.add(benchmark)
print(benchmark, 'is not valid.')
return False