def validate_benchmarks(benchmarks: List[str]): """Parses and validates list of benchmarks.""" benchmark_types = set() for benchmark in set(benchmarks): if benchmarks.count(benchmark) > 1: raise Exception('Benchmark "%s" is included more than once.' % benchmark) # Validate benchmarks here. It's possible someone might run an # experiment without going through presubmit. Better to catch an invalid # benchmark than see it in production. if not benchmark_utils.validate(benchmark): raise Exception('Benchmark "%s" is invalid.' % benchmark) benchmark_types.add(benchmark_utils.get_type(benchmark)) if (benchmark_utils.BenchmarkType.CODE.value in benchmark_types and benchmark_utils.BenchmarkType.BUG.value in benchmark_types): raise Exception( 'Cannot mix bug benchmarks with code coverage benchmarks.')
def validate_benchmark(self, path: Path): """Return True if |path| is part of a valid benchmark. Otherwise return False and print an error.""" benchmark = get_benchmark(path) if benchmark is None: return True if benchmark in self.invalid_benchmarks: # We know this is invalid and have already complained about it. return False valid = benchmark_utils.validate(benchmark) if valid: return True self.invalid_benchmarks.add(benchmark) print(benchmark, 'is not valid.') return False