def test_add_validation_callbacks_call_count():
"""Test that custom validation callbacks are called on validate()."""
a_call_count = 0
b_call_count = 0
def a(env):
nonlocal a_call_count
a_call_count += 1
def b(env):
nonlocal b_call_count
b_call_count += 1
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == []
assert a_call_count == 1
assert b_call_count == 0
benchmark.add_validation_callback(b)
errors = benchmark.validate(env=None)
assert errors == []
assert a_call_count == 2
assert b_call_count == 1
def test_validation_callback_flaky():
"""Test error propagation on callback which *may* fail."""
flaky = False
def a(env):
nonlocal flaky
del env
if flaky:
yield ValidationError(type="Runtime Error")
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == []
flaky = True
errors = benchmark.validate(env=None)
assert errors == [
ValidationError(type="Runtime Error"),
]
def test_validation_callback_error():
"""Test error propagation from custom validation callback."""
def a(env):
yield ValidationError(type="Compilation Error")
yield ValidationError(type="Runtime Error")
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == [
ValidationError(type="Compilation Error"),
ValidationError(type="Runtime Error"),
]
