def test_add_validation_callbacks_call_count():
"""Test that custom validation callbacks are called on validate()."""
a_call_count = 0
b_call_count = 0
def a(env):
nonlocal a_call_count
a_call_count += 1
def b(env):
nonlocal b_call_count
b_call_count += 1
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == []
assert a_call_count == 1
assert b_call_count == 0
benchmark.add_validation_callback(b)
errors = benchmark.validate(env=None)
assert errors == []
assert a_call_count == 2
assert b_call_count == 1
def test_validation_callback_error_iter():
"""Test error propagation from custom validation callback using iterable."""
def a(env):
yield ValidationError(type="Compilation Error")
yield ValidationError(type="Runtime Error")
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.ivalidate(env=None)
next(errors) == ValidationError(type="Compilation Error")
next(errors) == ValidationError(type="Runtime Error")
def test_validation_callback_error():
"""Test error propagation from custom validation callback."""
def a(env):
yield ValidationError(type="Compilation Error")
yield ValidationError(type="Runtime Error")
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == [
ValidationError(type="Compilation Error"),
ValidationError(type="Runtime Error"),
]
def test_add_validation_callbacks_values():
"""Test methods for adding and checking custom validation callbacks."""
def a(env):
pass
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
assert benchmark.validation_callbacks() == []
assert not benchmark.is_validatable()
benchmark.add_validation_callback(a)
assert benchmark.validation_callbacks() == [a]
assert benchmark.is_validatable()
benchmark.add_validation_callback(a)
assert benchmark.validation_callbacks() == [a, a]
def test_validation_callback_flaky():
"""Test error propagation on callback which *may* fail."""
flaky = False
def a(env):
nonlocal flaky
del env
if flaky:
yield ValidationError(type="Runtime Error")
benchmark = Benchmark(BenchmarkProto(uri="benchmark://example-v0/foobar"))
benchmark.add_validation_callback(a)
errors = benchmark.validate(env=None)
assert errors == []
flaky = True
errors = benchmark.validate(env=None)
assert errors == [
ValidationError(type="Runtime Error"),
]
