def test_validate_known_bad_trajectory(env: LlvmEnv, state):
env.apply(state)
for _ in range(VALIDATION_FLAKINESS):
result = env.validate()
if result.okay():
pytest.fail(
"Validation succeeded on state where it should have failed")
def test_fuzz(env: LlvmEnv):
"""This test generates a random trajectory and validates the semantics."""
benchmark = random.choice(VALIDATABLE_CBENCH_URIS)
num_actions = random.randint(*RANDOM_TRAJECTORY_LENGTH_RANGE)
print(benchmark)
while True:
env.reset(benchmark=benchmark)
for _ in range(num_actions):
_, _, done, _ = env.step(env.action_space.sample())
if done:
break # Broken trajectory, retry.
else:
print(f"Validating state {env.state}")
result = env.validate()
assert result.okay(), result
# Stop the test.
break
def test_validate_known_good_trajectory(env: LlvmEnv, state):
env.apply(state)
for _ in range(VALIDATION_FLAKINESS):
result = env.validate()
if not result.okay():
pytest.fail(f"Validation failed: {result}\n{result.dict()}")
