def test_validate_known_bad_trajectory(env: LlvmEnv, state): env.apply(state) for _ in range(VALIDATION_FLAKINESS): result = env.validate() if result.okay(): pytest.fail( "Validation succeeded on state where it should have failed")
def test_fuzz(env: LlvmEnv): """This test generates a random trajectory and validates the semantics.""" benchmark = random.choice(VALIDATABLE_CBENCH_URIS) num_actions = random.randint(*RANDOM_TRAJECTORY_LENGTH_RANGE) print(benchmark) while True: env.reset(benchmark=benchmark) for _ in range(num_actions): _, _, done, _ = env.step(env.action_space.sample()) if done: break # Broken trajectory, retry. else: print(f"Validating state {env.state}") result = env.validate() assert result.okay(), result # Stop the test. break
def test_validate_known_good_trajectory(env: LlvmEnv, state): env.apply(state) for _ in range(VALIDATION_FLAKINESS): result = env.validate() if not result.okay(): pytest.fail(f"Validation failed: {result}\n{result.dict()}")