def test_skip_optional_test_case(): """Check test_case skips a test when optional.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0, run_optional_tests=False) # Test is optional and validator should not be running optional tests, so it should # return hardcoded (None, "skipping optional"). output = dummy_test_case(validator, ({ "test": "dict" }, "message"), request="test_request", optional=True) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0 assert output[0] is None assert output[1] == "skipping optional" # Now check that the same test returns the correct values when not marked as optional output = dummy_test_case(validator, ({ "test": "dict" }, "message"), request="test_request", optional=False) assert validator.results.success_count == 1 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0 assert output[0] == {"test": "dict"} assert output[1] == "message"
def test_unexpected_failure_test_case(): """Check test_case catches unexpected errors as internal failures.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0) # Raise some unexpected exception and make sure it is logged as an internal error output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=FileNotFoundError("Unexpected error"), ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 1 assert output[0] is None assert output[1] == "FileNotFoundError: Unexpected error" assert ( validator.results.internal_failure_messages[-1][0] == "!: http://example.org/test_request - dummy_test_case - failed with internal error" ) assert validator.results.internal_failure_messages[-1][1] == [ "FileNotFoundError: Unexpected error" ] output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=FileNotFoundError("Unexpected error"), optional=True, ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 2 assert output[0] is None assert output[1] == "FileNotFoundError: Unexpected error" assert ( validator.results.internal_failure_messages[-1][0] == "!: http://example.org/test_request - dummy_test_case - failed with internal error" ) assert validator.results.internal_failure_messages[-1][1] == [ "FileNotFoundError: Unexpected error" ]
def test_normal_test_case(): """Check test_case under normal conditions.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0) output = dummy_test_case(validator, ([1, 2, 3], "message"), request="test_request") assert validator.results.success_count == 1 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0 assert output[0] == [1, 2, 3] assert output[1] == "message"
def test_ignored_test_case(): """Check test_case is ignored when receiving `None`.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0) # Test returns None, so should not increment success/failure output = dummy_test_case(validator, (None, "message"), request="test_request") assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0 assert output[0] is None assert output[1] == "message"
def test_optional_test_case(): """Check test_case for optional case.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0) output = dummy_test_case(validator, ("string response", "message"), request="test_request", optional=True) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 1 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0 assert output[0] == "string response" assert output[1] == "message"
def test_multistage_test_case(): """Check test_case's `multistage` functionality works as expected.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0) # Test that multistage requests do nothing but return unless they've failed output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", multistage=True, ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0 assert output[0] == {"test": "dict"} assert output[1] == "message" output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=ResponseError("Stage of test failed"), multistage=True, ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 1 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0 assert output[0] is None assert output[1] == "ResponseError: Stage of test failed" assert ( validator.results.failure_messages[-1][0] == "✖: http://example.org/test_request - dummy_test_case - failed with error" ) assert validator.results.failure_messages[-1][1] == [ "ResponseError: Stage of test failed" ]
def test_that_system_exit_is_fatal_in_test_case(): """Check that test_case treats `SystemExit` as fatal.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0, fail_fast=False) with pytest.raises(SystemExit, match="Fatal error"): dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=SystemExit("Fatal error"), optional=True, ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0
def test_fail_fast_test_case(): """Check test_case's `fail_fast` feature works as intended.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0, fail_fast=True) # Check that optional failures do not trigger fail fast output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=ResponseError("Optional test failed"), optional=True, ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == 1 assert validator.results.internal_failure_count == 0 assert output[0] is None assert output[1] == "ResponseError: Optional test failed" assert validator.results.optional_failure_messages[-1][0] == ( "http://example.org/test_request - dummy_test_case - failed with error" ) assert (validator.results.optional_failure_messages[-1][1] == "ResponseError: Optional test failed") # Check that the same non-optional failures do trigger fail fast with pytest.raises(SystemExit): output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=ResponseError("Non-optional test failed"), optional=False, ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 1 assert validator.results.optional_failure_count == 1 assert validator.results.internal_failure_count == 0 assert validator.results.failure_messages[-1][0] == ( "http://example.org/test_request - dummy_test_case - failed with error" ) assert (validator.results.failure_messages[-1][1] == "ResponseError: Non-optional test failed") # Check that an internal error also triggers fast with pytest.raises(SystemExit): output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=FileNotFoundError("Internal error"), ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 1 assert validator.results.optional_failure_count == 1 assert validator.results.internal_failure_count == 1 assert validator.results.internal_failure_messages[-1][0] == ( "http://example.org/test_request - dummy_test_case - failed with internal error" ) assert (validator.results.internal_failure_messages[-1][1] == "FileNotFoundError: Internal error")
def test_expected_failure_test_case(): """Check test_case reports a "failure" when `ResponseError` is raised.""" validator = ImplementationValidator(base_url="http://example.org", verbosity=0) # Test is optional and validator should not be running optional tests, so it should # return hardcoded (None, "skipping optional"). output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=ResponseError("Dummy error"), ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 1 assert validator.results.optional_failure_count == 0 assert validator.results.internal_failure_count == 0 assert output[0] is None assert output[1] == "ResponseError: Dummy error" assert ( validator.results.failure_messages[-1][0] == "http://example.org/test_request - dummy_test_case - failed with error" ) assert validator.results.failure_messages[-1][ 1] == "ResponseError: Dummy error" output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=ResponseError("Dummy error"), optional=True, ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 1 assert validator.results.optional_failure_count == 1 assert validator.results.internal_failure_count == 0 assert output[0] is None assert output[1] == "ResponseError: Dummy error" assert ( validator.results.optional_failure_messages[-1][0] == "http://example.org/test_request - dummy_test_case - failed with error" ) assert (validator.results.optional_failure_messages[-1][1] == "ResponseError: Dummy error") output = dummy_test_case( validator, ({ "test": "dict" }, "message"), request="test_request", raise_exception=json.JSONDecodeError("Dummy JSON error", "{}", 0), optional=True, ) assert validator.results.success_count == 0 assert validator.results.optional_success_count == 0 assert validator.results.failure_count == 1 assert validator.results.optional_failure_count == 2 assert validator.results.internal_failure_count == 0 assert output[0] is None assert ( output[1] == "Critical: unable to parse server response as JSON. JSONDecodeError: Dummy JSON error: line 1 column 1 (char 0)" ) assert ( validator.results.optional_failure_messages[-1][0] == "http://example.org/test_request - dummy_test_case - failed with error" ) assert ( validator.results.optional_failure_messages[-1][1] == "Critical: unable to parse server response as JSON. JSONDecodeError: Dummy JSON error: line 1 column 1 (char 0)" )
def test_versions_test_cases(): """Check the `/versions` test cases.""" from requests import Response from functools import partial unversioned_base_url = "https://example.org" versioned_base_url = unversioned_base_url + "/v1" def monkey_patched_get_endpoint(url, expected_status_code=200, content=b"version\n1", content_type="text/csv;header=present", **kwargs): r = Response() if expected_status_code == 200: r._content = content r.headers["content-type"] = content_type r.status_code = 200 else: r.status_code = 404 gets.append(url) return r, None # Test that the versioned base URL correctly tests the unversioned and then resets validator = ImplementationValidator(base_url=versioned_base_url, verbosity=5) gets = [] validator._get_endpoint = monkey_patched_get_endpoint validator._test_versions_endpoint() assert len(gets) == 2 assert validator.client.base_url == versioned_base_url assert validator.results.success_count == 1 assert validator.results.optional_success_count == 2 assert validator.results.failure_count == 0 # Test that the unversioned base URL correctly tests just the unversioned URL validator = ImplementationValidator(base_url=unversioned_base_url, verbosity=0) gets = [] validator._get_endpoint = monkey_patched_get_endpoint validator._test_versions_endpoint() assert len(gets) == 1 assert validator.client.base_url == unversioned_base_url assert validator.results.success_count == 1 assert validator.results.optional_success_count == 2 assert validator.results.failure_count == 0 # Test that bad content successfully triggers failures bad_content = (b"versions\n1", b"version\n", b"version\n1.1") for content in bad_content: validator = ImplementationValidator(base_url=versioned_base_url, verbosity=0) validator._get_endpoint = partial(monkey_patched_get_endpoint, content=content) validator._test_versions_endpoint() assert validator.client.base_url == versioned_base_url assert validator.results.success_count == 0 assert validator.results.failure_count == 1 assert validator.results.optional_failure_count == 0 # Test that missing content-type header triggers a hard failure validator = ImplementationValidator(base_url=versioned_base_url, verbosity=0) validator._get_endpoint = partial(monkey_patched_get_endpoint, content_type="") validator._test_versions_endpoint() assert validator.client.base_url == versioned_base_url assert validator.results.success_count == 0 assert validator.results.failure_count == 1 assert validator.results.optional_failure_count == 0 # Test that bad content-type headers successfully trigger *optional* failures bad_content_type = ( ("text/csv", 1), ("text/plain", 1), ("header=present", 1), ("x;y", 2), ("application/json;charset=utf-8", 2), ) for content_type, num_errors in bad_content_type: validator = ImplementationValidator(base_url=versioned_base_url, verbosity=0) validator._get_endpoint = partial(monkey_patched_get_endpoint, content_type=content_type) validator._test_versions_endpoint() assert validator.client.base_url == versioned_base_url assert validator.results.success_count == 1 assert validator.results.failure_count == 0 assert validator.results.optional_failure_count == num_errors