def test_skip_operations_with_recursive_references(schema_with_recursive_references): # When the test schema contains recursive references schema = oas_loaders.from_dict(schema_with_recursive_references) *_, after, finished = from_schema(schema).execute() # Then it causes an error with a proper error message assert after.status == Status.error assert RECURSIVE_REFERENCE_ERROR_MESSAGE in after.result.errors[0].exception
def test_graphql(graphql_url): schema = gql_loaders.from_url(graphql_url) initialized, *others, finished = list( from_schema(schema, hypothesis_settings=hypothesis.settings(max_examples=5, deadline=None)).execute() ) assert initialized.operations_count == 2 assert finished.passed_count == 2
def test_reproduce_code_with_overridden_headers(any_app_schema, base_url): # Note, headers are essentially the same, but keys are ordered differently due to implementation details of # real vs wsgi apps. It is the simplest solution, but not the most flexible one, though. if isinstance(any_app_schema.app, Flask): headers = { "User-Agent": USER_AGENT, "X-Token": "test", "Accept-Encoding": "gzip, deflate", "Accept": "*/*", "Connection": "keep-alive", } expected = f"requests.get('http://localhost/api/failure', headers={headers})" else: headers = { "User-Agent": USER_AGENT, "Accept-Encoding": "gzip, deflate", "Accept": "*/*", "Connection": "keep-alive", "X-Token": "test", } expected = f"requests.get('{base_url}/failure', headers={headers})" *_, after, finished = from_schema( any_app_schema, headers=headers, hypothesis_settings=hypothesis.settings(max_examples=1)).execute() assert finished.has_failures assert after.result.checks[1].example.requests_code == expected
def test_encoding_octet_stream(empty_open_api_3_schema, openapi3_base_url): # See: GH-1134 # When the operation contains the `application/octet-stream` media type # And has no `format: binary` in its schema empty_open_api_3_schema["paths"] = { "/data": { "post": { "requestBody": { "required": True, "content": { "application/octet-stream": { "schema": { "type": "string", }, }, }, }, "responses": {"200": {"description": "OK"}}, } } } schema = oas_loaders.from_dict(empty_open_api_3_schema, base_url=openapi3_base_url) initialized, before, after, finished = from_schema(schema).execute() # Then the test outcomes should not contain errors # And it should not lead to encoding errors assert after.status == Status.success assert not finished.has_failures assert not finished.has_errors
def test_response_conformance_invalid(any_app_schema): # When API operation returns a response that doesn't conform to the schema # And "response_schema_conformance" is specified init, *others, finished = from_schema( any_app_schema, checks=(response_schema_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1, deadline=None), ).execute() # Then there should be a failure assert finished.has_failures check = others[1].result.checks[-1] lines = check.message.split("\n") assert lines[0] == "The received response does not conform to the defined schema!" assert lines[2] == "Details: " validation_message = "'success' is a required property" assert lines[4] == validation_message assert check.context.instance == {"random": "key"} assert check.context.instance_path == [] assert check.context.schema == { "properties": {"success": {"type": "boolean"}}, "required": ["success"], "type": "object", } assert check.context.schema_path == ["required"] assert check.context.validation_message == validation_message
def test_serialize_interrupted(mocker, schema_url): mocker.patch( "schemathesis.runner.impl.solo.SingleThreadRunner._execute_impl", side_effect=KeyboardInterrupt) schema = schemathesis.from_uri(schema_url) events = from_schema(schema).execute() next(events) assert serialize_event(next(events)) == {"Interrupted": None}
def test_interrupted_in_test(openapi3_schema): # When an interrupt happens within a test body (check is called within a test body) def check(response, case): raise KeyboardInterrupt *_, event, finished = from_schema(openapi3_schema, checks=(check,)).execute() # Then the `Interrupted` event should be emitted assert isinstance(event, events.Interrupted)
def test_missing_path_parameter(any_app_schema): # When a path parameter is missing init, *others, finished = from_schema( any_app_schema, hypothesis_settings=hypothesis.settings(max_examples=3, deadline=None) ).execute() # Then it leads to an error assert finished.has_errors assert "InvalidSchema: Path parameter 'id' is not defined" in others[1].result.errors[0].exception
def test_empty_string_response_interaction(any_app_schema): # When there is a response that returns payload of length 0 init, *others, finished = from_schema(any_app_schema, store_interactions=True).execute() interactions = [event for event in others if isinstance(event, events.AfterExecution)][0].result.interactions for interaction in interactions: # There could be multiple calls # Then the stored response body should be an empty string assert interaction.response.body == "" assert interaction.response.encoding == "utf-8"
def test_invalid_path_parameter(schema_url): # When a path parameter is marked as not required # And schema validation is disabled schema = oas_loaders.from_uri(schema_url, validate_schema=False) init, *others, finished = from_schema(schema, hypothesis_settings=hypothesis.settings(max_examples=3)).execute() # Then Schemathesis enforces all path parameters to be required # And there should be no errors assert not finished.has_errors
def test_known_content_type(any_app_schema): # When API operation returns a response with a proper content type # And "content_type_conformance" is specified *_, finished = from_schema( any_app_schema, checks=(content_type_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1) ).execute() # Then there should be no a failures assert not finished.has_failures
def test_interactions(request, any_app_schema, workers): init, *others, finished = from_schema(any_app_schema, workers_num=workers, store_interactions=True).execute() base_url = ( "http://localhost/api" if isinstance(any_app_schema.app, Flask) else request.getfixturevalue("openapi3_base_url") ) # failure interactions = [ event for event in others if isinstance(event, events.AfterExecution) and event.status == Status.failure ][0].result.interactions assert len(interactions) == 2 failure = interactions[0] assert attr.asdict(failure.request) == { "uri": f"{base_url}/failure", "method": "GET", "body": None, "headers": { "Accept": ["*/*"], "Accept-Encoding": ["gzip, deflate"], "Connection": ["keep-alive"], "User-Agent": [USER_AGENT], }, } assert failure.response.status_code == 500 assert failure.response.message == "Internal Server Error" if isinstance(any_app_schema.app, Flask): assert failure.response.headers == {"Content-Type": ["text/html; charset=utf-8"], "Content-Length": ["290"]} else: assert failure.response.headers["Content-Type"] == ["text/plain; charset=utf-8"] assert failure.response.headers["Content-Length"] == ["26"] # success interactions = [ event for event in others if isinstance(event, events.AfterExecution) and event.status == Status.success ][0].result.interactions assert len(interactions) == 1 success = interactions[0] assert attr.asdict(success.request) == { "uri": f"{base_url}/success", "method": "GET", "body": None, "headers": { "Accept": ["*/*"], "Accept-Encoding": ["gzip, deflate"], "Connection": ["keep-alive"], "User-Agent": [USER_AGENT], }, } assert success.response.status_code == 200 assert success.response.message == "OK" assert json.loads(base64.b64decode(success.response.body)) == {"success": True} assert success.response.encoding == "utf-8" if isinstance(any_app_schema.app, Flask): assert success.response.headers == {"Content-Type": ["application/json"], "Content-Length": ["17"]} else: assert success.response.headers["Content-Type"] == ["application/json; charset=utf-8"]
def test_empty_response_interaction(any_app_schema): # When there is a GET request and a response that doesn't return content (e.g. 204) init, *others, finished = from_schema(any_app_schema, store_interactions=True).execute() interactions = [event for event in others if isinstance(event, events.AfterExecution)][0].result.interactions for interaction in interactions: # There could be multiple calls # Then the stored request has no body assert interaction.request.body is None # And its response as well assert interaction.response.body is None # And response encoding is missing assert interaction.response.encoding is None
def test_interrupted_outside_test(mocker, openapi3_schema): # See GH-1325 # When an interrupt happens outside of a test body mocker.patch("schemathesis.runner.events.AfterExecution.from_result", side_effect=KeyboardInterrupt) try: *_, event, finished = from_schema(openapi3_schema).execute() # Then the `Interrupted` event should be emitted assert isinstance(event, events.Interrupted) except KeyboardInterrupt: pytest.fail("KeyboardInterrupt should be handled")
def test_unknown_content_type(any_app_schema): # When API operation returns a response with content type, not specified in "produces" # And "content_type_conformance" is specified init, *others, finished = from_schema( any_app_schema, checks=(content_type_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1) ).execute() # Then there should be a failure assert finished.has_failures check = others[1].result.checks[0] assert check.name == "content_type_conformance" assert check.value == Status.failure
def test_unknown_response_code_with_default(any_app_schema): # When API operation returns a status code, that is not listed in "responses", but there is a "default" response # And "status_code_conformance" is specified init, *others, finished = from_schema( any_app_schema, checks=(status_code_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1) ).execute() # Then there should be no failure assert not finished.has_failures check = others[1].result.checks[0] assert check.name == "status_code_conformance" assert check.value == Status.success
def test_response_conformance_invalid(any_app_schema): # When API operation returns a response that doesn't conform to the schema # And "response_schema_conformance" is specified init, *others, finished = from_schema( any_app_schema, checks=(response_schema_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1) ).execute() # Then there should be a failure assert finished.has_failures lines = others[1].result.checks[-1].message.split("\n") assert lines[0] == "The received response does not conform to the defined schema!" assert lines[2] == "Details: " assert lines[4] == "'success' is a required property"
def test_serialize_event(schema_url): schema = schemathesis.from_uri(schema_url) events = from_schema(schema).execute() next(events) next(events) event = serialize_event(next(events)) assert "interactions" not in event["AfterExecution"]["result"] assert "logs" not in event["AfterExecution"]["result"] assert event["AfterExecution"]["result"]["checks"][0]["example"][ "query"] == { "id": ["0"] }
def test_url_joining(request, server, get_schema_path, schema_path): if schema_path == "petstore_v2.yaml": base_url = request.getfixturevalue("openapi2_base_url") else: base_url = request.getfixturevalue("openapi3_base_url") path = get_schema_path(schema_path) schema = oas_loaders.from_path(path, base_url=f"{base_url}/v3", endpoint="/pet/findByStatus") *_, after_execution, _ = from_schema(schema, hypothesis_settings=hypothesis.settings(max_examples=1)).execute() assert after_execution.result.path == "/api/v3/pet/findByStatus" assert ( f"http://127.0.0.1:{server['port']}/api/v3/pet/findByStatus" in after_execution.result.checks[0].example.requests_code )
def test_response_conformance_malformed_json(any_app_schema): # When API operation returns a response that contains a malformed JSON, but has a valid content type header # And "response_schema_conformance" is specified init, *others, finished = from_schema( any_app_schema, checks=(response_schema_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1) ).execute() # Then there should be a failure assert finished.has_failures assert not finished.has_errors message = others[1].result.checks[-1].message assert "The received response is not valid JSON:" in message assert "{malformed}" in message assert "Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" in message
def test_internal_exceptions(any_app_schema, mocker): # GH: #236 # When there is an exception during the test # And Hypothesis consider this test as a flaky one mocker.patch("schemathesis.Case.call", side_effect=ValueError) mocker.patch("schemathesis.Case.call_wsgi", side_effect=ValueError) init, *others, finished = from_schema( any_app_schema, hypothesis_settings=hypothesis.settings(max_examples=3, deadline=None) ).execute() # Then the execution result should indicate errors assert finished.has_errors # And an error from the buggy code should be collected exceptions = [i.exception.strip() for i in others[1].result.errors] assert "ValueError" in exceptions assert len(exceptions) == 1
def test_unknown_response_code(any_app_schema): # When API operation returns a status code, that is not listed in "responses" # And "status_code_conformance" is specified init, *others, finished = from_schema( any_app_schema, checks=(status_code_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1, deadline=None), ).execute() # Then there should be a failure assert finished.has_failures check = others[1].result.checks[0] assert check.name == "status_code_conformance" assert check.value == Status.failure assert check.context.status_code == 418 assert check.context.allowed_status_codes == [200] assert check.context.defined_status_codes == ["200"]
def test_headers_override(any_app_schema): def check_headers(response, case): if isinstance(any_app_schema.app, Flask): data = response.json else: data = response.json() assert data["X-Token"] == "test" init, *others, finished = from_schema( any_app_schema, checks=(check_headers,), headers={"X-Token": "test"}, hypothesis_settings=hypothesis.settings(max_examples=1, deadline=None), ).execute() assert not finished.has_failures assert not finished.has_errors
def test_hypothesis_errors_propagation(empty_open_api_3_schema, openapi3_base_url): # See: GH-1046 # When the operation contains a media type, that Schemathesis can't serialize # And there is still a supported media type empty_open_api_3_schema["paths"] = { "/data": { "post": { "requestBody": { "required": True, "content": { # This one is known "application/json": { "schema": { "type": "array", }, }, # This one is not "application/xml": { "schema": { "type": "array", } }, }, }, "responses": {"200": {"description": "OK"}}, } } } max_examples = 10 schema = oas_loaders.from_dict(empty_open_api_3_schema, base_url=openapi3_base_url) initialized, before, after, finished = from_schema( schema, hypothesis_settings=hypothesis.settings(max_examples=max_examples, deadline=None) ).execute() # Then the test outcomes should not contain errors assert after.status == Status.success # And there should be requested amount of test examples assert len(after.result.checks) == max_examples assert not finished.has_failures assert not finished.has_errors
def test_unsatisfiable_example(empty_open_api_3_schema): # See GH-904 # When filling missing properties during examples generation leads to unsatisfiable schemas empty_open_api_3_schema["paths"] = { "/success": { "post": { "parameters": [ # This parameter is not satisfiable { "name": "key", "in": "query", "required": True, "schema": {"type": "integer", "minimum": 5, "maximum": 4}, } ], "requestBody": { "content": { "application/json": { "schema": { "type": "object", "properties": { "foo": {"type": "string", "example": "foo example string"}, }, }, } } }, "responses": {"200": {"description": "OK"}}, } } } # Then the testing process should not raise an internal error schema = oas_loaders.from_dict(empty_open_api_3_schema) *_, after, finished = from_schema( schema, hypothesis_settings=hypothesis.settings(max_examples=1, deadline=None) ).execute() # And the tests are failing because of the unsatisfiable schema assert finished.has_errors assert "Unable to satisfy schema parameters for this API operation" in after.result.errors[0].exception
def test_asgi_interactions(fastapi_app): schema = oas_loaders.from_asgi("/openapi.json", fastapi_app) init, *ev, finished = from_schema(schema, store_interactions=True).execute() interaction = ev[1].result.interactions[0] assert interaction.status == Status.success assert interaction.request.uri == "http://testserver/users"
def test_count_operations(real_app_schema): # When `count_operations` is set to `False` event = next(from_schema(real_app_schema, count_operations=False).execute()) # Then the total number of operations is not calculated in the `Initialized` event assert event.operations_count is None
def test_exit_first(any_app_schema): results = list(from_schema(any_app_schema, exit_first=True).execute()) assert results[-1].has_failures is True assert results[-1].failed_count == 1
def test_exceptions(schema_url, app, loader_options, from_schema_options): schema = oas_loaders.from_uri(schema_url, **loader_options) results = from_schema(schema, **from_schema_options).execute() assert any([event.status == Status.error for event in results if isinstance(event, events.AfterExecution)])
def execute(schema, **options) -> events.Finished: *_, last = from_schema(schema, **options).execute() return last