def test_skip_operations_with_recursive_references(schema_with_recursive_references):
# When the test schema contains recursive references
schema = oas_loaders.from_dict(schema_with_recursive_references)
*_, after, finished = from_schema(schema).execute()
# Then it causes an error with a proper error message
assert after.status == Status.error
assert RECURSIVE_REFERENCE_ERROR_MESSAGE in after.result.errors[0].exception
def test_graphql(graphql_url):
schema = gql_loaders.from_url(graphql_url)
initialized, *others, finished = list(
from_schema(schema, hypothesis_settings=hypothesis.settings(max_examples=5, deadline=None)).execute()
)
assert initialized.operations_count == 2
assert finished.passed_count == 2
def test_reproduce_code_with_overridden_headers(any_app_schema, base_url):
# Note, headers are essentially the same, but keys are ordered differently due to implementation details of
# real vs wsgi apps. It is the simplest solution, but not the most flexible one, though.
if isinstance(any_app_schema.app, Flask):
headers = {
"User-Agent": USER_AGENT,
"X-Token": "test",
"Accept-Encoding": "gzip, deflate",
"Accept": "*/*",
"Connection": "keep-alive",
}
expected = f"requests.get('http://localhost/api/failure', headers={headers})"
else:
headers = {
"User-Agent": USER_AGENT,
"Accept-Encoding": "gzip, deflate",
"Accept": "*/*",
"Connection": "keep-alive",
"X-Token": "test",
}
expected = f"requests.get('{base_url}/failure', headers={headers})"
*_, after, finished = from_schema(
any_app_schema,
headers=headers,
hypothesis_settings=hypothesis.settings(max_examples=1)).execute()
assert finished.has_failures
assert after.result.checks[1].example.requests_code == expected
def test_encoding_octet_stream(empty_open_api_3_schema, openapi3_base_url):
# See: GH-1134
# When the operation contains the `application/octet-stream` media type
# And has no `format: binary` in its schema
empty_open_api_3_schema["paths"] = {
"/data": {
"post": {
"requestBody": {
"required": True,
"content": {
"application/octet-stream": {
"schema": {
"type": "string",
},
},
},
},
"responses": {"200": {"description": "OK"}},
}
}
}
schema = oas_loaders.from_dict(empty_open_api_3_schema, base_url=openapi3_base_url)
initialized, before, after, finished = from_schema(schema).execute()
# Then the test outcomes should not contain errors
# And it should not lead to encoding errors
assert after.status == Status.success
assert not finished.has_failures
assert not finished.has_errors
def test_response_conformance_invalid(any_app_schema):
# When API operation returns a response that doesn't conform to the schema
# And "response_schema_conformance" is specified
init, *others, finished = from_schema(
any_app_schema,
checks=(response_schema_conformance,),
hypothesis_settings=hypothesis.settings(max_examples=1, deadline=None),
).execute()
# Then there should be a failure
assert finished.has_failures
check = others[1].result.checks[-1]
lines = check.message.split("\n")
assert lines[0] == "The received response does not conform to the defined schema!"
assert lines[2] == "Details: "
validation_message = "'success' is a required property"
assert lines[4] == validation_message
assert check.context.instance == {"random": "key"}
assert check.context.instance_path == []
assert check.context.schema == {
"properties": {"success": {"type": "boolean"}},
"required": ["success"],
"type": "object",
}
assert check.context.schema_path == ["required"]
assert check.context.validation_message == validation_message
def test_serialize_interrupted(mocker, schema_url):
mocker.patch(
"schemathesis.runner.impl.solo.SingleThreadRunner._execute_impl",
side_effect=KeyboardInterrupt)
schema = schemathesis.from_uri(schema_url)
events = from_schema(schema).execute()
next(events)
assert serialize_event(next(events)) == {"Interrupted": None}
def test_interrupted_in_test(openapi3_schema):
# When an interrupt happens within a test body (check is called within a test body)
def check(response, case):
raise KeyboardInterrupt
*_, event, finished = from_schema(openapi3_schema, checks=(check,)).execute()
# Then the `Interrupted` event should be emitted
assert isinstance(event, events.Interrupted)
def test_missing_path_parameter(any_app_schema):
# When a path parameter is missing
init, *others, finished = from_schema(
any_app_schema, hypothesis_settings=hypothesis.settings(max_examples=3, deadline=None)
).execute()
# Then it leads to an error
assert finished.has_errors
assert "InvalidSchema: Path parameter 'id' is not defined" in others[1].result.errors[0].exception
def test_empty_string_response_interaction(any_app_schema):
# When there is a response that returns payload of length 0
init, *others, finished = from_schema(any_app_schema, store_interactions=True).execute()
interactions = [event for event in others if isinstance(event, events.AfterExecution)][0].result.interactions
for interaction in interactions: # There could be multiple calls
# Then the stored response body should be an empty string
assert interaction.response.body == ""
assert interaction.response.encoding == "utf-8"
def test_invalid_path_parameter(schema_url):
# When a path parameter is marked as not required
# And schema validation is disabled
schema = oas_loaders.from_uri(schema_url, validate_schema=False)
init, *others, finished = from_schema(schema, hypothesis_settings=hypothesis.settings(max_examples=3)).execute()
# Then Schemathesis enforces all path parameters to be required
# And there should be no errors
assert not finished.has_errors
def test_known_content_type(any_app_schema):
# When API operation returns a response with a proper content type
# And "content_type_conformance" is specified
*_, finished = from_schema(
any_app_schema, checks=(content_type_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1)
).execute()
# Then there should be no a failures
assert not finished.has_failures
def test_interactions(request, any_app_schema, workers):
init, *others, finished = from_schema(any_app_schema, workers_num=workers, store_interactions=True).execute()
base_url = (
"http://localhost/api"
if isinstance(any_app_schema.app, Flask)
else request.getfixturevalue("openapi3_base_url")
)
# failure
interactions = [
event for event in others if isinstance(event, events.AfterExecution) and event.status == Status.failure
][0].result.interactions
assert len(interactions) == 2
failure = interactions[0]
assert attr.asdict(failure.request) == {
"uri": f"{base_url}/failure",
"method": "GET",
"body": None,
"headers": {
"Accept": ["*/*"],
"Accept-Encoding": ["gzip, deflate"],
"Connection": ["keep-alive"],
"User-Agent": [USER_AGENT],
},
}
assert failure.response.status_code == 500
assert failure.response.message == "Internal Server Error"
if isinstance(any_app_schema.app, Flask):
assert failure.response.headers == {"Content-Type": ["text/html; charset=utf-8"], "Content-Length": ["290"]}
else:
assert failure.response.headers["Content-Type"] == ["text/plain; charset=utf-8"]
assert failure.response.headers["Content-Length"] == ["26"]
# success
interactions = [
event for event in others if isinstance(event, events.AfterExecution) and event.status == Status.success
][0].result.interactions
assert len(interactions) == 1
success = interactions[0]
assert attr.asdict(success.request) == {
"uri": f"{base_url}/success",
"method": "GET",
"body": None,
"headers": {
"Accept": ["*/*"],
"Accept-Encoding": ["gzip, deflate"],
"Connection": ["keep-alive"],
"User-Agent": [USER_AGENT],
},
}
assert success.response.status_code == 200
assert success.response.message == "OK"
assert json.loads(base64.b64decode(success.response.body)) == {"success": True}
assert success.response.encoding == "utf-8"
if isinstance(any_app_schema.app, Flask):
assert success.response.headers == {"Content-Type": ["application/json"], "Content-Length": ["17"]}
else:
assert success.response.headers["Content-Type"] == ["application/json; charset=utf-8"]
def test_empty_response_interaction(any_app_schema):
# When there is a GET request and a response that doesn't return content (e.g. 204)
init, *others, finished = from_schema(any_app_schema, store_interactions=True).execute()
interactions = [event for event in others if isinstance(event, events.AfterExecution)][0].result.interactions
for interaction in interactions: # There could be multiple calls
# Then the stored request has no body
assert interaction.request.body is None
# And its response as well
assert interaction.response.body is None
# And response encoding is missing
assert interaction.response.encoding is None
def test_interrupted_outside_test(mocker, openapi3_schema):
# See GH-1325
# When an interrupt happens outside of a test body
mocker.patch("schemathesis.runner.events.AfterExecution.from_result", side_effect=KeyboardInterrupt)
try:
*_, event, finished = from_schema(openapi3_schema).execute()
# Then the `Interrupted` event should be emitted
assert isinstance(event, events.Interrupted)
except KeyboardInterrupt:
pytest.fail("KeyboardInterrupt should be handled")
def test_unknown_content_type(any_app_schema):
# When API operation returns a response with content type, not specified in "produces"
# And "content_type_conformance" is specified
init, *others, finished = from_schema(
any_app_schema, checks=(content_type_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1)
).execute()
# Then there should be a failure
assert finished.has_failures
check = others[1].result.checks[0]
assert check.name == "content_type_conformance"
assert check.value == Status.failure
def test_unknown_response_code_with_default(any_app_schema):
# When API operation returns a status code, that is not listed in "responses", but there is a "default" response
# And "status_code_conformance" is specified
init, *others, finished = from_schema(
any_app_schema, checks=(status_code_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1)
).execute()
# Then there should be no failure
assert not finished.has_failures
check = others[1].result.checks[0]
assert check.name == "status_code_conformance"
assert check.value == Status.success
def test_response_conformance_invalid(any_app_schema):
# When API operation returns a response that doesn't conform to the schema
# And "response_schema_conformance" is specified
init, *others, finished = from_schema(
any_app_schema, checks=(response_schema_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1)
).execute()
# Then there should be a failure
assert finished.has_failures
lines = others[1].result.checks[-1].message.split("\n")
assert lines[0] == "The received response does not conform to the defined schema!"
assert lines[2] == "Details: "
assert lines[4] == "'success' is a required property"
def test_serialize_event(schema_url):
schema = schemathesis.from_uri(schema_url)
events = from_schema(schema).execute()
next(events)
next(events)
event = serialize_event(next(events))
assert "interactions" not in event["AfterExecution"]["result"]
assert "logs" not in event["AfterExecution"]["result"]
assert event["AfterExecution"]["result"]["checks"][0]["example"][
"query"] == {
"id": ["0"]
}
def test_url_joining(request, server, get_schema_path, schema_path):
if schema_path == "petstore_v2.yaml":
base_url = request.getfixturevalue("openapi2_base_url")
else:
base_url = request.getfixturevalue("openapi3_base_url")
path = get_schema_path(schema_path)
schema = oas_loaders.from_path(path, base_url=f"{base_url}/v3", endpoint="/pet/findByStatus")
*_, after_execution, _ = from_schema(schema, hypothesis_settings=hypothesis.settings(max_examples=1)).execute()
assert after_execution.result.path == "/api/v3/pet/findByStatus"
assert (
f"http://127.0.0.1:{server['port']}/api/v3/pet/findByStatus"
in after_execution.result.checks[0].example.requests_code
)
def test_response_conformance_malformed_json(any_app_schema):
# When API operation returns a response that contains a malformed JSON, but has a valid content type header
# And "response_schema_conformance" is specified
init, *others, finished = from_schema(
any_app_schema, checks=(response_schema_conformance,), hypothesis_settings=hypothesis.settings(max_examples=1)
).execute()
# Then there should be a failure
assert finished.has_failures
assert not finished.has_errors
message = others[1].result.checks[-1].message
assert "The received response is not valid JSON:" in message
assert "{malformed}" in message
assert "Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" in message
def test_internal_exceptions(any_app_schema, mocker):
# GH: #236
# When there is an exception during the test
# And Hypothesis consider this test as a flaky one
mocker.patch("schemathesis.Case.call", side_effect=ValueError)
mocker.patch("schemathesis.Case.call_wsgi", side_effect=ValueError)
init, *others, finished = from_schema(
any_app_schema, hypothesis_settings=hypothesis.settings(max_examples=3, deadline=None)
).execute()
# Then the execution result should indicate errors
assert finished.has_errors
# And an error from the buggy code should be collected
exceptions = [i.exception.strip() for i in others[1].result.errors]
assert "ValueError" in exceptions
assert len(exceptions) == 1
def test_unknown_response_code(any_app_schema):
# When API operation returns a status code, that is not listed in "responses"
# And "status_code_conformance" is specified
init, *others, finished = from_schema(
any_app_schema,
checks=(status_code_conformance,),
hypothesis_settings=hypothesis.settings(max_examples=1, deadline=None),
).execute()
# Then there should be a failure
assert finished.has_failures
check = others[1].result.checks[0]
assert check.name == "status_code_conformance"
assert check.value == Status.failure
assert check.context.status_code == 418
assert check.context.allowed_status_codes == [200]
assert check.context.defined_status_codes == ["200"]
def test_headers_override(any_app_schema):
def check_headers(response, case):
if isinstance(any_app_schema.app, Flask):
data = response.json
else:
data = response.json()
assert data["X-Token"] == "test"
init, *others, finished = from_schema(
any_app_schema,
checks=(check_headers,),
headers={"X-Token": "test"},
hypothesis_settings=hypothesis.settings(max_examples=1, deadline=None),
).execute()
assert not finished.has_failures
assert not finished.has_errors
def test_hypothesis_errors_propagation(empty_open_api_3_schema, openapi3_base_url):
# See: GH-1046
# When the operation contains a media type, that Schemathesis can't serialize
# And there is still a supported media type
empty_open_api_3_schema["paths"] = {
"/data": {
"post": {
"requestBody": {
"required": True,
"content": {
# This one is known
"application/json": {
"schema": {
"type": "array",
},
},
# This one is not
"application/xml": {
"schema": {
"type": "array",
}
},
},
},
"responses": {"200": {"description": "OK"}},
}
}
}
max_examples = 10
schema = oas_loaders.from_dict(empty_open_api_3_schema, base_url=openapi3_base_url)
initialized, before, after, finished = from_schema(
schema, hypothesis_settings=hypothesis.settings(max_examples=max_examples, deadline=None)
).execute()
# Then the test outcomes should not contain errors
assert after.status == Status.success
# And there should be requested amount of test examples
assert len(after.result.checks) == max_examples
assert not finished.has_failures
assert not finished.has_errors
def test_unsatisfiable_example(empty_open_api_3_schema):
# See GH-904
# When filling missing properties during examples generation leads to unsatisfiable schemas
empty_open_api_3_schema["paths"] = {
"/success": {
"post": {
"parameters": [
# This parameter is not satisfiable
{
"name": "key",
"in": "query",
"required": True,
"schema": {"type": "integer", "minimum": 5, "maximum": 4},
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"foo": {"type": "string", "example": "foo example string"},
},
},
}
}
},
"responses": {"200": {"description": "OK"}},
}
}
}
# Then the testing process should not raise an internal error
schema = oas_loaders.from_dict(empty_open_api_3_schema)
*_, after, finished = from_schema(
schema, hypothesis_settings=hypothesis.settings(max_examples=1, deadline=None)
).execute()
# And the tests are failing because of the unsatisfiable schema
assert finished.has_errors
assert "Unable to satisfy schema parameters for this API operation" in after.result.errors[0].exception
def test_asgi_interactions(fastapi_app):
schema = oas_loaders.from_asgi("/openapi.json", fastapi_app)
init, *ev, finished = from_schema(schema, store_interactions=True).execute()
interaction = ev[1].result.interactions[0]
assert interaction.status == Status.success
assert interaction.request.uri == "http://testserver/users"
def test_count_operations(real_app_schema):
# When `count_operations` is set to `False`
event = next(from_schema(real_app_schema, count_operations=False).execute())
# Then the total number of operations is not calculated in the `Initialized` event
assert event.operations_count is None
def test_exit_first(any_app_schema):
results = list(from_schema(any_app_schema, exit_first=True).execute())
assert results[-1].has_failures is True
assert results[-1].failed_count == 1
def test_exceptions(schema_url, app, loader_options, from_schema_options):
schema = oas_loaders.from_uri(schema_url, **loader_options)
results = from_schema(schema, **from_schema_options).execute()
assert any([event.status == Status.error for event in results if isinstance(event, events.AfterExecution)])
def execute(schema, **options) -> events.Finished:
*_, last = from_schema(schema, **options).execute()
return last
