def test_custom_loader(swagger_20, openapi2_base_url):
swagger_20.base_url = openapi2_base_url
# Custom loaders are not validated
*others, finished = list(
prepare({}, loader=lambda *args, **kwargs: swagger_20))
assert not finished.has_errors
assert not finished.has_failures
def test_known_content_type(args):
app, kwargs = args
# When endpoint returns a response with a proper content type
# And "content_type_conformance" is specified
*_, finished = prepare(**kwargs, checks=(content_type_conformance,), hypothesis_max_examples=1)
# Then there should be no a failures
assert not finished.has_failures
def test_asgi_interactions(loadable_fastapi_app):
init, *ev, finished = prepare(
"/openapi.json", app=loadable_fastapi_app, loader=loaders.from_asgi, store_interactions=True
)
interaction = ev[1].result.interactions[0]
assert interaction.status == Status.success
assert interaction.request.uri == "http://testserver/users"
def test_reproduce_code_with_overridden_headers(args, openapi3_base_url):
app, kwargs = args
# Note, headers are essentially the same, but keys are ordered differently due to implementation details of
# real vs wsgi apps. It is the simplest solution, but not the most flexible one, though.
if isinstance(app, Flask):
headers = {
"User-Agent": USER_AGENT,
"X-Token": "test",
"Accept-Encoding": "gzip, deflate",
"Accept": "*/*",
"Connection": "keep-alive",
}
expected = f"requests.get('http://localhost/api/failure', headers={headers})"
else:
headers = {
"User-Agent": USER_AGENT,
"Accept-Encoding": "gzip, deflate",
"Accept": "*/*",
"Connection": "keep-alive",
"X-Token": "test",
}
expected = f"requests.get('{openapi3_base_url}/failure', headers={headers})"
*_, after, finished = prepare(**kwargs,
headers=headers,
hypothesis_max_examples=1)
assert finished.has_failures
assert after.result.checks[1].example.requests_code == expected
def test_invalid_path_parameter(args):
app, kwargs = args
init, *others, finished = prepare(validate_schema=False, **kwargs)
assert finished.has_errors
assert (
"schemathesis.exceptions.InvalidSchema: Missing path parameter 'id'. "
"It either not defined or has no `required: true` in its definition"
in others[1].result.errors[0].exception)
def test_invalid_path_parameter(args):
# When a path parameter is marked as not required
app, kwargs = args
# And schema validation is disabled
init, *others, finished = prepare(validate_schema=False, hypothesis_max_examples=3, **kwargs)
# Then Schemathesis enforces all path parameters to be required
# And there should be no errors
assert not finished.has_errors
def test_missing_path_parameter(args):
# When a path parameter is missing
app, kwargs = args
init, *others, finished = prepare(hypothesis_max_examples=3, **kwargs)
# Then it leads to an error
assert finished.has_errors
assert "InvalidSchema: Path parameter 'id' is not defined" in others[
1].result.errors[0].exception
def test_response_conformance_malformed_json(args):
app, kwargs = args
# When endpoint returns a response that contains a malformed JSON, but has a valid content type header
# And "response_schema_conformance" is specified
init, *others, finished = prepare(**kwargs, checks=(response_schema_conformance,), hypothesis_max_examples=1)
# Then there should be a failure
assert finished.has_errors
error = others[1].result.errors[-1].exception
assert "Expecting property name enclosed in double quotes" in error
def test_skip_operations_with_recursive_references(
schema_with_recursive_references):
# When the test schema contains recursive references
*_, after, finished = prepare(schema_with_recursive_references,
loader=loaders.from_dict)
# Then it causes an error with a proper error message
assert after.status == Status.error
assert RECURSIVE_REFERENCE_ERROR_MESSAGE in after.result.errors[
0].exception
def test_url_joining(server, get_schema_path, base_url, schema_path):
path = get_schema_path(schema_path)
*_, after_execution, _ = prepare(path,
base_url=f"{base_url}/v3",
endpoint="/pet/findByStatus",
hypothesis_max_examples=1)
assert after_execution.result.path == "/api/v3/pet/findByStatus"
assert (f"http://127.0.0.1:{server['port']}/api/v3/pet/findByStatus"
in after_execution.result.checks[0].example.requests_code)
def test_unknown_content_type(args):
app, kwargs = args
# When endpoint returns a response with content type, not specified in "produces"
# And "content_type_conformance" is specified
init, *others, finished = prepare(**kwargs, checks=(content_type_conformance,), hypothesis_max_examples=1)
# Then there should be a failure
assert finished.has_failures
check = others[1].result.checks[0]
assert check.name == "content_type_conformance"
assert check.value == Status.failure
def test_unknown_response_code_with_default(args):
app, kwargs = args
# When endpoint returns a status code, that is not listed in "responses", but there is a "default" response
# And "status_code_conformance" is specified
init, *others, finished = prepare(**kwargs, checks=(status_code_conformance,), hypothesis_max_examples=1)
# Then there should be no failure
assert not finished.has_failures
check = others[1].result.checks[0]
assert check.name == "status_code_conformance"
assert check.value == Status.success
def execute(schema_uri,
checks=DEFAULT_CHECKS,
loader=from_uri,
**options) -> events.Finished:
generator = prepare(schema_uri=schema_uri,
checks=checks,
loader=loader,
**options)
all_events = list(generator)
return all_events[-1]
def test_response_conformance_invalid(args):
app, kwargs = args
# When endpoint returns a response that doesn't conform to the schema
# And "response_schema_conformance" is specified
init, *others, finished = prepare(**kwargs, checks=(response_schema_conformance,), hypothesis_max_examples=1)
# Then there should be a failure
assert finished.has_failures
lines = others[1].result.checks[-1].message.split("\n")
assert lines[0] == "The received response does not conform to the defined schema!"
assert lines[2] == "Details: "
assert lines[4] == "'success' is a required property"
def test_reproduce_code_with_overridden_headers(args, openapi3_base_url):
app, kwargs = args
headers = {"User-Agent": USER_AGENT, "X-Token": "test"}
*_, after, finished = prepare(**kwargs, headers=headers, hypothesis_max_examples=1)
assert finished.has_failures
if isinstance(app, Flask):
expected = f"requests.get('http://localhost/api/failure', headers={headers})"
else:
expected = f"requests.get('{openapi3_base_url}/failure', headers={headers})"
assert after.result.checks[1].example.requests_code == expected
def test_empty_string_response_interaction(openapi_version, args):
# When there is a response that returns payload of length 0
app, kwargs = args
init, *others, finished = prepare(**kwargs, store_interactions=True)
interactions = [
event for event in others if isinstance(event, events.AfterExecution)
][0].result.interactions
for interaction in interactions: # There could be multiple calls
# Then the stored response body should be an empty string
assert interaction.response.body == ""
assert interaction.response.encoding == "utf-8"
def test_unknown_response_code(args):
app, kwargs = args
# When API operation returns a status code, that is not listed in "responses"
# And "status_code_conformance" is specified
init, *others, finished = prepare(**kwargs,
checks=(status_code_conformance, ),
hypothesis_max_examples=1)
# Then there should be a failure
assert finished.has_failures
check = others[1].result.checks[0]
assert check.name == "status_code_conformance"
assert check.value == Status.failure
def test_interactions(request, args, workers):
app, kwargs = args
init, *others, finished = prepare(**kwargs, workers_num=workers, store_interactions=True)
base_url = "http://localhost/api" if isinstance(app, Flask) else request.getfixturevalue("openapi3_base_url")
# failure
interactions = [
event for event in others if isinstance(event, events.AfterExecution) and event.status == Status.failure
][0].result.interactions
assert len(interactions) == 2
failure = interactions[0]
assert attr.asdict(failure.request) == {
"uri": f"{base_url}/failure",
"method": "GET",
"body": "",
"headers": {
"Accept": ["*/*"],
"Accept-Encoding": ["gzip, deflate"],
"Connection": ["keep-alive"],
"User-Agent": [USER_AGENT],
},
}
assert failure.response.status_code == 500
assert failure.response.message == "Internal Server Error"
if isinstance(app, Flask):
assert failure.response.headers == {"Content-Type": ["text/html; charset=utf-8"], "Content-Length": ["290"]}
else:
assert failure.response.headers["Content-Type"] == ["text/plain; charset=utf-8"]
assert failure.response.headers["Content-Length"] == ["26"]
# success
interactions = [
event for event in others if isinstance(event, events.AfterExecution) and event.status == Status.success
][0].result.interactions
assert len(interactions) == 1
success = interactions[0]
assert attr.asdict(success.request) == {
"uri": f"{base_url}/success",
"method": "GET",
"body": "",
"headers": {
"Accept": ["*/*"],
"Accept-Encoding": ["gzip, deflate"],
"Connection": ["keep-alive"],
"User-Agent": [USER_AGENT],
},
}
assert success.response.status_code == 200
assert success.response.message == "OK"
assert json.loads(base64.b64decode(success.response.body)) == {"success": True}
if isinstance(app, Flask):
assert success.response.headers == {"Content-Type": ["application/json"], "Content-Length": ["17"]}
else:
assert success.response.headers["Content-Type"] == ["application/json; charset=utf-8"]
def test_non_default_loader(openapi_version, request, loader, fixture):
schema = request.getfixturevalue(fixture)
kwargs = {}
if loader is loaders.from_wsgi:
kwargs["app"] = request.getfixturevalue("loadable_flask_app")
else:
if loader is loaders.from_aiohttp:
kwargs["app"] = request.getfixturevalue("loadable_aiohttp_app")
kwargs["base_url"] = request.getfixturevalue("base_url")
init, *others, finished = prepare(schema, loader=loader, headers={"TEST": "foo"}, **kwargs)
assert not finished.has_errors
assert not finished.has_failures
def test_response_conformance_malformed_json(args):
app, kwargs = args
# When endpoint returns a response that contains a malformed JSON, but has a valid content type header
# And "response_schema_conformance" is specified
init, *others, finished = prepare(**kwargs, checks=(response_schema_conformance,), hypothesis_max_examples=1)
# Then there should be a failure
assert finished.has_failures
assert not finished.has_errors
message = others[1].result.checks[-1].message
assert "The received response is not valid JSON:" in message
assert "{malformed}" in message
assert "Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" in message
def test_url_joining(request, server, get_schema_path, schema_path):
if schema_path == "petstore_v2.yaml":
base_url = request.getfixturevalue("openapi2_base_url")
else:
base_url = request.getfixturevalue("openapi3_base_url")
path = get_schema_path(schema_path)
*_, after_execution, _ = prepare(path,
base_url=f"{base_url}/v3",
endpoint="/pet/findByStatus",
hypothesis_max_examples=1)
assert after_execution.result.path == "/api/v3/pet/findByStatus"
assert (f"http://127.0.0.1:{server['port']}/api/v3/pet/findByStatus"
in after_execution.result.checks[0].example.requests_code)
def test_empty_response_interaction(openapi_version, args):
# When there is a GET request and a response that doesn't return content (e.g. 204)
app, kwargs = args
init, *others, finished = prepare(**kwargs, store_interactions=True)
interactions = [
event for event in others if isinstance(event, events.AfterExecution)
][0].result.interactions
for interaction in interactions: # There could be multiple calls
# Then the stored request has no body
assert interaction.request.body is None
# And its response as well
assert interaction.response.body is None
# And response encoding is missing
assert interaction.response.encoding is None
def test_internal_exceptions(args, mocker):
# GH: #236
app, kwargs = args
# When there is an exception during the test
# And Hypothesis consider this test as a flaky one
mocker.patch("schemathesis.Case.call", side_effect=ValueError)
mocker.patch("schemathesis.Case.call_wsgi", side_effect=ValueError)
init, *others, finished = prepare(**kwargs, hypothesis_max_examples=3)
# Then the execution result should indicate errors
assert finished.has_errors
# And an error from the buggy code should be collected
exceptions = [i.exception.strip() for i in others[1].result.errors]
assert "ValueError" in exceptions
assert len(exceptions) == 1
def test_unsatisfiable_example(empty_open_api_3_schema):
# See GH-904
# When filling missing properties during examples generation leads to unsatisfiable schemas
empty_open_api_3_schema["paths"] = {
"/success": {
"post": {
"parameters": [
# This parameter is not satisfiable
{
"name": "key",
"in": "query",
"required": True,
"schema": {
"type": "integer",
"minimum": 5,
"maximum": 4
},
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"foo": {
"type": "string",
"example": "foo example string"
},
},
},
}
}
},
"responses": {
"200": {
"description": "OK"
}
},
}
}
}
# Then the testing process should not raise an internal error
*_, after, finished = prepare(empty_open_api_3_schema,
loader=loaders.from_dict,
hypothesis_max_examples=1)
# And the tests are failing because of the unsatisfiable schema
assert finished.has_errors
assert "Unable to satisfy schema parameters for this API operation" in after.result.errors[
0].exception
def test_headers_override(args):
app, kwargs = args
def check_headers(response, case):
if isinstance(app, Flask):
data = response.json
else:
data = response.json()
assert data["X-Token"] == "test"
init, *others, finished = prepare(
**kwargs, checks=(check_headers,), headers={"X-Token": "test"}, hypothesis_max_examples=1
)
assert not finished.has_failures
assert not finished.has_errors
def test_hypothesis_errors_propagation(empty_open_api_3_schema,
openapi3_base_url):
# See: GH-1046
# When the operation contains a media type, that Schemathesis can't serialize
# And there is still a supported media type
empty_open_api_3_schema["paths"] = {
"/data": {
"post": {
"requestBody": {
"required": True,
"content": {
# This one is known
"application/json": {
"schema": {
"type": "array",
},
},
# This one is not
"application/xml": {
"schema": {
"type": "array",
}
},
},
},
"responses": {
"200": {
"description": "OK"
}
},
}
}
}
max_examples = 10
initialized, before, after, finished = prepare(
empty_open_api_3_schema,
loader=schemathesis.from_dict,
base_url=openapi3_base_url,
hypothesis_max_examples=max_examples,
)
# Then the test outcomes should not contain errors
assert after.status == Status.success
# And there should be requested amount of test examples
assert len(after.result.checks) == max_examples
assert not finished.has_failures
assert not finished.has_errors
def test_flaky_exceptions(args, mocker):
app, kwargs = args
# GH: #236
error_idx = 0
def flaky(*args, **kwargs):
nonlocal error_idx
exception_class = [ValueError, TypeError, ZeroDivisionError, KeyError][error_idx % 4]
error_idx += 1
raise exception_class
# When there are many exceptions during the test
# And Hypothesis consider this test as a flaky one
mocker.patch("schemathesis.Case.call", side_effect=flaky)
mocker.patch("schemathesis.Case.call_wsgi", side_effect=flaky)
init, *others, finished = prepare(**kwargs, hypothesis_max_examples=3, hypothesis_derandomize=True)
# Then the execution result should indicate errors
assert finished.has_errors
assert "Tests on this endpoint produce unreliable results:" in others[1].result.errors[0].exception
def test_from_path_loader_ignore_network_parameters(openapi2_base_url):
# When `from_path` loader is used
# And network-related parameters are passed
all_events = list(
prepare(
openapi2_base_url,
loader=schemathesis.openapi.from_path,
auth=("user", "password"),
headers={"X-Foo": "Bar"},
auth_type="basic",
))
# Then those parameters should be ignored during schema loading
# And a proper error message should be displayed
assert len(all_events) == 1
assert isinstance(all_events[0], events.InternalError)
if platform.system() == "Windows":
exception_type = "builtins.OSError"
else:
exception_type = "builtins.FileNotFoundError"
assert all_events[0].exception_type == exception_type
def test_internal_exceptions(args, mocker):
app, kwargs = args
# GH: #236
error_idx = 0
exc_types = [ValueError, TypeError, ZeroDivisionError, KeyError]
def buggy_call(*args, **kwargs):
nonlocal error_idx
exception_class = exc_types[error_idx % 4]
error_idx += 1
raise exception_class
# When there are many exceptions during the test
# And Hypothesis consider this test as a flaky one
mocker.patch("schemathesis.Case.call", side_effect=buggy_call)
mocker.patch("schemathesis.Case.call_wsgi", side_effect=buggy_call)
init, *others, finished = prepare(**kwargs, hypothesis_max_examples=3)
# Then the execution result should indicate errors
assert finished.has_errors
# And all errors from the buggy code should be collected
exceptions = [i.exception.strip() for i in others[1].result.errors]
for exc_type in exc_types:
assert str(exc_type.__name__) in exceptions
assert len(exceptions) == len(exc_types)
def test_runner(schema_path):
runner = prepare(
schema_path,
loader=loaders.from_path,
dry_run=True,
hypothesis_max_examples=1,
validate_schema=False,
hypothesis_suppress_health_check=HealthCheck.all(),
hypothesis_phases=[Phase.explicit, Phase.generate],
count_operations=False,
)
schema_id = get_id(schema_path)
def check_xfailed(ev) -> bool:
if schema_id in XFAILING and ev.current_operation in XFAILING[
schema_id]:
if ev.result.errors:
message = XFAILING[schema_id][ev.current_operation]
# If is is failed for some other reason, then an assertion will be risen
return any(message in err.exception_with_traceback
for err in ev.result.errors)
pytest.fail("Expected a failure")
return False
def is_unsatisfiable(text):
return "Unable to satisfy schema parameters for this API operation" in text
def check_flaky(ev) -> bool:
if schema_id in FLAKY_SCHEMAS and ev.current_operation in FLAKY_SCHEMAS[
schema_id]:
if ev.result.errors:
# NOTE. There could be other errors if the "Unsatisfiable" case wasn't triggered.
# Could be added to expected errors later
return any(
is_unsatisfiable(err.exception_with_traceback)
for err in ev.result.errors)
return False
def check_unsatisfiable(ev):
# In some cases Schemathesis can't generate data - either due to a contradiction within the schema
if schema_id in UNSATISFIABLE_SCHEMAS and ev.current_operation in UNSATISFIABLE_SCHEMAS[
schema_id]:
exception = ev.result.errors[0].exception
if ("Unable to satisfy schema parameters for this API operation"
not in exception
and "Cannot create non-empty lists with elements"
not in exception
and "Cannot create a collection of " not in exception):
pytest.fail(
f"Expected unsatisfiable, but there is a different error: {exception}"
)
return True
return False
def check_recursive_references(ev):
if schema_id in RECURSIVE_REFERENCES and ev.current_operation in RECURSIVE_REFERENCES[
schema_id]:
for err in ev.result.errors:
if RECURSIVE_REFERENCE_ERROR_MESSAGE in err.exception_with_traceback:
# It is OK
return True
# These errors may be triggered not every time
return False
def check_not_parsable(ev):
return schema_id in NOT_PARSABLE_SCHEMAS and NOT_PARSABLE_SCHEMAS[
schema_id] in ev.exception_with_traceback
def check_invalid(ev):
if schema_id in INVALID_SCHEMAS:
if ev.result.errors:
return any("The API schema contains non-string keys" in
err.exception_with_traceback
for err in ev.result.errors)
return pytest.fail("Expected YAML parsing error")
return False
for event in runner:
if isinstance(event, events.AfterExecution):
if check_xfailed(event):
continue
if check_flaky(event):
continue
if check_recursive_references(event):
continue
if check_unsatisfiable(event):
continue
if check_invalid(event):
continue
assert not event.result.has_errors, event.current_operation
assert not event.result.has_failures, event.current_operation
if isinstance(event, events.InternalError):
if check_not_parsable(event):
continue
assert not isinstance(
event, events.InternalError), event.exception_with_traceback
