def test_execute_filter_method(args):
app, kwargs = args
# When `method` passed in `loader_options` corresponds to a method that is not defined in the app schema
kwargs.setdefault("loader_options", {})["method"] = ["POST"]
execute(**kwargs)
# Then runner will not make any requests
assert_incoming_requests_num(app, 0)
def test_hypothesis_deadline(schema_url, app):
# When `deadline` is passed in `hypothesis_options` in the `execute` call
execute(schema_url, hypothesis_options={"deadline": 500})
# Then it should be passed to `hypothesis.settings`
# And slow endpoint (250ms) should not be considered as breaking the deadline
assert len(app["incoming_requests"]) == 1
assert_request(app, 0, "GET", "/api/slow")
def test_execute_filter_endpoint(schema_url, app):
# When `endpoint` is passed in `loader_options` in the `execute` call
execute(schema_url, loader_options={"endpoint": ["success"]})
# Then the runner will make calls only to the specified endpoint
assert len(app["incoming_requests"]) == 1
assert_request(app, 0, "GET", "/api/success")
assert_not_request(app, "GET", "/api/failure")
def test_execute_filter_endpoint(args):
app, kwargs = args
# When `endpoint` is passed in `loader_options` in the `execute` call
kwargs.setdefault("loader_options", {})["endpoint"] = ["success"]
execute(**kwargs)
# Then the runner will make calls only to the specified endpoint
assert_incoming_requests_num(app, 1)
assert_request(app, 0, "GET", "/api/success")
assert_not_request(app, "GET", "/api/failure")
def test_auth(schema_url, app):
# When auth is specified in `api_options` as a tuple of 2 strings
execute(schema_url, api_options={"auth": ("test", "test")})
# Then each request should contain corresponding basic auth header
assert len(app["incoming_requests"]) == 3
headers = {"Authorization": "Basic dGVzdDp0ZXN0"}
assert_request(app, 0, "GET", "/api/failure", headers)
assert_request(app, 1, "GET", "/api/failure", headers)
assert_request(app, 2, "GET", "/api/success", headers)
def test_execute_with_headers(schema_url, app):
# When headers are specified for the `execute` call
headers = {"Authorization": "Bearer 123"}
execute(schema_url, api_options={"headers": headers})
# Then each request should contain these headers
assert len(app["incoming_requests"]) == 3
assert_request(app, 0, "GET", "/api/failure", headers)
assert_request(app, 1, "GET", "/api/failure", headers)
assert_request(app, 2, "GET", "/api/success", headers)
def test_base_url(base_url, schema_url, app, converter):
base_url = converter(base_url)
# When `base_url` is specified explicitly with or without trailing slash
execute(schema_url, loader_options={"base_url": base_url})
# Then each request should reach the app in both cases
assert_incoming_requests_num(app, 3)
assert_request(app, 0, "GET", "/api/failure")
assert_request(app, 1, "GET", "/api/failure")
assert_request(app, 2, "GET", "/api/success")
def test_auth(args):
app, kwargs = args
# When auth is specified in `api_options` as a tuple of 2 strings
execute(**kwargs, api_options={"auth": ("test", "test")})
# Then each request should contain corresponding basic auth header
assert_incoming_requests_num(app, 3)
headers = {"Authorization": "Basic dGVzdDp0ZXN0"}
assert_request(app, 0, "GET", "/api/failure", headers)
assert_request(app, 1, "GET", "/api/failure", headers)
assert_request(app, 2, "GET", "/api/success", headers)
def test_execute_with_headers(args):
app, kwargs = args
# When headers are specified for the `execute` call
headers = {"Authorization": "Bearer 123"}
execute(**kwargs, api_options={"headers": headers})
# Then each request should contain these headers
assert_incoming_requests_num(app, 3)
assert_request(app, 0, "GET", "/api/failure", headers)
assert_request(app, 1, "GET", "/api/failure", headers)
assert_request(app, 2, "GET", "/api/success", headers)
def test_flaky_exceptions(args, mocker):
app, kwargs = args
# GH: #236
error_idx = 0
def flaky(*args, **kwargs):
nonlocal error_idx
exception_class = [ValueError, TypeError, ZeroDivisionError,
KeyError][error_idx % 4]
error_idx += 1
raise exception_class
# When there are many different exceptions during the test
# And Hypothesis consider this test as a flaky one
mocker.patch("schemathesis.Case.call", side_effect=flaky)
mocker.patch("schemathesis.Case.call_wsgi", side_effect=flaky)
results = execute(**kwargs,
hypothesis_options={
"max_examples": 3,
"derandomize": True
})
# Then the execution result should indicate errors
assert results.has_errors
assert results.results[0].errors[0][0].args[0].startswith(
"Tests on this endpoint produce unreliable results:")
def test_invalid_path_parameter(args):
app, kwargs = args
results = execute(**kwargs)
assert results.has_errors
error, _ = results.results[0].errors[0]
assert isinstance(error, InvalidSchema)
assert str(error) == "Missing required property `required: true`"
def test_known_content_type(args):
app, kwargs = args
# When endpoint returns a response with a proper content type
# And "content_type_conformance" is specified
results = execute(**kwargs, checks=(content_type_conformance,), hypothesis_options={"max_examples": 1})
# Then there should be no a failures
assert not results.has_failures
def test_execute(args):
app, kwargs = args
# When the runner is executed against the default test app
stats = execute(**kwargs)
# Then there are three executed cases
# Two errors - the second one is a flakiness check
headers = {"User-Agent": f"schemathesis/{__version__}"}
assert_schema_requests_num(app, 1)
schema_requests = get_schema_requests(app)
assert schema_requests[0].headers.get(
"User-Agent") == headers["User-Agent"]
assert_incoming_requests_num(app, 3)
assert_request(app, 0, "GET", "/api/failure", headers)
assert_request(app, 1, "GET", "/api/failure", headers)
assert_request(app, 2, "GET", "/api/success", headers)
# And statistic is showing the breakdown of cases types
assert stats.total == {
"not_a_server_error": {
Status.success: 1,
Status.failure: 2,
"total": 3
}
}
def test_form_data(args):
app, kwargs = args
def is_ok(response, result):
assert response.status_code == 200
def check_content(response, result):
if isinstance(app, Flask):
data = response.json
else:
data = response.json()
assert isinstance(data["key"], str)
assert data["value"].lstrip("-").isdigit()
# When endpoint specifies parameters with `in=formData`
# Then responses should have 200 status, and not 415 (unsupported media type)
results = execute(**kwargs,
checks=(is_ok, check_content),
hypothesis_options={"max_examples": 3})
# And there should be no errors or failures
assert not results.has_errors
assert not results.has_failures
# And the application should receive 3 requests as specified in `max_examples`
assert_incoming_requests_num(app, 3)
# And the Content-Type of incoming requests should be `multipart/form-data`
incoming_requests = get_incoming_requests(app)
assert incoming_requests[0].headers["Content-Type"].startswith(
"multipart/form-data")
def test_response_conformance_valid(args):
app, kwargs = args
# When endpoint returns a response that conforms to the schema
# And "response_schema_conformance" is specified
results = execute(**kwargs, checks=(response_schema_conformance,), hypothesis_options={"max_examples": 1})
# Then there should be no failures or errors
assert not results.has_failures
assert not results.has_errors
def test_response_conformance_text(args):
app, kwargs = args
# When endpoint returns a response that is not JSON
# And "response_schema_conformance" is specified
results = execute(**kwargs, checks=(response_schema_conformance,), hypothesis_options={"max_examples": 1})
# Then the check should be ignored if the response headers are not application/json
assert not results.has_failures
assert not results.has_errors
def test_path_parameters_encoding(schema_url):
# NOTE. Flask still decodes %2F as / and returns 404
# When endpoint has a path parameter
results = execute(schema_url, checks=(status_code_conformance,), hypothesis_options={"derandomize": True})
# Then there should be no failures
# since all path parameters are quoted
assert not results.has_errors
assert not results.has_failures
def test_response_conformance_malformed_json(args):
app, kwargs = args
# When endpoint returns a response that contains a malformed JSON, but has a valid content type header
# And "response_schema_conformance" is specified
results = execute(**kwargs, checks=(response_schema_conformance,), hypothesis_options={"max_examples": 1})
# Then there should be a failure
assert results.has_errors
error = results.results[-1].errors[-1][0]
assert "Expecting property name enclosed in double quotes" in str(error)
def test_unknown_content_type(args):
app, kwargs = args
# When endpoint returns a response with content type, not specified in "produces"
# And "content_type_conformance" is specified
results = execute(**kwargs, checks=(content_type_conformance,), hypothesis_options={"max_examples": 1})
# Then there should be a failure
assert results.has_failures
check = results.results[0].checks[0]
assert check.name == "content_type_conformance"
assert check.value == Status.failure
def test_unknown_response_code_with_default(args):
app, kwargs = args
# When endpoint returns a status code, that is not listed in "responses", but there is a "default" response
# And "status_code_conformance" is specified
results = execute(**kwargs, checks=(status_code_conformance,), hypothesis_options={"max_examples": 1})
# Then there should be no failure
assert not results.has_failures
check = results.results[0].checks[0]
assert check.name == "status_code_conformance"
assert check.value == Status.success
def test_response_conformance_invalid(args):
app, kwargs = args
# When endpoint returns a response that doesn't conform to the schema
# And "response_schema_conformance" is specified
results = execute(**kwargs, checks=(response_schema_conformance,), hypothesis_options={"max_examples": 1})
# Then there should be a failure
assert results.has_failures
lines = results.results[0].checks[-1].message.split("\n")
assert lines[0] == "The received response does not conform to the defined schema!"
assert lines[2] == "Details: "
assert lines[4] == "'success' is a required property"
def test_unknown_response_code(schema_url, app):
# When endpoint returns a status code, that is not listed in "responses"
# And "status_code_conformance" is specified
results = execute(schema_url,
checks=(status_code_conformance, ),
hypothesis_options={"max_examples": 1})
# Then there should be a failure
assert results.has_failures
check = results.results[0].checks[0]
assert check.name == "status_code_conformance"
assert check.value == Status.failure
async def test_payload_explicit_example(args):
# When endpoint has an example specified
app, kwargs = args
kwargs.setdefault("hypothesis_options", {})["phases"] = [Phase.explicit]
result = execute(**kwargs)
# Then run should be successful
assert not result.has_errors
assert not result.has_failures
incoming_requests = get_incoming_requests(app)
if isinstance(app, Flask):
body = incoming_requests[0].json
else:
body = await incoming_requests[0].json()
# And this example should be sent to the app
assert body == {"name": "John"}
def test_execute(schema_url, app):
# When the runner is executed against the default test app
stats = execute(schema_url)
# Then there are three executed cases
# Two errors - the second one is a flakiness check
headers = {"User-Agent": f"schemathesis/{__version__}"}
assert len(app["incoming_requests"]) == 3
assert_request(app, 0, "GET", "/api/failure", headers)
assert_request(app, 1, "GET", "/api/failure", headers)
assert_request(app, 2, "GET", "/api/success", headers)
# And statistic is showing the breakdown of cases types
assert "not_a_server_error" in stats.data
assert dict(stats.data["not_a_server_error"]) == {
"total": 3,
"ok": 1,
"error": 2
}
def test_execute_base_url_found(base_url, schema_url, app):
# When base_url is specified
execute(schema_url, loader_options={"base_url": base_url})
# Then it should be used by the runner
assert_incoming_requests_num(app, 3)
def test_execute_base_url_not_found(base_url, schema_url, app):
# When base URL is pointing to an unknown location
execute(schema_url, loader_options={"base_url": f"{base_url}/404/"})
# Then the runner should use this base
# And they will not reach the application
assert_incoming_requests_num(app, 0)
def test_hypothesis_deadline(schema_url, app):
# When `deadline` is passed in `hypothesis_options` in the `execute` call
execute(schema_url, hypothesis_options={"deadline": 500})
def test_hypothesis_deadline(args):
app, kwargs = args
# When `deadline` is passed in `hypothesis_options` in the `execute` call
execute(**kwargs, hypothesis_options={"deadline": 500})
assert_incoming_requests_num(app, 1)
assert_request(app, 0, "GET", "/api/slow")
def test_execute_base_url_found(base_url, schema_url, app):
# When base_url is specified
execute(schema_url, api_options={"base_url": base_url})
# Then it should be used by the runner
assert len(app["incoming_requests"]) == 3
def test_execute_filter_method(schema_url, app):
# When `method` passed in `loader_options` corresponds to a method that is not defined in the app schema
execute(schema_url, loader_options={"method": ["POST"]})
# Then runner will not make any requests
assert len(app["incoming_requests"]) == 0
