def test_display_statistic(capsys, swagger_20, execution_context, operation,
response):
# Given multiple successful & failed checks in a single test
success = models.Check("not_a_server_error", models.Status.success,
response, 0, models.Case(operation))
failure = models.Check("not_a_server_error", models.Status.failure,
response, 0, models.Case(operation))
single_test_statistic = models.TestResult(
operation.method,
operation.full_path,
DataGenerationMethod.default(),
[
success,
success,
success,
failure,
failure,
models.Check("different_check", models.Status.success, response, 0,
models.Case(operation)),
],
)
results = models.TestResultSet([single_test_statistic])
event = Finished.from_results(results, running_time=1.0)
# When test results are displayed
default.display_statistic(execution_context, event)
lines = [line for line in capsys.readouterr().out.split("\n") if line]
failed = strip_style_win32(click.style("FAILED", bold=True, fg="red"))
passed = strip_style_win32(click.style("PASSED", bold=True, fg="green"))
# Then all check results should be properly displayed with relevant colors
assert lines[2:4] == [
f" not_a_server_error 3 / 5 passed {failed} ",
f" different_check 1 / 1 passed {passed} ",
]
def test_display_single_failure(capsys, swagger_20, execution_context, operation, body, response):
# Given a single test result with multiple successful & failed checks
success = models.Check("not_a_server_error", models.Status.success, response, 0, models.Case(operation, body=body))
failure = models.Check("not_a_server_error", models.Status.failure, response, 0, models.Case(operation, body=body))
test_statistic = models.TestResult(
operation.method,
operation.full_path,
DataGenerationMethod.default(),
[
success,
success,
success,
failure,
failure,
models.Check("different_check", models.Status.success, response, 0, models.Case(operation, body=body)),
],
)
# When this failure is displayed
default.display_failures_for_single_test(execution_context, SerializedTestResult.from_test_result(test_statistic))
out = capsys.readouterr().out
lines = out.split("\n")
# Then the path is displayed as a subsection
assert " GET /v1/success " in lines[0]
# And body should be displayed if it is not NOT_SET
if body is NOT_SET:
assert "Body" not in out
else:
assert strip_style_win32(click.style(f"Body : {body}", fg="red")) in lines
# And empty parameters are not present in the output
assert "Path parameters" not in out
# And not needed attributes are not displayed
assert "Path" not in out
assert "Method" not in out
assert "Base url" not in out
def make_case(schema: BaseSchema, definition: Dict[str, Any]) -> models.Case:
endpoint = models.Endpoint("/path",
"GET",
definition=EndpointDefinition(
definition, definition, None, []),
schema=schema)
return models.Case(endpoint)
def test_display_errors(swagger_20, capsys, results_set, execution_context,
show_errors_tracebacks, verbosity):
execution_context.verbosity = verbosity
# Given two test results - success and error
endpoint = models.Endpoint("/api/error", "GET", {}, swagger_20)
error = models.TestResult(endpoint, seed=123)
error.add_error(ConnectionError("Connection refused!"),
models.Case(endpoint, query={"a": 1}))
results_set.append(error)
execution_context.results.append(
SerializedTestResult.from_test_result(error))
event = Finished.from_results(results_set, 1.0)
# When the errors are displayed
execution_context.show_errors_tracebacks = show_errors_tracebacks
default.display_errors(execution_context, event)
out = capsys.readouterr().out.strip()
# Then section title is displayed
assert " ERRORS " in out
help_message_exists = (
"Add this option to your command line parameters to see full tracebacks: --show-errors-tracebacks"
in out)
# And help message is displayed only if tracebacks are not shown
assert help_message_exists is not show_errors_tracebacks
# And endpoint with an error is displayed as a subsection
assert " GET: /api/error " in out
# And the error itself is displayed
assert "ConnectionError: Connection refused!" in out
# And the example is displayed
assert "Query : {'a': 1}" in out
assert "Or add this option to your command line parameters: --hypothesis-seed=123" in out
def test_display_single_failure(capsys, swagger_20, execution_context,
endpoint, body):
# Given a single test result with multiple successful & failed checks
success = models.Check("not_a_server_error", models.Status.success)
failure = models.Check("not_a_server_error", models.Status.failure,
models.Case(endpoint, body=body))
test_statistic = models.TestResult(endpoint, [
success, success, success, failure, failure,
models.Check("different_check", models.Status.success)
])
# When this failure is displayed
default.display_failures_for_single_test(
execution_context,
SerializedTestResult.from_test_result(test_statistic))
out = capsys.readouterr().out
lines = out.split("\n")
# Then the endpoint name is displayed as a subsection
assert " GET: /success " in lines[0]
# And check name is displayed in red
assert lines[1] == strip_style_win32(
click.style("Check : not_a_server_error", fg="red"))
# And body should be displayed if it is not None
if body is None:
assert "Body" not in out
else:
assert strip_style_win32(
click.style(f"Body : {body}", fg="red")) in lines
# And empty parameters are not present in the output
assert "Path parameters" not in out
# And not needed attributes are not displayed
assert "Path" not in out
assert "Method" not in out
assert "Base url" not in out
def test_display_single_failure(capsys, swagger_20, endpoint, body):
# Given a single test result with multiple successful & failed checks
success = models.Check("not_a_server_error", models.Status.success)
failure = models.Check(
"not_a_server_error",
models.Status.failure,
models.Case("/success", "GET", base_url="http://example.com", body=body),
)
test_statistic = models.TestResult(
endpoint,
swagger_20,
[success, success, success, failure, failure, models.Check("different_check", models.Status.success)],
)
# When this failure is displayed
output.display_single_failure(test_statistic)
out = capsys.readouterr().out
lines = out.split("\n")
# Then the endpoint name is displayed as a subsection
assert " GET: /success " in lines[0]
# And check name is displayed in red
assert lines[1] == click.style("Check : not_a_server_error", fg="red")
# And body should be displayed if it is not None
if body is None:
assert "Body" not in out
else:
assert click.style(f"Body : {body}", fg="red") in lines
# And empty parameters are not present in the output
assert "Path parameters" not in out
# And not needed attributes are not displayed
assert "Path" not in out
assert "Method" not in out
assert "Base url" not in out
def test_display_failures(swagger_20, capsys, execution_context, results_set,
verbosity, response):
execution_context.verbosity = verbosity
# Given two test results - success and failure
operation = models.APIOperation("/api/failure",
"GET", {},
base_url="http://127.0.0.1:8080",
schema=swagger_20)
failure = models.TestResult(operation.method, operation.full_path,
DataGenerationMethod.default())
failure.add_failure("test", models.Case(operation), response, 0, "Message")
execution_context.results.append(
SerializedTestResult.from_test_result(failure))
results_set.append(failure)
event = Finished.from_results(results_set, 1.0)
# When the failures are displayed
default.display_failures(execution_context, event)
out = capsys.readouterr().out.strip()
# Then section title is displayed
assert " FAILURES " in out
# And operation with a failure is displayed as a subsection
assert " GET: /v1/api/failure " in out
assert "Message" in out
assert "Run this Python code to reproduce this failure: " in out
assert f"requests.get('http://127.0.0.1:8080/api/failure', headers={{'User-Agent': '{USER_AGENT}'}})" in out
def make_case(schema: BaseSchema, definition: Dict[str, Any]) -> models.Case:
operation = models.APIOperation("/path",
"GET",
definition=OperationDefinition(
definition, definition, None, []),
schema=schema)
return models.Case(operation)
def test_display_failures(swagger_20, capsys, execution_context, results_set,
verbosity, response):
execution_context.verbosity = verbosity
# Given two test results - success and failure
operation = models.APIOperation("/api/failure",
"GET", {},
base_url="http://127.0.0.1:8080",
schema=swagger_20)
failure = models.TestResult(
operation.method,
operation.full_path,
verbose_name=f"{operation.method} {operation.full_path}",
data_generation_method=DataGenerationMethod.default(),
)
failure.add_failure("test", models.Case(operation), response, 0, "Message",
None)
execution_context.results.append(
SerializedTestResult.from_test_result(failure))
results_set.append(failure)
event = Finished.from_results(results_set, 1.0)
# When the failures are displayed
default.display_failures(execution_context, event)
out = capsys.readouterr().out.strip()
# Then section title is displayed
assert " FAILURES " in out
# And operation with a failure is displayed as a subsection
assert " GET /v1/api/failure " in out
assert "Message" in out
assert "Run this cURL command to reproduce this failure:" in out
headers = f"-H 'Content-Length: 0' -H 'Content-Type: application/json' -H 'User-Agent: {USER_AGENT}'"
assert f"curl -X GET {headers} http://127.0.0.1:8080/api/failure" in out
def assert_content_type_conformance(raw_schema, content_type, is_error):
schema = schemathesis.from_dict(raw_schema)
endpoint = schema.endpoints["/users"]["get"]
case = models.Case(endpoint)
response = make_response(content_type=content_type)
if not is_error:
assert content_type_conformance(response, case) is None
else:
with pytest.raises(AssertionError):
content_type_conformance(response, case)
def assert_content_type_conformance(raw_schema, content_type, is_error, match=None):
schema = schemathesis.from_dict(raw_schema)
operation = schema["/users"]["get"]
case = models.Case(operation)
response = make_response(content_type=content_type)
if not is_error:
assert content_type_conformance(response, case) is None
else:
with pytest.raises(AssertionError, match=match):
content_type_conformance(response, case)
def test_display_failures(swagger_20, capsys, results_set):
# Given two test results - success and failure
failure = models.TestResult(models.Endpoint("/api/failure", "GET", {}), swagger_20)
failure.add_failure("test", models.Case("/api/failure", "GET", base_url="http://127.0.0.1:8080"))
results_set.append(failure)
# When the failures are displayed
output.display_failures(results_set)
out = capsys.readouterr().out.strip()
# Then section title is displayed
assert " FAILURES " in out
# And endpoint with a failure is displayed as a subsection
assert " GET: /api/failure " in out
# And check name is displayed
assert "Check : test" in out
assert "Run this Python code to reproduce this failure: " in out
assert "requests.get('http://127.0.0.1:8080/api/failure')" in out
def test_invalid_schema_on_content_type_check():
# When schema validation is disabled and it doesn't contain "responses" key
schema = schemathesis.from_dict(
{
"openapi": "3.0.2",
"info": {"title": "Test", "description": "Test", "version": "0.1.0"},
"paths": {"/users": {"get": {}}},
},
validate_schema=False,
)
endpoint = schema.endpoints["/users"]["get"]
case = models.Case(endpoint)
response = make_response(content_type="application/json")
# Then an error should be risen
with pytest.raises(InvalidSchema):
content_type_conformance(response, case)
def test_display_errors(swagger_20, capsys, results_set):
# Given two test results - success and error
endpoint = models.Endpoint("/api/error", "GET", {}, swagger_20)
error = models.TestResult(endpoint, seed=123)
error.add_error(ConnectionError("Connection refused!"), models.Case(endpoint, query={"a": 1}))
results_set.append(error)
# When the errors are displayed
default.display_errors(results_set)
out = capsys.readouterr().out.strip()
# Then section title is displayed
assert " ERRORS " in out
# And endpoint with an error is displayed as a subsection
assert " GET: /api/error " in out
# And the error itself is displayed
assert "ConnectionError: Connection refused!" in out
# And the example is displayed
assert "Query : {'a': 1}" in out
assert "Or add this option to your command line parameters: --hypothesis-seed=123" in out
def test_display_errors(swagger_20, capsys, results_set):
# Given two test results - success and error
error = models.TestResult(models.Endpoint("/api/error", "GET", {}), swagger_20)
error.add_error(
ConnectionError("Connection refused!"),
models.Case("/api/error", "GET", base_url="http://127.0.0.1:8080", query={"a": 1}),
)
results_set.append(error)
# When the errors are displayed
output.display_errors(results_set)
out = capsys.readouterr().out.strip()
# Then section title is displayed
assert " ERRORS " in out
# And endpoint with an error is displayed as a subsection
assert " GET: /api/error " in out
# And the error itself is displayed
assert "ConnectionError: Connection refused!" in out
# And the example is displayed
assert "Query : {'a': 1}" in out
def test_display_failures(swagger_20, capsys, execution_context, results_set):
# Given two test results - success and failure
endpoint = models.Endpoint("/api/failure", "GET", {}, base_url="http://127.0.0.1:8080", schema=swagger_20)
failure = models.TestResult(endpoint)
failure.add_failure("test", models.Case(endpoint), "Message")
execution_context.results.append(SerializedTestResult.from_test_result(failure))
results_set.append(failure)
event = Finished.from_results(results_set, 1.0)
# When the failures are displayed
default.display_failures(execution_context, event)
out = capsys.readouterr().out.strip()
# Then section title is displayed
assert " FAILURES " in out
# And endpoint with a failure is displayed as a subsection
assert " GET: /api/failure " in out
assert "Message" in out
# And check name is displayed
assert "Check : test" in out
assert "Run this Python code to reproduce this failure: " in out
assert "requests.get('http://127.0.0.1:8080/api/failure')" in out