def run_app(port: int, endpoints: List[Endpoint], spec: str,
framework: str) -> None:
if spec == "graphql":
app = _graphql.create_app()
app.run(port=port)
else:
if endpoints is not None:
if "__all__" in endpoints:
prepared_endpoints = tuple(endpoint.name
for endpoint in Endpoint)
else:
prepared_endpoints = tuple(endpoint.name
for endpoint in endpoints)
else:
prepared_endpoints = tuple(
endpoint.name for endpoint in Endpoint
if endpoint.name not in INVALID_ENDPOINTS)
version = {
"openapi2": OpenAPIVersion("2.0"),
"openapi3": OpenAPIVersion("3.0")
}[spec]
if framework == "aiohttp":
app = _aiohttp.create_openapi_app(prepared_endpoints, version)
web.run_app(app, port=port)
elif framework == "flask":
app = _flask.create_openapi_app(prepared_endpoints, version)
app.run(port=port)
def run_app(port: int, operations: List[Operation], spec: str,
framework: str) -> None:
if spec == "graphql":
app = _graphql.create_app()
app.run(port=port)
else:
if operations is not None:
prepared_operations = tuple(operation.name
for operation in operations)
if "all" in prepared_operations:
prepared_operations = tuple(operation.name
for operation in Operation
if operation.name != "all")
else:
prepared_operations = tuple(
operation.name for operation in Operation
if operation.name not in INVALID_OPERATIONS
and operation.name != "all")
version = {
"openapi2": OpenAPIVersion("2.0"),
"openapi3": OpenAPIVersion("3.0")
}[spec]
click.secho(
f"Schemathesis test server is running!\n\n"
f"API Schema is available at: http://0.0.0.0:{port}/schema.yaml\n",
bold=True,
)
if framework == "aiohttp":
app = _aiohttp.create_openapi_app(prepared_operations, version)
web.run_app(app, port=port)
elif framework == "flask":
app = _flask.create_openapi_app(prepared_operations, version)
app.run(port=port)
def create_openapi_app(operations: Tuple[str, ...] = ("success", "failure"),
version: OpenAPIVersion = OpenAPIVersion(
"2.0")) -> web.Application:
"""Factory for aioHTTP app.
Each handler except the one for schema saves requests in the list shared in the app instance and could be
used to verify generated requests.
>>> def test_something(app, server):
>>> # make some request to the app here
>>> assert app["incoming_requests"][0].method == "GET"
"""
incoming_requests = []
schema_requests = []
async def schema(request: web.Request) -> web.Response:
schema_data = request.app["config"]["schema_data"]
content = yaml.dump(schema_data)
schema_requests.append(request)
return web.Response(body=content)
async def set_cookies(request: web.Request) -> web.Response:
response = web.Response()
response.set_cookie("foo", "bar")
response.set_cookie("baz", "spam")
return response
def wrapper(handler_name: str) -> Callable:
handler = getattr(handlers, handler_name)
@wraps(handler)
async def inner(request: web.Request) -> web.Response:
await request.read() # to introspect the payload in tests
incoming_requests.append(request)
return await handler(request)
return inner
app = web.Application()
app.add_routes([
web.get("/schema.yaml", schema),
web.get("/api/cookies", set_cookies)
] + [
web.route(item.value[0], item.value[1], wrapper(item.name))
for item in Operation if item.name != "all"
])
async def answer(request: web.Request) -> web.Response:
return web.json_response(42)
app.add_routes([web.get("/answer.json", answer)])
app["users"] = {}
app["incoming_requests"] = incoming_requests
app["schema_requests"] = schema_requests
app["config"] = {
"should_fail": True,
"schema_data": make_openapi_schema(operations, version)
}
return app
def reset_app(
app: web.Application,
endpoints: Tuple[str, ...] = ("success", "failure"),
version: OpenAPIVersion = OpenAPIVersion("2.0"),
) -> None:
"""Clean up all internal containers of the application and resets its config."""
app["users"].clear()
app["requests_history"].clear()
app["incoming_requests"][:] = []
app["schema_requests"][:] = []
app["config"].update({"should_fail": True, "schema_data": make_openapi_schema(endpoints, version)})
def create_openapi_app(endpoints: Tuple[str, ...] = ("success", "failure"),
version: OpenAPIVersion = OpenAPIVersion(
"2.0")) -> Flask:
app = Flask("test_app")
app.config["should_fail"] = True
app.config["schema_data"] = make_openapi_schema(endpoints, version)
app.config["incoming_requests"] = []
app.config["schema_requests"] = []
app.config["internal_exception"] = False
app.config["users"] = {}
@app.before_request
def store_request():
current_request = _request_ctx_stack.top.request
if request.path == "/schema.yaml":
app.config["schema_requests"].append(current_request)
else:
app.config["incoming_requests"].append(current_request)
@app.route("/schema.yaml")
def schema():
schema_data = app.config["schema_data"]
content = yaml.dump(schema_data)
return Response(content, content_type="text/plain")
@app.route("/api/success", methods=["GET"])
def success():
if app.config["internal_exception"]:
1 / 0
return jsonify({"success": True})
@app.route("/api/recursive", methods=["GET"])
def recursive():
return jsonify({"children": [{"children": [{"children": []}]}]})
@app.route("/api/payload", methods=["POST"])
def payload():
return jsonify(request.json)
@app.route("/api/get_payload", methods=["GET"])
def get_payload():
return jsonify(request.json)
@app.route("/api/headers", methods=["GET"])
def headers():
values = dict(request.headers)
return Response(json.dumps(values),
content_type="application/json",
headers=values)
@app.route("/api/failure", methods=["GET"])
def failure():
raise InternalServerError
@app.route("/api/multiple_failures", methods=["GET"])
def multiple_failures():
id_value = int(request.args["id"])
if id_value == 0:
raise InternalServerError
if id_value > 0:
raise GatewayTimeout
return jsonify({"result": "OK"})
@app.route("/api/slow", methods=["GET"])
def slow():
sleep(0.1)
return jsonify({"slow": True})
@app.route("/api/path_variable/<key>", methods=["GET"])
def path_variable(key):
return jsonify({"success": True})
@app.route("/api/unsatisfiable", methods=["POST"])
def unsatisfiable():
return jsonify({"result": "IMPOSSIBLE!"})
@app.route("/api/invalid", methods=["POST"])
def invalid():
return jsonify({"success": True})
@app.route("/api/flaky", methods=["GET"])
def flaky():
if app.config["should_fail"]:
app.config["should_fail"] = False
raise InternalServerError
return jsonify({"result": "flaky!"})
@app.route("/api/multipart", methods=["POST"])
def multipart():
files = {
name: value.stream.read().decode()
for name, value in request.files.items()
}
return jsonify(**files, **request.form.to_dict())
@app.route("/api/upload_file", methods=["POST"])
def upload_file():
return jsonify({"size": request.content_length})
@app.route("/api/form", methods=["POST"])
def form():
if request.headers[
"Content-Type"] != "application/x-www-form-urlencoded":
raise InternalServerError("Not an urlencoded request!")
return jsonify({"size": request.content_length})
@app.route("/api/teapot", methods=["POST"])
def teapot():
return jsonify({"success": True}), 418
@app.route("/api/text", methods=["GET"])
def text():
return Response("Text response", content_type="text/plain")
@app.route("/api/malformed_json", methods=["GET"])
def malformed_json():
return Response("{malformed}", content_type="application/json")
@app.route("/api/invalid_response", methods=["GET"])
def invalid_response():
return jsonify({"random": "key"})
@app.route("/api/custom_format", methods=["GET"])
def custom_format():
return jsonify({"value": request.args["id"]})
@app.route("/api/invalid_path_parameter/<id>", methods=["GET"])
def invalid_path_parameter(id):
return jsonify({"success": True})
@app.route("/api/users/", methods=["POST"])
def create_user():
data = request.json
user_id = len(app.config["users"]) + 1
app.config["users"][user_id] = {**data, "id": user_id}
return jsonify({"id": user_id}), 201
@app.route("/api/users/<int:user_id>", methods=["GET"])
def get_user(user_id):
try:
user = app.config["users"][user_id]
return jsonify(user)
except KeyError:
return jsonify({"message": "Not found"}), 404
@app.route("/api/users/<int:user_id>", methods=["PATCH"])
def update_user(user_id):
try:
user = app.config["users"][user_id]
user["username"] = request.json["username"]
return jsonify(user)
except KeyError:
return jsonify({"message": "Not found"}), 404
return app
def create_openapi_app(operations: Tuple[str, ...] = ("success", "failure"),
version: OpenAPIVersion = OpenAPIVersion(
"2.0")) -> Flask:
app = Flask("test_app")
app.config["should_fail"] = True
app.config["schema_data"] = make_openapi_schema(operations, version)
app.config["incoming_requests"] = []
app.config["schema_requests"] = []
app.config["internal_exception"] = False
app.config["users"] = {}
@app.before_request
def store_request():
current_request = _request_ctx_stack.top.request
if request.path == "/schema.yaml":
app.config["schema_requests"].append(current_request)
else:
app.config["incoming_requests"].append(current_request)
@app.route("/schema.yaml")
def schema():
schema_data = app.config["schema_data"]
content = yaml.dump(schema_data)
return Response(content, content_type="text/plain")
@app.route("/api/success", methods=["GET"])
def success():
if app.config["internal_exception"]:
1 / 0
return jsonify({"success": True})
@app.route("/api/foo:bar", methods=["GET"])
def reserved():
return jsonify({"success": True})
@app.route("/api/recursive", methods=["GET"])
def recursive():
return jsonify({"children": [{"children": [{"children": []}]}]})
@app.route("/api/payload", methods=["POST"])
def payload():
try:
data = request.json
try:
PAYLOAD_VALIDATOR.validate(data)
except jsonschema.ValidationError:
return jsonify({"detail": "Validation error"}), 400
except BadRequest:
data = {"name": "Nothing!"}
return jsonify(data)
@app.route("/api/get_payload", methods=["GET"])
def get_payload():
return jsonify(request.json)
@app.route("/api/basic", methods=["GET"])
def basic():
if "Authorization" in request.headers and request.headers[
"Authorization"] == "Basic dGVzdDp0ZXN0":
return jsonify({"secret": 42})
return {"detail": "Unauthorized"}, 401
@app.route("/api/empty", methods=["GET"])
def empty():
return Response(status=204)
@app.route("/api/empty_string", methods=["GET"])
def empty_string():
return Response(response="")
@app.route("/api/headers", methods=["GET"])
def headers():
values = dict(request.headers)
return Response(json.dumps(values),
content_type="application/json",
headers=values)
@app.route("/api/failure", methods=["GET"])
def failure():
raise InternalServerError
@app.route("/api/multiple_failures", methods=["GET"])
def multiple_failures():
try:
id_value = int(request.args["id"])
except KeyError:
return jsonify({"detail": "Missing `id`"}), 400
except ValueError:
return jsonify({"detail": "Invalid `id`"}), 400
if id_value == 0:
raise InternalServerError
if id_value > 0:
raise GatewayTimeout
return jsonify({"result": "OK"})
@app.route("/api/slow", methods=["GET"])
def slow():
sleep(0.1)
return jsonify({"success": True})
@app.route("/api/path_variable/<key>", methods=["GET"])
def path_variable(key):
return jsonify({"success": True})
@app.route("/api/unsatisfiable", methods=["POST"])
def unsatisfiable():
return jsonify({"result": "IMPOSSIBLE!"})
@app.route("/api/invalid", methods=["POST"])
def invalid():
return jsonify({"success": True})
@app.route("/api/performance", methods=["POST"])
def performance():
data = request.json
number = str(data).count("0")
if number > 0:
sleep(0.01 * number)
if number > 10:
raise InternalServerError
return jsonify({"success": True})
@app.route("/api/flaky", methods=["GET"])
def flaky():
if app.config["should_fail"]:
app.config["should_fail"] = False
raise InternalServerError
return jsonify({"result": "flaky!"})
@app.route("/api/multipart", methods=["POST"])
def multipart():
files = {
name: value.stream.read().decode()
for name, value in request.files.items()
}
return jsonify(**files, **request.form.to_dict())
@app.route("/api/upload_file", methods=["POST"])
def upload_file():
return jsonify({"size": request.content_length})
@app.route("/api/form", methods=["POST"])
def form():
expect_content_type("application/x-www-form-urlencoded")
data = request.form
for field in ("first_name", "last_name"):
if field not in data:
return jsonify({"detail": f"Missing `{field}`"}), 400
if not isinstance(data[field], str):
return jsonify({"detail": f"Invalid `{field}`"}), 400
return jsonify({"size": request.content_length})
@app.route("/api/csv", methods=["POST"])
def csv_payload():
expect_content_type("text/csv")
data = request.get_data(as_text=True)
if data:
reader = csv.DictReader(data.splitlines())
data = list(reader)
else:
data = []
return jsonify(data)
@app.route("/api/teapot", methods=["POST"])
def teapot():
return jsonify({"success": True}), 418
@app.route("/api/text", methods=["GET"])
def text():
return Response("Text response", content_type="text/plain")
@app.route("/api/cp866", methods=["GET"])
def cp866():
# NOTE. Setting `Response.charset` don't have effect in test client as it re-wraps this response with the
# default one where `charset` is `utf-8`
return Response("Тест".encode("cp866"),
content_type="text/plain;charset=cp866")
@app.route("/api/text", methods=["POST"])
def plain_text_body():
expect_content_type("text/plain")
return Response(request.data, content_type="text/plain")
@app.route("/api/malformed_json", methods=["GET"])
def malformed_json():
return Response("{malformed}", content_type="application/json")
@app.route("/api/invalid_response", methods=["GET"])
def invalid_response():
return jsonify({"random": "key"})
@app.route("/api/custom_format", methods=["GET"])
def custom_format():
if "id" not in request.args:
return jsonify({"detail": "Missing `id`"}), 400
if not request.args["id"].isdigit():
return jsonify({"detail": "Invalid `id`"}), 400
return jsonify({"value": request.args["id"]})
@app.route("/api/invalid_path_parameter/<id>", methods=["GET"])
def invalid_path_parameter(id):
return jsonify({"success": True})
@app.route("/api/users/", methods=["POST"])
def create_user():
data = request.json
for field in ("first_name", "last_name"):
if field not in data:
return jsonify({"detail": f"Missing `{field}`"}), 400
if not isinstance(data[field], str):
return jsonify({"detail": f"Invalid `{field}`"}), 400
user_id = str(uuid4())
app.config["users"][user_id] = {**data, "id": user_id}
return jsonify({"id": user_id}), 201
@app.route("/api/users/<user_id>", methods=["GET"])
def get_user(user_id):
try:
user = app.config["users"][user_id]
# The full name is done specifically via concatenation to trigger a bug when the last name is `None`
full_name = user["first_name"] + " " + user["last_name"]
return jsonify({"id": user["id"], "full_name": full_name})
except KeyError:
return jsonify({"message": "Not found"}), 404
@app.route("/api/users/<user_id>", methods=["PATCH"])
def update_user(user_id):
try:
user = app.config["users"][user_id]
data = request.json
for field in ("first_name", "last_name"):
if field not in data:
return jsonify({"detail": f"Missing `{field}`"}), 400
# Here we don't check the input value type to emulate a bug in another operation
user[field] = data[field]
return jsonify(user)
except KeyError:
return jsonify({"message": "Not found"}), 404
return app
def create_openapi_app(
endpoints: Tuple[str, ...] = ("success", "failure"), version: OpenAPIVersion = OpenAPIVersion("2.0")
) -> Flask:
app = Flask("test_app")
app.config["should_fail"] = True
app.config["schema_data"] = make_openapi_schema(endpoints, version)
app.config["incoming_requests"] = []
app.config["schema_requests"] = []
app.config["internal_exception"] = False
app.config["users"] = {}
app.config["requests_history"] = defaultdict(list)
@app.before_request
def store_request():
current_request = _request_ctx_stack.top.request
if request.path == "/schema.yaml":
app.config["schema_requests"].append(current_request)
else:
app.config["incoming_requests"].append(current_request)
@app.route("/schema.yaml")
def schema():
schema_data = app.config["schema_data"]
content = yaml.dump(schema_data)
return Response(content, content_type="text/plain")
@app.route("/api/success", methods=["GET"])
def success():
if app.config["internal_exception"]:
1 / 0
return jsonify({"success": True})
@app.route("/api/recursive", methods=["GET"])
def recursive():
return jsonify({"children": [{"children": [{"children": []}]}]})
@app.route("/api/payload", methods=["POST"])
def payload():
try:
data = request.json
try:
PAYLOAD_VALIDATOR.validate(data)
except jsonschema.ValidationError:
return jsonify({"detail": "Validation error"}), 400
except BadRequest:
data = {"name": "Nothing!"}
return jsonify(data)
@app.route("/api/get_payload", methods=["GET"])
def get_payload():
return jsonify(request.json)
@app.route("/api/headers", methods=["GET"])
def headers():
values = dict(request.headers)
return Response(json.dumps(values), content_type="application/json", headers=values)
@app.route("/api/failure", methods=["GET"])
def failure():
raise InternalServerError
@app.route("/api/multiple_failures", methods=["GET"])
def multiple_failures():
try:
id_value = int(request.args["id"])
except KeyError:
return jsonify({"detail": "Missing `id`"}), 400
except ValueError:
return jsonify({"detail": "Invalid `id`"}), 400
if id_value == 0:
raise InternalServerError
if id_value > 0:
raise GatewayTimeout
return jsonify({"result": "OK"})
@app.route("/api/slow", methods=["GET"])
def slow():
sleep(0.1)
return jsonify({"success": True})
@app.route("/api/path_variable/<key>", methods=["GET"])
def path_variable(key):
return jsonify({"success": True})
@app.route("/api/unsatisfiable", methods=["POST"])
def unsatisfiable():
return jsonify({"result": "IMPOSSIBLE!"})
@app.route("/api/invalid", methods=["POST"])
def invalid():
return jsonify({"success": True})
@app.route("/api/performance", methods=["POST"])
def performance():
data = request.json
number = str(data).count("0")
if number > 0:
sleep(0.01 * number)
if number > 10:
raise InternalServerError
return jsonify({"success": True})
@app.route("/api/flaky", methods=["GET"])
def flaky():
if app.config["should_fail"]:
app.config["should_fail"] = False
raise InternalServerError
return jsonify({"result": "flaky!"})
@app.route("/api/multipart", methods=["POST"])
def multipart():
files = {name: value.stream.read().decode() for name, value in request.files.items()}
return jsonify(**files, **request.form.to_dict())
@app.route("/api/upload_file", methods=["POST"])
def upload_file():
return jsonify({"size": request.content_length})
@app.route("/api/form", methods=["POST"])
def form():
expect_content_type("application/x-www-form-urlencoded")
data = request.form
for field in ("first_name", "last_name"):
if field not in data:
return jsonify({"detail": f"Missing `{field}`"}), 400
if not isinstance(data[field], str):
return jsonify({"detail": f"Invalid `{field}`"}), 400
return jsonify({"size": request.content_length})
@app.route("/api/csv", methods=["POST"])
def csv_payload():
expect_content_type("text/csv")
data = request.get_data(as_text=True)
if data:
reader = csv.DictReader(data.splitlines())
data = list(reader)
else:
data = []
return jsonify(data)
@app.route("/api/teapot", methods=["POST"])
def teapot():
return jsonify({"success": True}), 418
@app.route("/api/text", methods=["GET"])
def text():
return Response("Text response", content_type="text/plain")
@app.route("/api/text", methods=["POST"])
def plain_text_body():
expect_content_type("text/plain")
return Response(request.data, content_type="text/plain")
@app.route("/api/malformed_json", methods=["GET"])
def malformed_json():
return Response("{malformed}", content_type="application/json")
@app.route("/api/invalid_response", methods=["GET"])
def invalid_response():
return jsonify({"random": "key"})
@app.route("/api/custom_format", methods=["GET"])
def custom_format():
if "id" not in request.args:
return jsonify({"detail": "Missing `id`"}), 400
if not request.args["id"].isdigit():
return jsonify({"detail": "Invalid `id`"}), 400
return jsonify({"value": request.args["id"]})
@app.route("/api/invalid_path_parameter/<id>", methods=["GET"])
def invalid_path_parameter(id):
return jsonify({"success": True})
@app.route("/api/users/", methods=["POST"])
def create_user():
data = request.json
if "username" not in data:
return jsonify({"detail": "Missing `username`"}), 400
if not isinstance(data["username"], str):
return jsonify({"detail": "Invalid `username`"}), 400
user_id = len(app.config["users"]) + 1
app.config["users"][user_id] = {**data, "id": user_id}
app.config["requests_history"][user_id].append("POST")
return jsonify({"id": user_id}), 201
@app.route("/api/users/<int:user_id>", methods=["GET"])
def get_user(user_id):
try:
user = app.config["users"][user_id]
app.config["requests_history"][user_id].append("GET")
return jsonify(user)
except KeyError:
return jsonify({"message": "Not found"}), 404
@app.route("/api/users/<int:user_id>", methods=["PATCH"])
def update_user(user_id):
try:
user = app.config["users"][user_id]
history = app.config["requests_history"][user_id]
history.append("PATCH")
if history == ["POST", "GET", "PATCH", "GET", "PATCH"]:
raise InternalServerError("We got a problem!")
data = request.json
if "username" not in data:
return jsonify({"detail": "Missing `username`"}), 400
if not isinstance(data["username"], str):
return jsonify({"detail": "Invalid `username`"}), 400
user["username"] = data["username"]
return jsonify(user)
except KeyError:
return jsonify({"message": "Not found"}), 404
return app