def _getspec(resource_name, resource_url, download_spec=False):
"""
Returns the openapi spec
:param resource_name: (str) the name of the resource.
:param resource_url: (str) URL to download the resource spec file.
:return: (openapi_core.schema.specs.models.Spec) The Spec object associated with this resource.
"""
if download_spec:
try:
response = requests.get(resource_url)
if response.status_code == 200:
try:
spec_dict = yaml.load(response.content)
return create_spec(spec_dict)
# for now, if there are errors trying to fetch the latest spec, we fall back to the spec files defined in the
# the python-sdk package;
except:
pass
except:
pass
try:
# for now, hardcode the paths; we could look these up based on a canonical URL once that is
# established.
spec_path = f'/home/tapis/tapy/dyna/resources/openapi_v3-{resource_name}.yml'
spec_dict = yaml.load(open(spec_path, 'r'))
return create_spec(spec_dict)
except Exception as e:
print(
f"Got exception trying to load spec_path: {spec_path}; exception: {e}"
)
raise e
async def create_openapi_specs(location,
session: ClientSession = None) -> OpenApiSpec:
"""Loads specs from a given location (url or path),
validates them and returns a working instance
If location is an url, the specs are loaded asyncronously
Both location types (url and file) are intentionally managed
by the same function call to enforce developer always supporting
both options. Notice that the url location enforces
the consumer context to be asyncronous.
:param location: url or path
:return: validated openapi specifications object
:rtype: OpenApiSpec
"""
if URL(str(location)).host:
if session is None:
raise ValueError("Client session required in arguments")
spec_dict, spec_url = await _load_from_url(session, URL(location))
else:
path = Path(location).expanduser().resolve() # pylint: disable=no-member
spec_dict, spec_url = _load_from_path(path)
return openapi_core.create_spec(spec_dict, spec_url)
def register() -> None:
if hupper.is_active(): # pragma: no cover
hupper.get_reloader().watch_files([filepath])
spec_dict = read_yaml_file(filepath)
validate_spec(spec_dict)
spec = create_spec(spec_dict)
def spec_view(request: Request) -> FileResponse:
return FileResponse(filepath,
request=request,
content_type="text/yaml")
config.add_route(route_name, route)
config.add_view(route_name=route_name, view=spec_view)
custom_formatters = config.registry.settings.get(
"pyramid_openapi3_formatters")
config.registry.settings["pyramid_openapi3"] = {
"filepath": filepath,
"spec_route_name": route_name,
"spec": spec,
"request_validator": RequestValidator(spec, custom_formatters),
"response_validator": ResponseValidator(spec, custom_formatters),
}
def validate_requests_and_responses(spec_dict, api_url, parameters=None):
parameters = parameters if parameters else {'paths': {}}
spec = create_spec(spec_dict)
global counters
for path, path_object in spec.paths.items():
counters['paths'] += 1
for method, operation in path_object.operations.items():
method_upper = method.upper()
if method_upper not in counters['methods']:
counters['methods'][method_upper] = 1
else:
counters['methods'][method_upper] += 1
print('{} {}'.format(method_upper, path))
for real_path, path_pattern, path_params in path_parameter_substitute(path, parameters):
if method == 'get':
if path in parameters['paths'] and 'get' in parameters['paths'][path]:
for params in parameters['paths'][path]['get']:
if not isinstance(params, dict):
skip('GET method example payload is incorrect: {}'.format(params))
continue
url = api_url + real_path
req = requests.Request('GET', url, params=params)
counters['errors'] += validate_request(req, path_pattern, path_params, spec, url)
else:
url = api_url + real_path
req = requests.Request('GET', url)
counters['errors'] += validate_request(req, path_pattern, path_params, spec, url)
elif method == 'post':
if path in parameters['paths'] and 'post' in parameters['paths'][path]:
for post_payload_string in parameters['paths'][path]['post']:
url = api_url + real_path
req = requests.Request('POST', url, data=post_payload_string.strip(),
headers={'content-type': list(operation.request_body.content.keys())[0]})
counters['errors'] += validate_request(req, path_pattern, path_params, spec, url)
else:
skip('POST method has no example payload to test')
continue
else:
skip('no GET or POST methods defined for this path')
continue
# Print total stats
print()
print(color(' Total paths discovered: {}'.format(counters['paths']), fg='white', style='bold'))
print(color(' Methods discovered: ', fg='white', style='bold'))
for method, count in counters['methods'].items():
print(color(' {}: {}'.format(method, count), fg='white', style='bold'))
print(color(' Total requests made: {}'.format(counters['requests']), fg='white', style='bold'))
if counters['skips']:
print(color(' [SKIP] Total {:d} skips '.format(counters['skips']), fg='white', bg='yellow', style='bold'))
if counters['errors']:
print(color(' [FAIL] Total {:d} errors found '.format(counters['errors']), fg='white', bg='red', style='bold'))
return 1
else:
return 0
def setup(app: web.Application):
"""Setup the rest API module in the application in aiohttp fashion.
- users "rest" section of configuration (see schema in rest_config.py)
- loads and validate openapi specs from a remote (e.g. apihub) or local location
- connects openapi specs paths to handlers (see rest_routes.py)
- enables error, validation and envelope middlewares on API routes
IMPORTANT: this is a critical subsystem. Any failure should stop
the system startup. It CANNOT be simply disabled & continue
"""
log.debug("Setting up %s ...", __name__)
spec_path = resources.get_path("api/v0/openapi.yaml")
with spec_path.open() as fh:
spec_dict = yaml.safe_load(fh)
api_specs = openapi_core.create_spec(spec_dict, spec_path.as_uri())
# validated openapi specs
app[APP_OPENAPI_SPECS_KEY] = api_specs
# Connects handlers
routes = rest_routes.create(api_specs)
app.router.add_routes(routes)
log.debug("routes:\n\t%s", "\n\t".join(map(str, routes)))
# Enable error, validation and envelop middleware on API routes
base_path = get_base_path(api_specs)
append_rest_middlewares(app, base_path)
async def prepare(self) -> None:
maybe_coro = super().prepare()
if maybe_coro and asyncio.iscoroutine(maybe_coro): # pragma: no cover
await maybe_coro
spec = create_spec(yaml.safe_load(self.render_string("openapi.yaml")))
validator = RequestValidator(spec)
result = validator.validate(self.request)
try:
result.raise_for_errors()
except OperationNotFound:
self.set_status(405)
self.finish()
except InvalidContentType:
self.set_status(415)
self.finish()
except (DeserializeError, ValidateError):
self.set_status(400)
self.finish()
except InvalidSecurity:
self.set_status(401)
self.finish()
except OpenAPIError:
raise
self.validated = result
def test_bad_url_parameters(self) -> None:
spec = create_spec({
"openapi": "3.0.0",
"info": {
"title": "Test specification",
"version": "0.1"
},
"paths": {
"/{id}": {
"get": {
"parameters": [{
"name": "id",
"in": "path",
"required": True,
"schema": {
"type": "integer"
},
}],
"responses": {
"default": {
"description": "Root response"
}
},
}
}
},
})
validator = RequestValidator(spec)
self.fetch("/abcd")
assert self.request is not None
result = validator.validate(self.request)
with self.assertRaises(OpenAPIError):
result.raise_for_errors()
def test_simple_request_fails_without_parameters(
self, parameters: Parameters) -> None:
spec = create_spec({
"openapi": "3.0.0",
"info": {
"title": "Test specification",
"version": "0.1"
},
"paths": {
"/": {
"get": {
"parameters": parameters.as_openapi(),
"responses": {
"default": {
"description": "Root response"
}
},
}
}
},
})
validator = RequestValidator(spec)
self.fetch("/")
assert self.request is not None
result = validator.validate(self.request)
with self.assertRaises(MissingRequiredParameter):
result.raise_for_errors()
def __init__(
self,
api_spec_path: Optional[str] = None,
server: Optional[str] = None,
logger: logging.Logger = _default_logger,
):
"""
Initialize the API spec.
:param api_spec_path: Directory API path and filename of the API spec YAML source file.
"""
self._validator = None # type: Optional[RequestValidator]
self.logger = logger
if api_spec_path is not None:
try:
api_spec_dict = read_yaml_file(api_spec_path)
if server is not None:
api_spec_dict["servers"] = [{"url": server}]
api_spec = create_spec(api_spec_dict)
self._validator = RequestValidator(api_spec)
except OpenAPIValidationError as e: # pragma: nocover
self.logger.error(
f"API specification YAML source file not correctly formatted: {str(e)}"
)
except Exception:
self.logger.exception(
"API specification YAML source file not correctly formatted."
)
raise
def check_reload(self) -> None:
# Because importing yaml takes significant time, and we only
# use python-yaml for our API docs, importing it lazily here
# is a significant optimization to `manage.py` startup.
#
# There is a bit of a race here...we may have two processes
# accessing this module level object and both trying to
# populate self.data at the same time. Hopefully this will
# only cause some extra processing at startup and not data
# corruption.
import yaml
from jsonref import JsonRef
with open(self.openapi_path) as f:
mtime = os.fstat(f.fileno()).st_mtime
# Using == rather than >= to cover the corner case of users placing an
# earlier version than the current one
if self.mtime == mtime:
return
openapi = yaml.load(f, Loader=yaml.CSafeLoader)
spec = create_spec(openapi)
self._request_validator = RequestValidator(spec)
self._openapi = naively_merge_allOf_dict(JsonRef.replace_refs(openapi))
self.create_endpoints_dict()
self.mtime = mtime
async def test_remuneration_endpoint(client):
resp = await client.get("/schema")
spec = create_spec(json.loads(resp.body))
resp = await client.post(
"/remuneration",
body={
"beneficiaire.age": 20,
"formation.region": 27,
"formation.codes_financeur": [2],
},
)
assert resp.status == HTTPStatus.OK
assert "remunerations" in json.loads(resp.body)
remunerations = json.loads(resp.body)["remunerations"]
assert remunerations
print(remunerations[0])
assert "remuneration" in remunerations[0]
assert "Version" in resp.headers
validator = ResponseValidator(spec)
request = MockRequest("http://trefle.pole-emploi.fr", "post",
"/remuneration")
response = MockResponse(resp.body, resp.status.value)
result = validator.validate(request, response)
result.raise_for_errors()
def test_can_create_specs_from_path(openapi_path):
spec_dict, spec_url = _load_from_path(Path(openapi_path))
specs = openapi_core.create_spec(spec_dict, spec_url)
assert specs
assert isinstance(specs, OpenApiSpec)
async def test_simulate_endpoint(client):
resp = await client.get("/schema")
spec = create_spec(json.loads(resp.body))
resp = await client.post(
"/financement",
body={
"beneficiaire.solde_cpf": 10,
"beneficiaire.remuneration": 1400,
"beneficiaire.droit_prive": True,
"beneficiaire.contrat": "cdi",
"formation.eligible_cpf": True,
"formation.heures": 100,
"beneficiaire.entreprise.commune": "2A004",
"beneficiaire.entreprise.idcc": 2706,
},
)
assert resp.status == HTTPStatus.OK
assert "financements" in json.loads(resp.body)
financements = json.loads(resp.body)["financements"]
print(financements[0])
assert financements
assert financements[0].get("eligible")
assert "Version" in resp.headers
validator = ResponseValidator(spec)
request = MockRequest("http://trefle.pole-emploi.fr", "post",
"/financement")
response = MockResponse(resp.body, resp.status.value)
result = validator.validate(request, response)
result.raise_for_errors()
def reload(self) -> None:
# Because importing yamole (and in turn, yaml) takes
# significant time, and we only use python-yaml for our API
# docs, importing it lazily here is a significant optimization
# to `manage.py` startup.
#
# There is a bit of a race here...we may have two processes
# accessing this module level object and both trying to
# populate self.data at the same time. Hopefully this will
# only cause some extra processing at startup and not data
# corruption.
from yamole import YamoleParser
with open(self.path) as f:
yaml_parser = YamoleParser(f)
self.data = yaml_parser.data
validator_spec = create_spec(self.data)
self.core_data = RequestValidator(validator_spec)
self.create_regex_dict()
self.last_update = os.path.getmtime(self.path)
# Form a set of all documented events
self.documented_events.clear()
schema = (self.data['paths']['/events']['get']['responses']
['200']['content']['application/json']['schema'])
for events in schema['properties']['events']['items']['oneOf']:
op_str = ':'
if 'op' in events['properties']:
op_str = f':{events["properties"]["op"]["enum"][0]}'
self.documented_events.add(events['properties']['type']['enum'][0]
+ op_str)
def test_simple_request(self, parameters: Parameters) -> None:
spec = create_spec({
"openapi": "3.0.0",
"info": {
"title": "Test specification",
"version": "0.1"
},
"paths": {
"/": {
"get": {
"parameters": parameters.as_openapi(),
"responses": {
"default": {
"description": "Root response"
}
},
}
}
},
})
validator = RequestValidator(spec)
self.fetch(
"/?" + urlencode(parameters.query_parameters),
headers=HTTPHeaders(parameters.headers),
)
assert self.request is not None
result = validator.validate(self.request)
result.raise_for_errors()
def check_reload(self) -> None:
# Because importing yamole (and in turn, yaml) takes
# significant time, and we only use python-yaml for our API
# docs, importing it lazily here is a significant optimization
# to `manage.py` startup.
#
# There is a bit of a race here...we may have two processes
# accessing this module level object and both trying to
# populate self.data at the same time. Hopefully this will
# only cause some extra processing at startup and not data
# corruption.
from yamole import YamoleParser
mtime = os.path.getmtime(self.openapi_path)
# Using == rather than >= to cover the corner case of users placing an
# earlier version than the current one
if self.mtime == mtime:
return
with open(self.openapi_path) as f:
yamole_parser = YamoleParser(f)
self._openapi = yamole_parser.data
spec = create_spec(self._openapi)
self._request_validator = RequestValidator(spec)
self.create_endpoints_dict()
self.mtime = mtime
def register() -> None:
settings = config.registry.settings.get(apiname)
if settings and settings.get("spec") is not None:
raise ConfigurationError(
"Spec has already been configured. You may only call "
"pyramid_openapi3_spec or pyramid_openapi3_spec_directory once"
)
if hupper.is_active(): # pragma: no cover
hupper.get_reloader().watch_files([filepath])
spec_dict = read_yaml_file(filepath)
validate_spec(spec_dict)
spec = create_spec(spec_dict)
def spec_view(request: Request) -> FileResponse:
return FileResponse(filepath, request=request, content_type="text/yaml")
config.add_route(route_name, route)
config.add_view(route_name=route_name, permission=permission, view=spec_view)
custom_formatters = config.registry.settings.get("pyramid_openapi3_formatters")
config.registry.settings[apiname] = {
"filepath": filepath,
"spec_route_name": route_name,
"spec": spec,
"request_validator": RequestValidator(
spec, custom_formatters=custom_formatters
),
"response_validator": ResponseValidator(
spec, custom_formatters=custom_formatters
),
}
APIS.append(apiname)
def load_openapi_spec(self):
openapi_file = self.spec_path / 'openapi.v1beta0.yml'
if openapi_file.is_file():
with open(openapi_file, 'r') as fp:
yaml_spec = yaml.load(fp)
self.specs['openapi'] = openapi_core.create_spec(
json.loads(json.dumps(yaml_spec))) # openapi_validator not work with ints after yaml.load
def get_openapi_spec():
global openapi_spec
if not openapi_spec:
with open(os.path.join(BASE_PATH, "openapi.yml"), "r") as spec_file:
spec_dict = load(spec_file, Loader=yaml.Loader)
openapi_spec = create_spec(spec_dict)
return openapi_spec
def create_schema(self):
path = os.path.join(os.path.dirname(__file__), "openapi-derefed.json")
with open(path, "r") as json_file:
data = json.load(json_file)
data["servers"][0]["url"] = settings.SENTRY_OPTIONS["system.url-prefix"]
del data["components"]
return create_spec(data)
def load_openapi_specs(spec_path: Optional[Path] = None) -> OpenApiSpecs:
if spec_path is None:
spec_path = get_openapi_specs_path()
with spec_path.open() as fh:
spec_dict = yaml.safe_load(fh)
specs: OpenApiSpecs = openapi_core.create_spec(spec_dict, spec_path.as_uri())
return specs
def load_spec():
"""Validate openapi spec."""
napp_dir = Path(__file__).parent
yml_file = napp_dir / "openapi.yml"
spec_dict, _ = read_from_filename(yml_file)
validate_spec(spec_dict)
return create_spec(spec_dict)
def set_api(self, file: FileStorage):
if file.filename.endswith('.json'):
content_type = 'text/json'
elif file.filename.endswith('.yaml') or file.filename.endswith('.yml'):
content_type = 'text/yaml'
else:
raise EBadRequest(
'File must have a .json, .yaml, or .yml extension')
#
spec_dir = self.spec_dir()
# if os.path.exists(spec_dir):
# raise EConflict("An API was already uploaded to this project.")
#
content = _preprocess_spec_file(file)
api = create_spec(content)
modules: Dict[str, Module]
graphs: Dict[str, FlowGraph]
modules, graphs = convert(api)
info: Info = api.info
print('JCT INFO ====================')
print('JCT INFO ====================')
print('JCT INFO ====================')
print("INFO", info.__dict__)
print('JCT INFO ====================')
print('JCT INFO ====================')
print('JCT INFO ====================')
self.target.api_filename = file.filename
self.target.api_version = info.version
self.target.description = info.title
#
mdir = self.modules.init_dirs()
# modules = _parse_and_save_module_list(api, self.modmap_filename())
for mod_id in modules.keys():
mod_file = Patterns.project_module_file(self.tag, mod_id)
save_yaml(mod_file, modules[mod_id].flatten())
graph_file = Patterns.project_graph_file(self.tag, mod_id)
save_yaml(graph_file, graphs[mod_id].flatten())
# self.modules.add_module(mod_id, graphs[mod_id])
# _save_raw_module(mod_dir, mod)
# self.modules.save()
#
# self.target.api_version = content['info']['version']
# self.target.api_filename = file.filename
#
self.dirty = True
# api = OpenApi.from_data(content)
#
_save_api_file(spec_dir, file)
return self
def create_specs(openapi_path: Path) -> OpenApiSpec:
warnings.warn("Use instead create_openapi_specs",
category=DeprecationWarning)
# TODO: spec_from_file and spec_from_url
with openapi_path.open() as f:
spec_dict = yaml.safe_load(f)
spec = openapi_core.create_spec(spec_dict, spec_url=openapi_path.as_uri())
return spec
def spec(self) -> SpecPath:
"""The OpenAPI 3 specification.
Override this in your request handlers to customize how your OpenAPI 3
spec is loaded and validated.
:rtype: :class:`openapi_core.schema.specs.model.Spec`
"""
return create_spec(self.spec_dict, validate_spec=False)
def load_spec(self, spec_file):
log.debug('%s.load_spec: %s' % (self.__class__.__name__, spec_file))
# TODO: supporting loading from url instead of just file
# TODO: How to handle/interpret/respect spec.servers[].url's?
# TODO: Or should this be generated/injected into the spec_dict on startup?
#spec_file = '/home/sar/vcs/nameko-openapi/petstore.yaml'
spec_dict = yaml.safe_load(open(spec_file))
self.spec = openapi_core.create_spec(spec_dict)
self.request_validator = RequestValidator(self.spec)
self.response_validator = ResponseValidator(self.spec)
self._loaded.send(self.spec)
def test_can_create_specs_from_path(openapi_path: str):
# NOTE: get as 'str' so path fixture can be rendered in test log
oas_path = Path(openapi_path)
with oas_path.open() as fh:
spec_dict = yaml.safe_load(fh)
# will raise if openapi_core cannot handle OAS
specs = openapi_core.create_spec(spec_dict, oas_path.as_uri())
assert specs
assert isinstance(specs, OpenApiSpec)
def get_openapi_spec():
"""Get the OpenAPI spec object."""
try:
from openapi_core import OpenAPISpec as Spec
create_spec = Spec.create
except ImportError:
from openapi_core import create_spec
openapi_spec_dict = get_openapi_spec_dict()
return create_spec(openapi_spec_dict)
def create_schema(self):
if not APIDocsTestCase.cached_schema:
path = os.path.join(os.path.dirname(__file__),
"openapi-derefed.json")
with open(path) as json_file:
data = json.load(json_file)
data["servers"][0]["url"] = settings.SENTRY_OPTIONS[
"system.url-prefix"]
del data["components"]
APIDocsTestCase.cached_schema = create_spec(data)
return APIDocsTestCase.cached_schema
def register() -> None:
settings = config.registry.settings.get(apiname)
if settings and settings.get("spec") is not None:
raise ConfigurationError(
"Spec has already been configured. You may only call "
"pyramid_openapi3_spec or pyramid_openapi3_spec_directory once"
)
if route.endswith((".yaml", ".yml", ".json")):
raise ConfigurationError(
"Having route be a filename is not allowed when using a spec directory"
)
path = Path(filepath).resolve()
if hupper.is_active(): # pragma: no cover
hupper.get_reloader().watch_files(list(path.parent.iterdir()))
spec_dict = read_yaml_file(path)
spec_url = path.as_uri()
validate_spec(spec_dict, spec_url=spec_url)
spec = create_spec(spec_dict, spec_url=spec_url)
config.add_static_view(route, str(path.parent), permission=permission)
config.add_route(route_name, f"{route}/{path.name}")
custom_formatters = config.registry.settings.get(
"pyramid_openapi3_formatters")
custom_deserializers = config.registry.settings.get(
"pyramid_openapi3_deserializers")
config.registry.settings[apiname] = {
"filepath":
filepath,
"spec_route_name":
route_name,
"spec":
spec,
"request_validator":
RequestValidator(
spec,
custom_formatters=custom_formatters,
custom_media_type_deserializers=custom_deserializers,
),
"response_validator":
ResponseValidator(
spec,
custom_formatters=custom_formatters,
custom_media_type_deserializers=custom_deserializers,
),
}
config.registry.settings.setdefault("pyramid_openapi3_apinames",
[]).append(apiname)
def parse_spec(inp_file):
"""Parse a yaml file into a specification object."""
try:
y_spec = yaml.load(inp_file, Loader=yaml.SafeLoader)
spec = create_spec(y_spec)
except jsonschema.exceptions.RefResolutionError:
logging.error(
"Could not load specification. Check your network or try again")
raise err.BeaconTestError()
except openapi_spec_validator.exceptions.OpenAPIValidationError:
logging.error(
"Could not read specification. Check tat your file is valid")
raise err.BeaconTestError()
return spec
def sent_request(client14, container):
path = container['request_data']['path']
resp = client14.simulate_get('{}/spec/1.4'.format(path))
assert HTTP_OK == resp.status
spec = create_spec(resp.json)
req = FalconOpenAPIWrapper(app, **container['request_data'])
validator = RequestValidator(spec)
result = validator.validate(req)
assert not result.errors
validator = ResponseValidator(spec)
result = validator.validate(req, req)
assert not result.errors
container['response'] = req.request
def _load_spec(version):
global _LOADED_SPECS
if _LOADED_SPECS.get(version):
return _LOADED_SPECS[version]
spec_path = os.path.join(SCHEMAS_DIR,
'v{}'.format(version) if version else 'latest',
'patchwork.yaml')
with open(spec_path, 'r') as fh:
data = yaml.load(fh)
_LOADED_SPECS[version] = openapi_core.create_spec(data)
return _LOADED_SPECS[version]
def register() -> None:
spec_dict = read_yaml_file(filepath)
validate_spec(spec_dict)
spec = create_spec(spec_dict)
def spec_view(request: Request) -> FileResponse:
return FileResponse(filepath, request=request, content_type="text/yaml")
config.add_route(route_name, route)
config.add_view(route_name=route_name, view=spec_view)
custom_formatters = config.registry.settings.get("pyramid_openapi3_formatters")
config.registry.settings["pyramid_openapi3"] = {
"filepath": filepath,
"spec_route_name": route_name,
"spec": spec,
"request_validator": RequestValidator(spec, custom_formatters),
"response_validator": ResponseValidator(spec, custom_formatters),
}