def _getspec(resource_name, resource_url, download_spec=False): """ Returns the openapi spec :param resource_name: (str) the name of the resource. :param resource_url: (str) URL to download the resource spec file. :return: (openapi_core.schema.specs.models.Spec) The Spec object associated with this resource. """ if download_spec: try: response = requests.get(resource_url) if response.status_code == 200: try: spec_dict = yaml.load(response.content) return create_spec(spec_dict) # for now, if there are errors trying to fetch the latest spec, we fall back to the spec files defined in the # the python-sdk package; except: pass except: pass try: # for now, hardcode the paths; we could look these up based on a canonical URL once that is # established. spec_path = f'/home/tapis/tapy/dyna/resources/openapi_v3-{resource_name}.yml' spec_dict = yaml.load(open(spec_path, 'r')) return create_spec(spec_dict) except Exception as e: print( f"Got exception trying to load spec_path: {spec_path}; exception: {e}" ) raise e
async def create_openapi_specs(location, session: ClientSession = None) -> OpenApiSpec: """Loads specs from a given location (url or path), validates them and returns a working instance If location is an url, the specs are loaded asyncronously Both location types (url and file) are intentionally managed by the same function call to enforce developer always supporting both options. Notice that the url location enforces the consumer context to be asyncronous. :param location: url or path :return: validated openapi specifications object :rtype: OpenApiSpec """ if URL(str(location)).host: if session is None: raise ValueError("Client session required in arguments") spec_dict, spec_url = await _load_from_url(session, URL(location)) else: path = Path(location).expanduser().resolve() # pylint: disable=no-member spec_dict, spec_url = _load_from_path(path) return openapi_core.create_spec(spec_dict, spec_url)
def register() -> None: if hupper.is_active(): # pragma: no cover hupper.get_reloader().watch_files([filepath]) spec_dict = read_yaml_file(filepath) validate_spec(spec_dict) spec = create_spec(spec_dict) def spec_view(request: Request) -> FileResponse: return FileResponse(filepath, request=request, content_type="text/yaml") config.add_route(route_name, route) config.add_view(route_name=route_name, view=spec_view) custom_formatters = config.registry.settings.get( "pyramid_openapi3_formatters") config.registry.settings["pyramid_openapi3"] = { "filepath": filepath, "spec_route_name": route_name, "spec": spec, "request_validator": RequestValidator(spec, custom_formatters), "response_validator": ResponseValidator(spec, custom_formatters), }
def validate_requests_and_responses(spec_dict, api_url, parameters=None): parameters = parameters if parameters else {'paths': {}} spec = create_spec(spec_dict) global counters for path, path_object in spec.paths.items(): counters['paths'] += 1 for method, operation in path_object.operations.items(): method_upper = method.upper() if method_upper not in counters['methods']: counters['methods'][method_upper] = 1 else: counters['methods'][method_upper] += 1 print('{} {}'.format(method_upper, path)) for real_path, path_pattern, path_params in path_parameter_substitute(path, parameters): if method == 'get': if path in parameters['paths'] and 'get' in parameters['paths'][path]: for params in parameters['paths'][path]['get']: if not isinstance(params, dict): skip('GET method example payload is incorrect: {}'.format(params)) continue url = api_url + real_path req = requests.Request('GET', url, params=params) counters['errors'] += validate_request(req, path_pattern, path_params, spec, url) else: url = api_url + real_path req = requests.Request('GET', url) counters['errors'] += validate_request(req, path_pattern, path_params, spec, url) elif method == 'post': if path in parameters['paths'] and 'post' in parameters['paths'][path]: for post_payload_string in parameters['paths'][path]['post']: url = api_url + real_path req = requests.Request('POST', url, data=post_payload_string.strip(), headers={'content-type': list(operation.request_body.content.keys())[0]}) counters['errors'] += validate_request(req, path_pattern, path_params, spec, url) else: skip('POST method has no example payload to test') continue else: skip('no GET or POST methods defined for this path') continue # Print total stats print() print(color(' Total paths discovered: {}'.format(counters['paths']), fg='white', style='bold')) print(color(' Methods discovered: ', fg='white', style='bold')) for method, count in counters['methods'].items(): print(color(' {}: {}'.format(method, count), fg='white', style='bold')) print(color(' Total requests made: {}'.format(counters['requests']), fg='white', style='bold')) if counters['skips']: print(color(' [SKIP] Total {:d} skips '.format(counters['skips']), fg='white', bg='yellow', style='bold')) if counters['errors']: print(color(' [FAIL] Total {:d} errors found '.format(counters['errors']), fg='white', bg='red', style='bold')) return 1 else: return 0
def setup(app: web.Application): """Setup the rest API module in the application in aiohttp fashion. - users "rest" section of configuration (see schema in rest_config.py) - loads and validate openapi specs from a remote (e.g. apihub) or local location - connects openapi specs paths to handlers (see rest_routes.py) - enables error, validation and envelope middlewares on API routes IMPORTANT: this is a critical subsystem. Any failure should stop the system startup. It CANNOT be simply disabled & continue """ log.debug("Setting up %s ...", __name__) spec_path = resources.get_path("api/v0/openapi.yaml") with spec_path.open() as fh: spec_dict = yaml.safe_load(fh) api_specs = openapi_core.create_spec(spec_dict, spec_path.as_uri()) # validated openapi specs app[APP_OPENAPI_SPECS_KEY] = api_specs # Connects handlers routes = rest_routes.create(api_specs) app.router.add_routes(routes) log.debug("routes:\n\t%s", "\n\t".join(map(str, routes))) # Enable error, validation and envelop middleware on API routes base_path = get_base_path(api_specs) append_rest_middlewares(app, base_path)
async def prepare(self) -> None: maybe_coro = super().prepare() if maybe_coro and asyncio.iscoroutine(maybe_coro): # pragma: no cover await maybe_coro spec = create_spec(yaml.safe_load(self.render_string("openapi.yaml"))) validator = RequestValidator(spec) result = validator.validate(self.request) try: result.raise_for_errors() except OperationNotFound: self.set_status(405) self.finish() except InvalidContentType: self.set_status(415) self.finish() except (DeserializeError, ValidateError): self.set_status(400) self.finish() except InvalidSecurity: self.set_status(401) self.finish() except OpenAPIError: raise self.validated = result
def test_bad_url_parameters(self) -> None: spec = create_spec({ "openapi": "3.0.0", "info": { "title": "Test specification", "version": "0.1" }, "paths": { "/{id}": { "get": { "parameters": [{ "name": "id", "in": "path", "required": True, "schema": { "type": "integer" }, }], "responses": { "default": { "description": "Root response" } }, } } }, }) validator = RequestValidator(spec) self.fetch("/abcd") assert self.request is not None result = validator.validate(self.request) with self.assertRaises(OpenAPIError): result.raise_for_errors()
def test_simple_request_fails_without_parameters( self, parameters: Parameters) -> None: spec = create_spec({ "openapi": "3.0.0", "info": { "title": "Test specification", "version": "0.1" }, "paths": { "/": { "get": { "parameters": parameters.as_openapi(), "responses": { "default": { "description": "Root response" } }, } } }, }) validator = RequestValidator(spec) self.fetch("/") assert self.request is not None result = validator.validate(self.request) with self.assertRaises(MissingRequiredParameter): result.raise_for_errors()
def __init__( self, api_spec_path: Optional[str] = None, server: Optional[str] = None, logger: logging.Logger = _default_logger, ): """ Initialize the API spec. :param api_spec_path: Directory API path and filename of the API spec YAML source file. """ self._validator = None # type: Optional[RequestValidator] self.logger = logger if api_spec_path is not None: try: api_spec_dict = read_yaml_file(api_spec_path) if server is not None: api_spec_dict["servers"] = [{"url": server}] api_spec = create_spec(api_spec_dict) self._validator = RequestValidator(api_spec) except OpenAPIValidationError as e: # pragma: nocover self.logger.error( f"API specification YAML source file not correctly formatted: {str(e)}" ) except Exception: self.logger.exception( "API specification YAML source file not correctly formatted." ) raise
def check_reload(self) -> None: # Because importing yaml takes significant time, and we only # use python-yaml for our API docs, importing it lazily here # is a significant optimization to `manage.py` startup. # # There is a bit of a race here...we may have two processes # accessing this module level object and both trying to # populate self.data at the same time. Hopefully this will # only cause some extra processing at startup and not data # corruption. import yaml from jsonref import JsonRef with open(self.openapi_path) as f: mtime = os.fstat(f.fileno()).st_mtime # Using == rather than >= to cover the corner case of users placing an # earlier version than the current one if self.mtime == mtime: return openapi = yaml.load(f, Loader=yaml.CSafeLoader) spec = create_spec(openapi) self._request_validator = RequestValidator(spec) self._openapi = naively_merge_allOf_dict(JsonRef.replace_refs(openapi)) self.create_endpoints_dict() self.mtime = mtime
async def test_remuneration_endpoint(client): resp = await client.get("/schema") spec = create_spec(json.loads(resp.body)) resp = await client.post( "/remuneration", body={ "beneficiaire.age": 20, "formation.region": 27, "formation.codes_financeur": [2], }, ) assert resp.status == HTTPStatus.OK assert "remunerations" in json.loads(resp.body) remunerations = json.loads(resp.body)["remunerations"] assert remunerations print(remunerations[0]) assert "remuneration" in remunerations[0] assert "Version" in resp.headers validator = ResponseValidator(spec) request = MockRequest("http://trefle.pole-emploi.fr", "post", "/remuneration") response = MockResponse(resp.body, resp.status.value) result = validator.validate(request, response) result.raise_for_errors()
def test_can_create_specs_from_path(openapi_path): spec_dict, spec_url = _load_from_path(Path(openapi_path)) specs = openapi_core.create_spec(spec_dict, spec_url) assert specs assert isinstance(specs, OpenApiSpec)
async def test_simulate_endpoint(client): resp = await client.get("/schema") spec = create_spec(json.loads(resp.body)) resp = await client.post( "/financement", body={ "beneficiaire.solde_cpf": 10, "beneficiaire.remuneration": 1400, "beneficiaire.droit_prive": True, "beneficiaire.contrat": "cdi", "formation.eligible_cpf": True, "formation.heures": 100, "beneficiaire.entreprise.commune": "2A004", "beneficiaire.entreprise.idcc": 2706, }, ) assert resp.status == HTTPStatus.OK assert "financements" in json.loads(resp.body) financements = json.loads(resp.body)["financements"] print(financements[0]) assert financements assert financements[0].get("eligible") assert "Version" in resp.headers validator = ResponseValidator(spec) request = MockRequest("http://trefle.pole-emploi.fr", "post", "/financement") response = MockResponse(resp.body, resp.status.value) result = validator.validate(request, response) result.raise_for_errors()
def reload(self) -> None: # Because importing yamole (and in turn, yaml) takes # significant time, and we only use python-yaml for our API # docs, importing it lazily here is a significant optimization # to `manage.py` startup. # # There is a bit of a race here...we may have two processes # accessing this module level object and both trying to # populate self.data at the same time. Hopefully this will # only cause some extra processing at startup and not data # corruption. from yamole import YamoleParser with open(self.path) as f: yaml_parser = YamoleParser(f) self.data = yaml_parser.data validator_spec = create_spec(self.data) self.core_data = RequestValidator(validator_spec) self.create_regex_dict() self.last_update = os.path.getmtime(self.path) # Form a set of all documented events self.documented_events.clear() schema = (self.data['paths']['/events']['get']['responses'] ['200']['content']['application/json']['schema']) for events in schema['properties']['events']['items']['oneOf']: op_str = ':' if 'op' in events['properties']: op_str = f':{events["properties"]["op"]["enum"][0]}' self.documented_events.add(events['properties']['type']['enum'][0] + op_str)
def test_simple_request(self, parameters: Parameters) -> None: spec = create_spec({ "openapi": "3.0.0", "info": { "title": "Test specification", "version": "0.1" }, "paths": { "/": { "get": { "parameters": parameters.as_openapi(), "responses": { "default": { "description": "Root response" } }, } } }, }) validator = RequestValidator(spec) self.fetch( "/?" + urlencode(parameters.query_parameters), headers=HTTPHeaders(parameters.headers), ) assert self.request is not None result = validator.validate(self.request) result.raise_for_errors()
def check_reload(self) -> None: # Because importing yamole (and in turn, yaml) takes # significant time, and we only use python-yaml for our API # docs, importing it lazily here is a significant optimization # to `manage.py` startup. # # There is a bit of a race here...we may have two processes # accessing this module level object and both trying to # populate self.data at the same time. Hopefully this will # only cause some extra processing at startup and not data # corruption. from yamole import YamoleParser mtime = os.path.getmtime(self.openapi_path) # Using == rather than >= to cover the corner case of users placing an # earlier version than the current one if self.mtime == mtime: return with open(self.openapi_path) as f: yamole_parser = YamoleParser(f) self._openapi = yamole_parser.data spec = create_spec(self._openapi) self._request_validator = RequestValidator(spec) self.create_endpoints_dict() self.mtime = mtime
def register() -> None: settings = config.registry.settings.get(apiname) if settings and settings.get("spec") is not None: raise ConfigurationError( "Spec has already been configured. You may only call " "pyramid_openapi3_spec or pyramid_openapi3_spec_directory once" ) if hupper.is_active(): # pragma: no cover hupper.get_reloader().watch_files([filepath]) spec_dict = read_yaml_file(filepath) validate_spec(spec_dict) spec = create_spec(spec_dict) def spec_view(request: Request) -> FileResponse: return FileResponse(filepath, request=request, content_type="text/yaml") config.add_route(route_name, route) config.add_view(route_name=route_name, permission=permission, view=spec_view) custom_formatters = config.registry.settings.get("pyramid_openapi3_formatters") config.registry.settings[apiname] = { "filepath": filepath, "spec_route_name": route_name, "spec": spec, "request_validator": RequestValidator( spec, custom_formatters=custom_formatters ), "response_validator": ResponseValidator( spec, custom_formatters=custom_formatters ), } APIS.append(apiname)
def load_openapi_spec(self): openapi_file = self.spec_path / 'openapi.v1beta0.yml' if openapi_file.is_file(): with open(openapi_file, 'r') as fp: yaml_spec = yaml.load(fp) self.specs['openapi'] = openapi_core.create_spec( json.loads(json.dumps(yaml_spec))) # openapi_validator not work with ints after yaml.load
def get_openapi_spec(): global openapi_spec if not openapi_spec: with open(os.path.join(BASE_PATH, "openapi.yml"), "r") as spec_file: spec_dict = load(spec_file, Loader=yaml.Loader) openapi_spec = create_spec(spec_dict) return openapi_spec
def create_schema(self): path = os.path.join(os.path.dirname(__file__), "openapi-derefed.json") with open(path, "r") as json_file: data = json.load(json_file) data["servers"][0]["url"] = settings.SENTRY_OPTIONS["system.url-prefix"] del data["components"] return create_spec(data)
def load_openapi_specs(spec_path: Optional[Path] = None) -> OpenApiSpecs: if spec_path is None: spec_path = get_openapi_specs_path() with spec_path.open() as fh: spec_dict = yaml.safe_load(fh) specs: OpenApiSpecs = openapi_core.create_spec(spec_dict, spec_path.as_uri()) return specs
def load_spec(): """Validate openapi spec.""" napp_dir = Path(__file__).parent yml_file = napp_dir / "openapi.yml" spec_dict, _ = read_from_filename(yml_file) validate_spec(spec_dict) return create_spec(spec_dict)
def set_api(self, file: FileStorage): if file.filename.endswith('.json'): content_type = 'text/json' elif file.filename.endswith('.yaml') or file.filename.endswith('.yml'): content_type = 'text/yaml' else: raise EBadRequest( 'File must have a .json, .yaml, or .yml extension') # spec_dir = self.spec_dir() # if os.path.exists(spec_dir): # raise EConflict("An API was already uploaded to this project.") # content = _preprocess_spec_file(file) api = create_spec(content) modules: Dict[str, Module] graphs: Dict[str, FlowGraph] modules, graphs = convert(api) info: Info = api.info print('JCT INFO ====================') print('JCT INFO ====================') print('JCT INFO ====================') print("INFO", info.__dict__) print('JCT INFO ====================') print('JCT INFO ====================') print('JCT INFO ====================') self.target.api_filename = file.filename self.target.api_version = info.version self.target.description = info.title # mdir = self.modules.init_dirs() # modules = _parse_and_save_module_list(api, self.modmap_filename()) for mod_id in modules.keys(): mod_file = Patterns.project_module_file(self.tag, mod_id) save_yaml(mod_file, modules[mod_id].flatten()) graph_file = Patterns.project_graph_file(self.tag, mod_id) save_yaml(graph_file, graphs[mod_id].flatten()) # self.modules.add_module(mod_id, graphs[mod_id]) # _save_raw_module(mod_dir, mod) # self.modules.save() # # self.target.api_version = content['info']['version'] # self.target.api_filename = file.filename # self.dirty = True # api = OpenApi.from_data(content) # _save_api_file(spec_dir, file) return self
def create_specs(openapi_path: Path) -> OpenApiSpec: warnings.warn("Use instead create_openapi_specs", category=DeprecationWarning) # TODO: spec_from_file and spec_from_url with openapi_path.open() as f: spec_dict = yaml.safe_load(f) spec = openapi_core.create_spec(spec_dict, spec_url=openapi_path.as_uri()) return spec
def spec(self) -> SpecPath: """The OpenAPI 3 specification. Override this in your request handlers to customize how your OpenAPI 3 spec is loaded and validated. :rtype: :class:`openapi_core.schema.specs.model.Spec` """ return create_spec(self.spec_dict, validate_spec=False)
def load_spec(self, spec_file): log.debug('%s.load_spec: %s' % (self.__class__.__name__, spec_file)) # TODO: supporting loading from url instead of just file # TODO: How to handle/interpret/respect spec.servers[].url's? # TODO: Or should this be generated/injected into the spec_dict on startup? #spec_file = '/home/sar/vcs/nameko-openapi/petstore.yaml' spec_dict = yaml.safe_load(open(spec_file)) self.spec = openapi_core.create_spec(spec_dict) self.request_validator = RequestValidator(self.spec) self.response_validator = ResponseValidator(self.spec) self._loaded.send(self.spec)
def test_can_create_specs_from_path(openapi_path: str): # NOTE: get as 'str' so path fixture can be rendered in test log oas_path = Path(openapi_path) with oas_path.open() as fh: spec_dict = yaml.safe_load(fh) # will raise if openapi_core cannot handle OAS specs = openapi_core.create_spec(spec_dict, oas_path.as_uri()) assert specs assert isinstance(specs, OpenApiSpec)
def get_openapi_spec(): """Get the OpenAPI spec object.""" try: from openapi_core import OpenAPISpec as Spec create_spec = Spec.create except ImportError: from openapi_core import create_spec openapi_spec_dict = get_openapi_spec_dict() return create_spec(openapi_spec_dict)
def create_schema(self): if not APIDocsTestCase.cached_schema: path = os.path.join(os.path.dirname(__file__), "openapi-derefed.json") with open(path) as json_file: data = json.load(json_file) data["servers"][0]["url"] = settings.SENTRY_OPTIONS[ "system.url-prefix"] del data["components"] APIDocsTestCase.cached_schema = create_spec(data) return APIDocsTestCase.cached_schema
def register() -> None: settings = config.registry.settings.get(apiname) if settings and settings.get("spec") is not None: raise ConfigurationError( "Spec has already been configured. You may only call " "pyramid_openapi3_spec or pyramid_openapi3_spec_directory once" ) if route.endswith((".yaml", ".yml", ".json")): raise ConfigurationError( "Having route be a filename is not allowed when using a spec directory" ) path = Path(filepath).resolve() if hupper.is_active(): # pragma: no cover hupper.get_reloader().watch_files(list(path.parent.iterdir())) spec_dict = read_yaml_file(path) spec_url = path.as_uri() validate_spec(spec_dict, spec_url=spec_url) spec = create_spec(spec_dict, spec_url=spec_url) config.add_static_view(route, str(path.parent), permission=permission) config.add_route(route_name, f"{route}/{path.name}") custom_formatters = config.registry.settings.get( "pyramid_openapi3_formatters") custom_deserializers = config.registry.settings.get( "pyramid_openapi3_deserializers") config.registry.settings[apiname] = { "filepath": filepath, "spec_route_name": route_name, "spec": spec, "request_validator": RequestValidator( spec, custom_formatters=custom_formatters, custom_media_type_deserializers=custom_deserializers, ), "response_validator": ResponseValidator( spec, custom_formatters=custom_formatters, custom_media_type_deserializers=custom_deserializers, ), } config.registry.settings.setdefault("pyramid_openapi3_apinames", []).append(apiname)
def parse_spec(inp_file): """Parse a yaml file into a specification object.""" try: y_spec = yaml.load(inp_file, Loader=yaml.SafeLoader) spec = create_spec(y_spec) except jsonschema.exceptions.RefResolutionError: logging.error( "Could not load specification. Check your network or try again") raise err.BeaconTestError() except openapi_spec_validator.exceptions.OpenAPIValidationError: logging.error( "Could not read specification. Check tat your file is valid") raise err.BeaconTestError() return spec
def sent_request(client14, container): path = container['request_data']['path'] resp = client14.simulate_get('{}/spec/1.4'.format(path)) assert HTTP_OK == resp.status spec = create_spec(resp.json) req = FalconOpenAPIWrapper(app, **container['request_data']) validator = RequestValidator(spec) result = validator.validate(req) assert not result.errors validator = ResponseValidator(spec) result = validator.validate(req, req) assert not result.errors container['response'] = req.request
def _load_spec(version): global _LOADED_SPECS if _LOADED_SPECS.get(version): return _LOADED_SPECS[version] spec_path = os.path.join(SCHEMAS_DIR, 'v{}'.format(version) if version else 'latest', 'patchwork.yaml') with open(spec_path, 'r') as fh: data = yaml.load(fh) _LOADED_SPECS[version] = openapi_core.create_spec(data) return _LOADED_SPECS[version]
def register() -> None: spec_dict = read_yaml_file(filepath) validate_spec(spec_dict) spec = create_spec(spec_dict) def spec_view(request: Request) -> FileResponse: return FileResponse(filepath, request=request, content_type="text/yaml") config.add_route(route_name, route) config.add_view(route_name=route_name, view=spec_view) custom_formatters = config.registry.settings.get("pyramid_openapi3_formatters") config.registry.settings["pyramid_openapi3"] = { "filepath": filepath, "spec_route_name": route_name, "spec": spec, "request_validator": RequestValidator(spec, custom_formatters), "response_validator": ResponseValidator(spec, custom_formatters), }