def schema_to_scopes(schema: dict) -> Iterable[str]: """ Constructs and yields query scopes of the form: <action>:<query_kind>:<arg_name>:<arg_val> where arg_val may be a query kind, or the name of an aggregation unit if applicable, and <action> is run or get_result. Additionally yields the "get_result&available_dates" scope. One scope is yielded for each viable query structure, so for queries which contain two child queries five scopes are yielded. If that query has 3 possible aggregation units, then 13 scopes are yielded altogether. Parameters ---------- flowmachine_query_schemas : dict Schema dict to turn into scopes list Yields ------ str Scope strings Examples -------- >>> list(schema_to_scopes({"FlowmachineQuerySchema": {"oneOf": [{"$ref": "DUMMY"}]},"DUMMY": {"properties": {"query_kind": {"enum": ["dummy"]}}},},)) ["get_result&dummy", "run&dummy", "get_result&available_dates"], """ resolved = ResolvingParser(spec_string=dumps(schema)) yield from per_query_scopes(queries=ResolvingParser( spec_string=dumps(schema)).specification["components"]["schemas"] ["FlowmachineQuerySchema"])
def test_issue_5_integer_keys(): # Must fail in implicit strict mode. with pytest.raises(SwaggerValidationError): ResolvingParser('tests/issue_5.yaml') # Must fail in explicit strict mode. with pytest.raises(SwaggerValidationError): ResolvingParser('tests/issue_5.yaml', strict=True) # Must succeed in non-strict/lenient mode parser = ResolvingParser('tests/issue_5.yaml', strict=False) assert '200' in parser.specification['paths']['/test']['post']['responses']
def parse(version,path): parser = ResolvingParser(path) spec = parser.specification api_list = paths_handle(spec) return api_list pass
def do_fuzzing(mytestcase, headers): self = mytestcase baseurl = "" parser = ResolvingParser("openapi/vAPI.yaml") spec = parser.specification # contains fully resolved specs as a dict # print(json.dumps(parser.specification.get("paths").get("/employees/expenses/{expenses_id}/attachments").get("post"),indent=2)) for path, pathvalues in spec.get("paths",{}).items(): for method,methodvalues in pathvalues.items(): pathvars = {} # postvars = {} if method == 'get': if 'parameters' in methodvalues.keys(): pathvars = methodvalues.get("parameters",{}) responses = list(methodvalues.get("responses",{}).keys()) # print("--------------------------------------------") # print("GET fuzzing {}".format(path)) do_get_fuzzing(mytestcase=self, baseurl=baseurl, headers=headers, path=path, pathvars=pathvars, responses=responses) if method == 'post': responses = list(methodvalues.get("responses",{}).keys()) if 'requestBody' in methodvalues.keys() and 'parameters' in methodvalues.keys(): pathvars = methodvalues.get("parameters") postvars = methodvalues.get("requestBody",{}).get("content",{}).get("application/json",{}).get("schema",{}).get("properties",{}) # print("--------------------------------------------") # print("POST fuzzing param URL {}:".format(path)) do_post_fuzzing(mytestcase=self, baseurl=baseurl, headers=headers, path=path, pathvars=pathvars, postvars=postvars, responses=responses) elif 'requestBody' in methodvalues.keys(): postvars = methodvalues.get("requestBody",{}).get("content",{}).get("application/json",{}).get("schema",{}).get("properties",{}) # print("--------------------------------------------") # print("POST fuzzing non-param URL {}:".format(path)) do_post_fuzzing(mytestcase=self, baseurl=baseurl, headers=headers, path=path, postvars=postvars, responses=responses)
def parse_and_resolve(schema: Dict[str, Any], *, remove_desciptions: bool = False) -> Dict[str, Any]: # Insert schema into OpenAPI specification skeleton openapi_spec = ruamel.yaml.load(OPENAPI_V3_SKELETON, Loader=ruamel.yaml.SafeLoader) openapi_spec['components']['schemas']['crd_schema'] = schema with NamedTemporaryFile('w+', encoding='utf-8', suffix='.yaml') as openapi_spec_f: # Use default_flow_style=False to always force block-style for collections, otherwise ruamel.yaml's C parser # has problems parsing results like "collection: {$ref: ...}". ruamel.yaml.dump(openapi_spec, openapi_spec_f, default_flow_style=False) openapi_spec_f.flush() openapi_spec_f.seek(0, SEEK_SET) # Parse file and resolve references parser = ResolvingParser(openapi_spec_f.name, backend='openapi-spec-validator') resolved_schema = parser.specification['components']['schemas'][ 'crd_schema'] # Remove any Kubernetes extensions remove_k8s_extentions(resolved_schema) # Remove descriptions if requested if remove_desciptions: remove_k8s_descriptions(resolved_schema, schema) return resolved_schema
def test_issue_65_partial_resolution_files(): specs = '''openapi: "3.0.0" info: title: '' version: '1.0.0' paths: {} components: schemas: SampleArray: type: array items: $ref: '#/components/schemas/ItemType' ItemType: type: integer ''' from prance.util import resolver parser = ResolvingParser(spec_string=specs, resolve_types=resolver.RESOLVE_FILES) from prance.util.path import path_get val = path_get(parser.specification, ('components', 'schemas', 'SampleArray', 'items')) assert '$ref' in val
def __init__(self, schema, base_path=None, default_policy=None): schema = 'python://' + schema.strip('/') try: parser = ResolvingParser(schema, backend='openapi-spec-validator') except (FileNotFoundError, IsADirectoryError): raise NotFoundError('OpenAPI Schema not found') self._spec = parser.specification self._paths = self._spec.get('paths', []) self._policy = {} if base_path: self._spec['basePath'] = base_path.rstrip('/') self._request_body = {} self._response_body = {} self._request_params = {} self._operation_ids = {} self._routes = [] self._schemas = [] for path in self._paths: route = self.base_path + '/' + path.strip('/') for method in self._paths[path]: op_id = self._paths[path][method].get('operationId') params = self._paths[path][method].get('parameters', []) resps = self._paths[path][method].get('responses', []) if op_id: if op_id not in self._operation_ids: self._operation_ids[op_id] = [] self._request_params[op_id] = {} self._request_body[op_id] = None self._response_body[op_id] = {} for param in params: if param.get('in', '') == 'query': p_name = param.get('name') if p_name: self._request_params[op_id][p_name] = param elif param.get('in', '') == 'body': self._request_body[op_id] = param for resp in resps: self._response_body[op_id][resp] = resps[resp] _route = ( method, route, op_id, default_policy, ) self._routes.append(_route) if op_id not in self._operation_ids: self._operation_ids[op_id] = [] self._operation_ids[op_id].append(route)
def test_issue_5_integer_keys(): # Must fail in implicit strict mode. with pytest.raises(SwaggerValidationError): ResolvingParser('tests/issue_5.yaml', backend='swagger-spec-validator') # Must fail in explicit strict mode. with pytest.raises(SwaggerValidationError): ResolvingParser('tests/issue_5.yaml', backend='swagger-spec-validator', strict=True) # Must succeed in non-strict/lenient mode parser = ResolvingParser('tests/issue_5.yaml', backend='swagger-spec-validator', strict=False) assert '200' in parser.specification['paths']['/test']['post']['responses'] # Must succeed with default (flex) parser; note the parser does not stringify the response code parser = ResolvingParser('tests/issue_5.yaml', backend='flex') assert 200 in parser.specification['paths']['/test']['post']['responses']
def test_issue_39_sequence_indices(): # Must not fail to parse parser = ResolvingParser('tests/specs/issue_39.yaml', backend = 'openapi-spec-validator') # The /useCase path should have two values in its response example. example = parser.specification['paths']['/useCase']['get']['responses']['200']['content']['application/json']['examples']['response'] assert 'value' in example assert len(example['value']) == 2 # However, the /test path should have only one of the strings. example = parser.specification['paths']['/test']['get']['responses']['200']['content']['application/json']['example'] assert example == 'some really long or specific string'
def test_issue_83_skip_propagation(): # Throw with strict parsing parser = ResolvingParser('tests/specs/issue_83/bad_spec.yml', lazy = True, backend = 'openapi-spec-validator', strict = True) with pytest.raises(ValidationError): parser.parse() # Do not throw with non-strict parsing parser = ResolvingParser('tests/specs/issue_83/bad_spec.yml', lazy = True, backend = 'openapi-spec-validator', strict = False) parser.parse()
def __init__(self, api_file=""): """ Sets the api_file and the parser attributes up for the class. :param api_file: str url, optional """ if api_file: self.api_file = api_file if not self.api_file: raise ImplementationError("There must be an API file set to use the APIParser class.") self.parser = ResolvingParser(self.api_file) self.milmove_data = MilMoveData() # for generating fake requests self.processed_bodies = [] # list of APIEndpointBody objects self.discriminated = False # indicates if the parser is working with the data for a discriminator
def test_issue_51_encoding_error(): # Parsing used to throw - but shouldn't after heuristic change. parser = ResolvingParser('tests/specs/issue_51/openapi-main.yaml', lazy = True, backend = 'openapi-spec-validator', strict = True) parser.parse() # Parsing with setting an explicit and wrong file encoding should raise # an error, effectively reverting to the old behaviour parser = ResolvingParser('tests/specs/issue_51/openapi-main.yaml', lazy = True, backend = 'openapi-spec-validator', strict = True, encoding = 'iso-8859-2') from yaml.reader import ReaderError with pytest.raises(ReaderError): parser.parse()
def _parse_file(path, base=None): try: parser = ResolvingParser(str(path), strict=False) return parser.specification except ( AssertionError, AttributeError, ComposerError, FileNotFoundError, ResolutionError, ScannerError, UnicodeDecodeError, ValidationError, ) as err: log.info( "repos.views.openapi.invalid", path=str(path.relative_to(base)), error=err )
def setup(cls, config_path, openapi_path): with open(config_path) as config_file: config = json.load(config_file) cls.base_url = utils.setup_base_url(config) cls.session = utils.setup_session(config) cls.test_cases = config['test_cases'] cls.local_test = config['local_test'] with open(openapi_path) as openapi_file: openapi = yaml.load(openapi_file, Loader=yaml.SafeLoader) if 'swagger' in openapi: backend = 'flex' elif 'openapi' in openapi: backend = 'openapi-spec-validator' else: exit('Error: could not determine openapi document version') parser = ResolvingParser(openapi_path, backend=backend) cls.openapi = parser.specification
def __init__(self, name, import_name, swagger_spec, static_folder=None, static_url_path=None, template_folder=None, url_prefix=None, subdomain=None, url_defaults=None, root_path=None): init = super(SwaggerBlueprint, self).__init__ init(name, import_name, static_folder, static_url_path, template_folder, url_prefix, subdomain, url_defaults, root_path) self._content = get_content(swagger_spec) self._parser = ResolvingParser(swagger_spec, backend='openapi-spec-validator') self.spec = self._parser.specification self.ops = self._get_operations()
def model_parser(self): if self._model_parser is None: self._model_parser = ResolvingParser( self.name, strict=False, resolve_types=RESOLVE_HTTP | RESOLVE_FILES, resolve_method=TRANSLATE_EXTERNAL) try: schemas = self._model_parser.specification["components"][ "schemas"] for class_name in list(schemas.keys()): if ".yml_schemas_" in class_name: new_class_name = class_name.split(".yml_schemas_")[-1] schemas[new_class_name] = schemas[class_name] del schemas[class_name] except KeyError: pass # no schema return self._model_parser
def __init__(self, contents: dict, properties: dict): self.contents = contents self.phases = [ MappingPhase('openapi', True, self.consume_version), MappingPhase('info', True, self.consume_info), MappingPhase('servers', False, self.consume_servers), MappingPhase('paths', True, self.consume_paths), MappingPhase('components', False, self.consume_components), MappingPhase('security', False, self.consume_security), MappingPhase('tags', False, self.consume_tags), MappingPhase('externalDocs', False, self.consume_external_docs) ] self.resolved_spec = ResolvingParser( spec_string=json.dumps(self.contents)) if not properties: properties = {} self.properties = properties self.op_spec = self.resolved_spec.specification
def validate_spec(): spec_file = cfg.CONF.spec_file generate_spec = cfg.CONF.generate if not os.path.exists(spec_file) and not generate_spec: msg = ('No spec file found in location %s. ' % spec_file + 'Provide a valid spec file or ' + 'pass --generate-api-spec to genrate a spec.') raise Exception(msg) if generate_spec: if not spec_file: raise Exception('Supply a path to write to spec file to.') spec_string = spec_loader.generate_spec('st2common', 'openapi.yaml.j2') with open(spec_file, 'w') as f: f.write(spec_string) f.flush() parser = ResolvingParser(spec_file) spec = parser.specification return _validate_definitions(spec)
def validate_spec(): spec_file = cfg.CONF.spec_file generate_spec = cfg.CONF.generate if not os.path.exists(spec_file) and not generate_spec: msg = ("No spec file found in location %s. " % spec_file + "Provide a valid spec file or " + "pass --generate-api-spec to genrate a spec.") raise Exception(msg) if generate_spec: if not spec_file: raise Exception("Supply a path to write to spec file to.") spec_string = spec_loader.generate_spec("st2common", "openapi.yaml.j2") with open(spec_file, "w") as f: f.write(spec_string) f.flush() parser = ResolvingParser(spec_file) spec = parser.specification return _validate_definitions(spec)
def process_shapes_file(shape_format, shapes_graph, rdf_file_path, repo_url, branch, repo_description): """Process a file, check its content and add entry to the shapes graph Large function, contain parsing for all formats: RDF, OBO, ShEx, OpenAPI, etc """ relative_filepath = str(rdf_file_path)[12:] github_file_url = generate_github_file_url(repo_url, relative_filepath, branch) file_uri = URIRef(github_file_url) shape_found = False g = Graph() if shape_format == 'obo': # Get OBO ontologies try: graph = obonet.read_obo(github_file_url) # for id_, data in graph.nodes(data=True): for id_, data in graph.nodes(data=True): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add( (file_uri, RDF.type, SIO['SIO_000623'])) # OBO ontology shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) shape_label = data.get('name') if not shape_label: shape_label = id_ shapes_graph.add( (file_uri, DCTERMS.hasPart, Literal(shape_label))) except Exception as e: add_to_report('In repository: ' + repo_url + "\n> " + str(e), github_file_url) # Index OpenAPI files elif shape_format == 'openapi': try: parser = ResolvingParser(github_file_url) shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, SCHEMA['APIReference'])) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) file_descriptions = [] if parser.specification['info']['title']: file_descriptions.append(parser.specification['info']['title']) if parser.specification['info']['description']: file_descriptions.append( parser.specification['info']['description']) if len(file_descriptions) > 0: shapes_graph.add((file_uri, RDFS.comment, Literal(' - '.join(file_descriptions)))) # if not shape_label: # shape_label = id_ # TODO: get operations hasPart? shapes_graph.add((file_uri, DCTERMS.hasPart, Literal('OpenAPI'))) except Exception as e: pass # TODO: YARRML? Search for prefixes and mappings at the root of YAML # add_to_report('In repository: ' + repo_url + "\n> " # + str(e), github_file_url) # Search for shex files elif shape_format == 'shex': # No parsing possible for shex shape_found = True # TODO: use https://schema.org/SoftwareSourceCode ? shapes_graph.add((file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, SHEX.Schema)) shapes_graph.add((file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add((file_uri, DCTERMS.hasPart, Literal('ShEx model'))) shapes_graph.add((file_uri, SCHEMA.codeRepository, URIRef(repo_url))) # Convert ShEx to RDF shex and parse it # shex_rdf = '' # if rdf_file_path.endswith('.shex'): # with open(root / '../' + rdf_file_path, 'a') as f: # shex_rdf = generate_shexj.parse(f.read()) # # if rdf_file_path.endswith('.shexj'): # # with open(root / '../' + rdf_file_path, 'a') as f: # # shex_rdf = f.read() # logging.debug(shex_rdf) # # for shape in g.subjects(RDF.type, SHEX.ShapeAnd): # # add_shape_to_graph(shapes_graph, rdf_file_path, github_file_url, repo_url, shape, SHEX.schema) # # for shape in g.subjects(RDF.type, SHEX.Shape): # # add_shape_to_graph(shapes_graph, rdf_file_path, github_file_url, repo_url, shape, SHEX.schema) # Parse SPARQL query files elif shape_format == 'sparql': # TODO: sparql+queries search failing might be due to a test SPARQL query hanging for long time shape_found = True shapes_graph.add((file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, SH.SPARQLFunction)) shapes_graph.add((file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add((file_uri, SCHEMA.codeRepository, URIRef(repo_url))) try: with open(rdf_file_path.absolute()) as file: sparql_query = file.read() # Parse SPARQL query (added fix for some malformed queries with =+ instead of #+) sparql_query = "\n".join([ '#+' + row.lstrip('=+') for row in sparql_query.split('\n') if row.startswith('=+') ]) yaml_string = "\n".join([ row.lstrip('#+') for row in sparql_query.split('\n') if row.startswith('#+') ]) query_string = "\n".join([ row for row in sparql_query.split('\n') if not row.startswith('#+') ]) shapes_graph.add( (file_uri, SCHEMA['query'], Literal(sparql_query))) grlc_metadata = {} try: # Invalid YAMLs will produce empty metadata grlc_metadata = yaml.load(yaml_string, Loader=yaml.FullLoader) except: pass # Get grlc query metadata if grlc_metadata: file_descriptions = [] if 'endpoint' in grlc_metadata: sparql_endpoint = grlc_metadata['endpoint'] try: shapes_graph.add((file_uri, VOID.sparqlEndpoint, URIRef(sparql_endpoint))) test_sparql_endpoint(sparql_endpoint) except Exception as e: logging.debug( 'Issue parsing SPARQL endpoint from .rq file') logging.debug(e) if 'summary' in grlc_metadata and grlc_metadata['summary']: file_descriptions.append(grlc_metadata['summary']) if 'description' in grlc_metadata and grlc_metadata[ 'description']: file_descriptions.append(grlc_metadata['description']) # Add the query description to the graph if len(file_descriptions) > 0: shapes_graph.add( (file_uri, RDFS.comment, Literal(' - '.join(file_descriptions)))) # If default params described for grlc SPARQL query we add them as shapes if 'defaults' in grlc_metadata: for args in grlc_metadata['defaults']: for arg, default_label in args.items(): shapes_graph.add( (file_uri, DCTERMS.hasPart, Literal(arg))) try: # Parse the query to get its operation (select, construct..) parsed_query = translateQuery( Query.parseString(query_string, parseAll=True)) query_operation = re.sub(r"(\w)([A-Z])", r"\1 \2", parsed_query.algebra.name) shapes_graph.add( (file_uri, DCTERMS.hasPart, Literal(query_operation))) except: shapes_graph.add( (file_uri, DCTERMS.hasPart, Literal('SPARQL Query'))) except: logging.error('❌️ Issue opening file: ' + str(rdf_file_path)) # Parse RDF files else: try: if shape_format == 'trig': # Different graph required for trig to work g = ConjunctiveGraph() g.parse(str(rdf_file_path.absolute()), format=shape_format) except Exception as e: if shape_format == 'xml' and (str(rdf_file_path).endswith('.owl') or str(rdf_file_path).endswith('.rdf')): # Try parsing with turtle for .owl and .rdf files try: g.parse(str(rdf_file_path.absolute()), format='ttl') except: add_to_report( 'RDF parsed as ' + shape_format + ', in repository: ' + repo_url + "\n> " + str(e), github_file_url) else: add_to_report( 'RDF parsed as ' + shape_format + ', in repository: ' + repo_url + "\n> " + str(e), github_file_url) # Search for SHACL shapes for shape in g.subjects(RDF.type, SH.NodeShape): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, SH.Shape)) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) shape_label = shape for label in g.objects(shape, RDFS.label): # Try to get the label of the shape shape_label = label # Fixing shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label))) # Search for CSV on the Web RDF (csvw) # https://medium.swirrl.com/how-to-publish-csv-on-the-web-csvw-4ea6cbb603b4 # https://www.w3.org/ns/csvw for shape_file in g.subjects(RDF.type, CSVW.Schema): # for shape_file in g.objects(None, CSVW.tableSchema): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, CSVW.Schema)) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) # Get file label # for file_label in g.objects(shape_file, RDFS.label): # shapes_graph.add((file_uri, RDFS.comment, Literal(str(file_label)))) # break # Get columns label for col_label in g.objects(shape_file, CSVW.column): shapes_graph.add( (file_uri, DCTERMS.hasPart, Literal(str(col_label)))) # Search for DCAT Datasets for shape_file in g.subjects(RDF.type, DCAT.Dataset): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, DCAT.Dataset)) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) # Get file label for file_label in g.objects(shape_file, RDFS.label): shapes_graph.add( (file_uri, RDFS.comment, Literal(str(file_label)))) break # shape_label = shape_file # for label in g.objects(shape_file, RDFS.label): # # Try to get the label of the shape # shape_label = label # # Fixing # shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label))) # Search for nanopublication templates for shape_file in g.subjects(RDF.type, NP_TEMPLATE.AssertionTemplate): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add( (file_uri, RDF.type, NP_TEMPLATE.AssertionTemplate)) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) # Get template label for template_label in g.objects(shape_file, RDFS.label): shapes_graph.add( (file_uri, RDFS.comment, Literal(str(template_label)))) break # TODO: get the shapes inside nanopub_inputs = [ NP_TEMPLATE.GuidedChoicePlaceholder, NP_TEMPLATE.LiteralPlaceholder, NP_TEMPLATE.RestrictedChoicePlaceholder, NP_TEMPLATE.UriPlaceholder ] for np_input in nanopub_inputs: for shape in g.subjects(RDF.type, np_input): shape_label = shape for label in g.objects(shape, RDFS.label): # Try to get the label of the shape shape_label = label # Fixing shapes_graph.add( (file_uri, DCTERMS.hasPart, Literal(shape_label))) # Search for RML and R2RML mappings for shape in g.subjects(RDF.type, R2RML.SubjectMap): shape_found = True is_rml_mappings = False # Differenciate RML and R2RML mappings if (None, RML.logicalSource, None) in g: shapes_graph.add((file_uri, RDF.type, RML.LogicalSource)) else: shapes_graph.add((file_uri, RDF.type, R2RML.TriplesMap)) shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) shape_label = shape # Try to get the label or URI of the subjectMap for label in g.objects(shape, R2RML.template): shape_label = label for label in g.objects(shape, RDFS.label): shape_label = label shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label))) # Search for OWL classes for shape in g.subjects(RDF.type, OWL.Class): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, OWL.Ontology)) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) shape_label = shape for label in g.objects(shape, RDFS.label): # Try to get the label of the class shape_label = label shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label))) # Get rdfs:label of owl:Ontology and shaclTest:Validate for file description file_descriptions = [] for shape in g.subjects(RDF.type, OWL.Ontology): # Get one of the labels for ontology_label in g.objects(shape, RDFS.label): if len(file_descriptions) < 1: file_descriptions.append(str(ontology_label)) if len(file_descriptions) == 0: for label in g.objects(shape, DC.title): file_descriptions.append(str(label)) if len(file_descriptions) == 0: for label in g.objects(shape, DCTERMS.title): file_descriptions.append(str(label)) # Now add the description for comment in g.objects(shape, RDFS.comment): file_descriptions.append(str(comment)) for label in g.objects(shape, RDFS.comment): file_descriptions.append(str(label)) for description in g.objects(shape, DCTERMS.description): file_descriptions.append(str(description)) for shape in g.subjects( RDF.type, URIRef('http://www.w3.org/ns/shacl-test#Validate')): for ontology_label in g.objects(shape, RDFS.label): file_descriptions.append(str(ontology_label)) if len(file_descriptions) > 0: shapes_graph.add((file_uri, RDFS.comment, Literal(' - '.join(file_descriptions)))) # Get SKOS concepts and concept scheme for shape in g.subjects(RDF.type, SKOS.Concept): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, SKOS.ConceptScheme)) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) shape_label = shape for label in g.objects(shape, SKOS.prefLabel): # Try to get the label of the class shape_label = label shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label))) for shape in g.subjects(RDF.type, SKOS.ConceptScheme): # Get one of the labels for ontology_label in g.objects(shape, RDFS.label): if len(file_descriptions) < 1: file_descriptions.append(str(ontology_label)) if len(file_descriptions) == 0: for label in g.objects(shape, DC.title): file_descriptions.append(str(label)) if len(file_descriptions) == 0: for label in g.objects(shape, DCTERMS.title): file_descriptions.append(str(label)) # Now add the description for comment in g.objects(shape, RDFS.comment): file_descriptions.append(str(comment)) for label in g.objects(shape, RDFS.comment): file_descriptions.append(str(label)) for description in g.objects(shape, DCTERMS.description): file_descriptions.append(str(description)) # Search for ShEx Shapes and ShapeAnd # TODO: Improve for shape in g.subjects(RDF.type, SHEX.ShapeAnd): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, SHEX.Schema)) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) shape_label = shape for label in g.objects(shape, RDFS.label): # Try to get the label of the shape shape_label = label shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label))) for shape in g.subjects(RDF.type, SHEX.Shape): shape_found = True shapes_graph.add( (file_uri, RDF.type, SCHEMA['SoftwareSourceCode'])) shapes_graph.add((file_uri, RDF.type, SHEX.Schema)) shapes_graph.add( (file_uri, RDFS.label, Literal(rdf_file_path.name))) shapes_graph.add( (file_uri, SCHEMA.codeRepository, URIRef(repo_url))) shape_label = shape for label in g.objects(shape, RDFS.label): # Try to get the label of the shape shape_label = label shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label))) # Add the git repo to the graph if shape_found: logging.debug('[' + datetime.now().strftime("%m/%d/%Y, %H:%M:%S") + '] ' + "✔️ Shape found in file " + github_file_url) shapes_graph.add((URIRef(repo_url), RDF.type, SCHEMA['DataCatalog'])) shapes_graph.add( (URIRef(repo_url), RDFS.label, Literal(repo_url.rsplit('/', 1)[1]))) if (repo_description): shapes_graph.add( (URIRef(repo_url), RDFS.comment, Literal(repo_description))) return shapes_graph
def __init__(self, fname): self.parser = ResolvingParser(fname)
def __load_swagger_spec(self, file_path): return ResolvingParser(url=file_path)
def issue_1_parser(): return ResolvingParser('tests/specs/issue_1.json')
def petstore_parser_from_string(): yaml = None with open('tests/specs/petstore.yaml', 'rb') as f: x = f.read() yaml = x.decode('utf8') return ResolvingParser(spec_string=yaml)
def petstore_parser(): return ResolvingParser('tests/specs/petstore.yaml')
def resolve_schema_refs(schema) -> dict: """Replaces the $refs within an openapi3.0 schema with the referenced components""" parser = ResolvingParser(spec_string=schema, backend="openapi-spec-validator") return parser.specification
def create_app(runtime_environment): connexion_options = {"swagger_ui": True} # This feels like a hack but it is needed. The logging configuration # needs to be setup before the flask app is initialized. configure_logging() app_config = Config(runtime_environment) app_config.log_configuration() connexion_app = connexion.App("inventory", specification_dir="./swagger/", options=connexion_options) # Read the swagger.yml file to configure the endpoints parser = ResolvingParser(SPECIFICATION_FILE, resolve_types=RESOLVE_FILES) parser.parse() for api_url in app_config.api_urls: if api_url: connexion_app.add_api( parser.specification, arguments={"title": "RestyResolver Example"}, resolver=RestyResolver("api"), validate_responses=True, strict_validation=True, base_path=api_url, ) logger.info("Listening on API: %s", api_url) # Add an error handler that will convert our top level exceptions # into error responses connexion_app.add_error_handler(InventoryException, render_exception) flask_app = connexion_app.app flask_app.config["SQLALCHEMY_ECHO"] = False flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False flask_app.config["SQLALCHEMY_DATABASE_URI"] = app_config.db_uri flask_app.config["SQLALCHEMY_POOL_SIZE"] = app_config.db_pool_size flask_app.config["SQLALCHEMY_POOL_TIMEOUT"] = app_config.db_pool_timeout flask_app.config["INVENTORY_CONFIG"] = app_config db.init_app(flask_app) register_shutdown(db.get_engine(flask_app).dispose, "Closing database") flask_app.register_blueprint(monitoring_blueprint, url_prefix=app_config.mgmt_url_path_prefix) @flask_app.before_request def set_request_id(): threadctx.request_id = request.headers.get(REQUEST_ID_HEADER, UNKNOWN_REQUEST_ID_VALUE) if runtime_environment.event_producer_enabled: flask_app.event_producer = EventProducer(app_config) register_shutdown(flask_app.event_producer.close, "Closing EventProducer") else: logger.warning( "WARNING: The event producer has been disabled. " "The message queue based event notifications have been disabled.") payload_tracker_producer = None if not runtime_environment.payload_tracker_enabled: # If we are running in "testing" mode, then inject the NullProducer. payload_tracker_producer = payload_tracker.NullProducer() logger.warning( "WARNING: Using the NullProducer for the payload tracker producer. " "No payload tracker events will be sent to to payload tracker.") payload_tracker.init_payload_tracker(app_config, producer=payload_tracker_producer) # HTTP request metrics if runtime_environment.metrics_endpoint_enabled: PrometheusMetrics( flask_app, defaults_prefix="inventory", group_by="url_rule", path=None, excluded_paths=[ "^/metrics$", "^/health$", "^/version$", r"^/favicon\.ico$" ], ) # initialize metrics to zero initialize_metrics(app_config) return flask_app
'string': {'default': DataType(type_hint='str'), 'byte': DataType(type_hint='str'), 'binary': DataType(type_hint='bytes') }, # 'data': date,}, #As defined by full-date - RFC3339 'boolean': {'default': DataType(type_hint='bool')} } def get_data_type(_type, format=None) -> DataType: _format: str = format or 'default' return data_types[_type][_format] resolving_parser = ResolvingParser('api.yaml', backend='openapi-spec-validator') base_parser = BaseParser('api.yaml', backend='openapi-spec-validator') class Parser: def __init__(self, data_model_type: Type[DataModel], data_model_field_type: Type[DataModelField]): self.data_model_type: Type[DataModel] = data_model_type self.data_model_field_type: Type[DataModelField] = data_model_field_type self.models = [] def parse_object(self, name: str, obj: Dict): requires: Set[str] = set(obj.get('required', [])) d_list: List[DataModelField] = [] for field_name, filed in obj['properties'].items(): # object d_list.append(self.data_model_field_type(
from prance import ResolvingParser import json parser = ResolvingParser('swag.json') swag = parser.specification # contains fully resolved specs as a dict open('resolved.json', 'w').close() with open('resolved.json', 'w') as outfile: json.dump(swag, outfile, indent=4)
def with_externals_parser(mock_get): mock_get.side_effect = mock_get_petstore return ResolvingParser('tests/specs/with_externals.yaml')