def schema_to_scopes(schema: dict) -> Iterable[str]:
"""
Constructs and yields query scopes of the form:
<action>:<query_kind>:<arg_name>:<arg_val>
where arg_val may be a query kind, or the name of an aggregation unit if applicable, and <action> is run or get_result.
Additionally yields the "get_result&available_dates" scope.
One scope is yielded for each viable query structure, so for queries which contain two child queries
five scopes are yielded. If that query has 3 possible aggregation units, then 13 scopes are yielded altogether.
Parameters
----------
flowmachine_query_schemas : dict
Schema dict to turn into scopes list
Yields
------
str
Scope strings
Examples
--------
>>> list(schema_to_scopes({"FlowmachineQuerySchema": {"oneOf": [{"$ref": "DUMMY"}]},"DUMMY": {"properties": {"query_kind": {"enum": ["dummy"]}}},},))
["get_result&dummy", "run&dummy", "get_result&available_dates"],
"""
resolved = ResolvingParser(spec_string=dumps(schema))
yield from per_query_scopes(queries=ResolvingParser(
spec_string=dumps(schema)).specification["components"]["schemas"]
["FlowmachineQuerySchema"])
def test_issue_5_integer_keys():
# Must fail in implicit strict mode.
with pytest.raises(SwaggerValidationError):
ResolvingParser('tests/issue_5.yaml')
# Must fail in explicit strict mode.
with pytest.raises(SwaggerValidationError):
ResolvingParser('tests/issue_5.yaml', strict=True)
# Must succeed in non-strict/lenient mode
parser = ResolvingParser('tests/issue_5.yaml', strict=False)
assert '200' in parser.specification['paths']['/test']['post']['responses']
def parse(version,path):
parser = ResolvingParser(path)
spec = parser.specification
api_list = paths_handle(spec)
return api_list
pass
def do_fuzzing(mytestcase, headers):
self = mytestcase
baseurl = ""
parser = ResolvingParser("openapi/vAPI.yaml")
spec = parser.specification # contains fully resolved specs as a dict
# print(json.dumps(parser.specification.get("paths").get("/employees/expenses/{expenses_id}/attachments").get("post"),indent=2))
for path, pathvalues in spec.get("paths",{}).items():
for method,methodvalues in pathvalues.items():
pathvars = {}
# postvars = {}
if method == 'get':
if 'parameters' in methodvalues.keys():
pathvars = methodvalues.get("parameters",{})
responses = list(methodvalues.get("responses",{}).keys())
# print("--------------------------------------------")
# print("GET fuzzing {}".format(path))
do_get_fuzzing(mytestcase=self, baseurl=baseurl, headers=headers, path=path, pathvars=pathvars, responses=responses)
if method == 'post':
responses = list(methodvalues.get("responses",{}).keys())
if 'requestBody' in methodvalues.keys() and 'parameters' in methodvalues.keys():
pathvars = methodvalues.get("parameters")
postvars = methodvalues.get("requestBody",{}).get("content",{}).get("application/json",{}).get("schema",{}).get("properties",{})
# print("--------------------------------------------")
# print("POST fuzzing param URL {}:".format(path))
do_post_fuzzing(mytestcase=self, baseurl=baseurl, headers=headers, path=path, pathvars=pathvars, postvars=postvars, responses=responses)
elif 'requestBody' in methodvalues.keys():
postvars = methodvalues.get("requestBody",{}).get("content",{}).get("application/json",{}).get("schema",{}).get("properties",{})
# print("--------------------------------------------")
# print("POST fuzzing non-param URL {}:".format(path))
do_post_fuzzing(mytestcase=self, baseurl=baseurl, headers=headers, path=path, postvars=postvars, responses=responses)
def parse_and_resolve(schema: Dict[str, Any],
*,
remove_desciptions: bool = False) -> Dict[str, Any]:
# Insert schema into OpenAPI specification skeleton
openapi_spec = ruamel.yaml.load(OPENAPI_V3_SKELETON,
Loader=ruamel.yaml.SafeLoader)
openapi_spec['components']['schemas']['crd_schema'] = schema
with NamedTemporaryFile('w+', encoding='utf-8',
suffix='.yaml') as openapi_spec_f:
# Use default_flow_style=False to always force block-style for collections, otherwise ruamel.yaml's C parser
# has problems parsing results like "collection: {$ref: ...}".
ruamel.yaml.dump(openapi_spec,
openapi_spec_f,
default_flow_style=False)
openapi_spec_f.flush()
openapi_spec_f.seek(0, SEEK_SET)
# Parse file and resolve references
parser = ResolvingParser(openapi_spec_f.name,
backend='openapi-spec-validator')
resolved_schema = parser.specification['components']['schemas'][
'crd_schema']
# Remove any Kubernetes extensions
remove_k8s_extentions(resolved_schema)
# Remove descriptions if requested
if remove_desciptions:
remove_k8s_descriptions(resolved_schema, schema)
return resolved_schema
def test_issue_65_partial_resolution_files():
specs = '''openapi: "3.0.0"
info:
title: ''
version: '1.0.0'
paths: {}
components:
schemas:
SampleArray:
type: array
items:
$ref: '#/components/schemas/ItemType'
ItemType:
type: integer
'''
from prance.util import resolver
parser = ResolvingParser(spec_string=specs,
resolve_types=resolver.RESOLVE_FILES)
from prance.util.path import path_get
val = path_get(parser.specification,
('components', 'schemas', 'SampleArray', 'items'))
assert '$ref' in val
def __init__(self, schema, base_path=None, default_policy=None):
schema = 'python://' + schema.strip('/')
try:
parser = ResolvingParser(schema, backend='openapi-spec-validator')
except (FileNotFoundError, IsADirectoryError):
raise NotFoundError('OpenAPI Schema not found')
self._spec = parser.specification
self._paths = self._spec.get('paths', [])
self._policy = {}
if base_path:
self._spec['basePath'] = base_path.rstrip('/')
self._request_body = {}
self._response_body = {}
self._request_params = {}
self._operation_ids = {}
self._routes = []
self._schemas = []
for path in self._paths:
route = self.base_path + '/' + path.strip('/')
for method in self._paths[path]:
op_id = self._paths[path][method].get('operationId')
params = self._paths[path][method].get('parameters', [])
resps = self._paths[path][method].get('responses', [])
if op_id:
if op_id not in self._operation_ids:
self._operation_ids[op_id] = []
self._request_params[op_id] = {}
self._request_body[op_id] = None
self._response_body[op_id] = {}
for param in params:
if param.get('in', '') == 'query':
p_name = param.get('name')
if p_name:
self._request_params[op_id][p_name] = param
elif param.get('in', '') == 'body':
self._request_body[op_id] = param
for resp in resps:
self._response_body[op_id][resp] = resps[resp]
_route = (
method,
route,
op_id,
default_policy,
)
self._routes.append(_route)
if op_id not in self._operation_ids:
self._operation_ids[op_id] = []
self._operation_ids[op_id].append(route)
def test_issue_5_integer_keys():
# Must fail in implicit strict mode.
with pytest.raises(SwaggerValidationError):
ResolvingParser('tests/issue_5.yaml', backend='swagger-spec-validator')
# Must fail in explicit strict mode.
with pytest.raises(SwaggerValidationError):
ResolvingParser('tests/issue_5.yaml',
backend='swagger-spec-validator',
strict=True)
# Must succeed in non-strict/lenient mode
parser = ResolvingParser('tests/issue_5.yaml',
backend='swagger-spec-validator',
strict=False)
assert '200' in parser.specification['paths']['/test']['post']['responses']
# Must succeed with default (flex) parser; note the parser does not stringify the response code
parser = ResolvingParser('tests/issue_5.yaml', backend='flex')
assert 200 in parser.specification['paths']['/test']['post']['responses']
def test_issue_39_sequence_indices():
# Must not fail to parse
parser = ResolvingParser('tests/specs/issue_39.yaml', backend = 'openapi-spec-validator')
# The /useCase path should have two values in its response example.
example = parser.specification['paths']['/useCase']['get']['responses']['200']['content']['application/json']['examples']['response']
assert 'value' in example
assert len(example['value']) == 2
# However, the /test path should have only one of the strings.
example = parser.specification['paths']['/test']['get']['responses']['200']['content']['application/json']['example']
assert example == 'some really long or specific string'
def test_issue_83_skip_propagation():
# Throw with strict parsing
parser = ResolvingParser('tests/specs/issue_83/bad_spec.yml',
lazy = True, backend = 'openapi-spec-validator',
strict = True)
with pytest.raises(ValidationError):
parser.parse()
# Do not throw with non-strict parsing
parser = ResolvingParser('tests/specs/issue_83/bad_spec.yml',
lazy = True, backend = 'openapi-spec-validator',
strict = False)
parser.parse()
def __init__(self, api_file=""):
"""
Sets the api_file and the parser attributes up for the class.
:param api_file: str url, optional
"""
if api_file:
self.api_file = api_file
if not self.api_file:
raise ImplementationError("There must be an API file set to use the APIParser class.")
self.parser = ResolvingParser(self.api_file)
self.milmove_data = MilMoveData() # for generating fake requests
self.processed_bodies = [] # list of APIEndpointBody objects
self.discriminated = False # indicates if the parser is working with the data for a discriminator
def test_issue_51_encoding_error():
# Parsing used to throw - but shouldn't after heuristic change.
parser = ResolvingParser('tests/specs/issue_51/openapi-main.yaml',
lazy = True, backend = 'openapi-spec-validator',
strict = True)
parser.parse()
# Parsing with setting an explicit and wrong file encoding should raise
# an error, effectively reverting to the old behaviour
parser = ResolvingParser('tests/specs/issue_51/openapi-main.yaml',
lazy = True, backend = 'openapi-spec-validator',
strict = True,
encoding = 'iso-8859-2')
from yaml.reader import ReaderError
with pytest.raises(ReaderError):
parser.parse()
def _parse_file(path, base=None):
try:
parser = ResolvingParser(str(path), strict=False)
return parser.specification
except (
AssertionError,
AttributeError,
ComposerError,
FileNotFoundError,
ResolutionError,
ScannerError,
UnicodeDecodeError,
ValidationError,
) as err:
log.info(
"repos.views.openapi.invalid", path=str(path.relative_to(base)), error=err
)
def setup(cls, config_path, openapi_path):
with open(config_path) as config_file:
config = json.load(config_file)
cls.base_url = utils.setup_base_url(config)
cls.session = utils.setup_session(config)
cls.test_cases = config['test_cases']
cls.local_test = config['local_test']
with open(openapi_path) as openapi_file:
openapi = yaml.load(openapi_file, Loader=yaml.SafeLoader)
if 'swagger' in openapi:
backend = 'flex'
elif 'openapi' in openapi:
backend = 'openapi-spec-validator'
else:
exit('Error: could not determine openapi document version')
parser = ResolvingParser(openapi_path, backend=backend)
cls.openapi = parser.specification
def __init__(self,
name,
import_name,
swagger_spec,
static_folder=None,
static_url_path=None,
template_folder=None,
url_prefix=None,
subdomain=None,
url_defaults=None,
root_path=None):
init = super(SwaggerBlueprint, self).__init__
init(name, import_name, static_folder, static_url_path,
template_folder, url_prefix, subdomain, url_defaults, root_path)
self._content = get_content(swagger_spec)
self._parser = ResolvingParser(swagger_spec,
backend='openapi-spec-validator')
self.spec = self._parser.specification
self.ops = self._get_operations()
def model_parser(self):
if self._model_parser is None:
self._model_parser = ResolvingParser(
self.name,
strict=False,
resolve_types=RESOLVE_HTTP | RESOLVE_FILES,
resolve_method=TRANSLATE_EXTERNAL)
try:
schemas = self._model_parser.specification["components"][
"schemas"]
for class_name in list(schemas.keys()):
if ".yml_schemas_" in class_name:
new_class_name = class_name.split(".yml_schemas_")[-1]
schemas[new_class_name] = schemas[class_name]
del schemas[class_name]
except KeyError:
pass # no schema
return self._model_parser
def __init__(self, contents: dict, properties: dict):
self.contents = contents
self.phases = [
MappingPhase('openapi', True, self.consume_version),
MappingPhase('info', True, self.consume_info),
MappingPhase('servers', False, self.consume_servers),
MappingPhase('paths', True, self.consume_paths),
MappingPhase('components', False, self.consume_components),
MappingPhase('security', False, self.consume_security),
MappingPhase('tags', False, self.consume_tags),
MappingPhase('externalDocs', False, self.consume_external_docs)
]
self.resolved_spec = ResolvingParser(
spec_string=json.dumps(self.contents))
if not properties:
properties = {}
self.properties = properties
self.op_spec = self.resolved_spec.specification
def validate_spec():
spec_file = cfg.CONF.spec_file
generate_spec = cfg.CONF.generate
if not os.path.exists(spec_file) and not generate_spec:
msg = ('No spec file found in location %s. ' % spec_file +
'Provide a valid spec file or ' +
'pass --generate-api-spec to genrate a spec.')
raise Exception(msg)
if generate_spec:
if not spec_file:
raise Exception('Supply a path to write to spec file to.')
spec_string = spec_loader.generate_spec('st2common', 'openapi.yaml.j2')
with open(spec_file, 'w') as f:
f.write(spec_string)
f.flush()
parser = ResolvingParser(spec_file)
spec = parser.specification
return _validate_definitions(spec)
def validate_spec():
spec_file = cfg.CONF.spec_file
generate_spec = cfg.CONF.generate
if not os.path.exists(spec_file) and not generate_spec:
msg = ("No spec file found in location %s. " % spec_file +
"Provide a valid spec file or " +
"pass --generate-api-spec to genrate a spec.")
raise Exception(msg)
if generate_spec:
if not spec_file:
raise Exception("Supply a path to write to spec file to.")
spec_string = spec_loader.generate_spec("st2common", "openapi.yaml.j2")
with open(spec_file, "w") as f:
f.write(spec_string)
f.flush()
parser = ResolvingParser(spec_file)
spec = parser.specification
return _validate_definitions(spec)
def process_shapes_file(shape_format, shapes_graph, rdf_file_path, repo_url,
branch, repo_description):
"""Process a file, check its content and add entry to the shapes graph
Large function, contain parsing for all formats: RDF, OBO, ShEx, OpenAPI, etc
"""
relative_filepath = str(rdf_file_path)[12:]
github_file_url = generate_github_file_url(repo_url, relative_filepath,
branch)
file_uri = URIRef(github_file_url)
shape_found = False
g = Graph()
if shape_format == 'obo':
# Get OBO ontologies
try:
graph = obonet.read_obo(github_file_url)
# for id_, data in graph.nodes(data=True):
for id_, data in graph.nodes(data=True):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add(
(file_uri, RDF.type, SIO['SIO_000623'])) # OBO ontology
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
shape_label = data.get('name')
if not shape_label:
shape_label = id_
shapes_graph.add(
(file_uri, DCTERMS.hasPart, Literal(shape_label)))
except Exception as e:
add_to_report('In repository: ' + repo_url + "\n> " + str(e),
github_file_url)
# Index OpenAPI files
elif shape_format == 'openapi':
try:
parser = ResolvingParser(github_file_url)
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, SCHEMA['APIReference']))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
file_descriptions = []
if parser.specification['info']['title']:
file_descriptions.append(parser.specification['info']['title'])
if parser.specification['info']['description']:
file_descriptions.append(
parser.specification['info']['description'])
if len(file_descriptions) > 0:
shapes_graph.add((file_uri, RDFS.comment,
Literal(' - '.join(file_descriptions))))
# if not shape_label:
# shape_label = id_
# TODO: get operations hasPart?
shapes_graph.add((file_uri, DCTERMS.hasPart, Literal('OpenAPI')))
except Exception as e:
pass
# TODO: YARRML? Search for prefixes and mappings at the root of YAML
# add_to_report('In repository: ' + repo_url + "\n> "
# + str(e), github_file_url)
# Search for shex files
elif shape_format == 'shex':
# No parsing possible for shex
shape_found = True
# TODO: use https://schema.org/SoftwareSourceCode ?
shapes_graph.add((file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, SHEX.Schema))
shapes_graph.add((file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add((file_uri, DCTERMS.hasPart, Literal('ShEx model')))
shapes_graph.add((file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
# Convert ShEx to RDF shex and parse it
# shex_rdf = ''
# if rdf_file_path.endswith('.shex'):
# with open(root / '../' + rdf_file_path, 'a') as f:
# shex_rdf = generate_shexj.parse(f.read())
# # if rdf_file_path.endswith('.shexj'):
# # with open(root / '../' + rdf_file_path, 'a') as f:
# # shex_rdf = f.read()
# logging.debug(shex_rdf)
# # for shape in g.subjects(RDF.type, SHEX.ShapeAnd):
# # add_shape_to_graph(shapes_graph, rdf_file_path, github_file_url, repo_url, shape, SHEX.schema)
# # for shape in g.subjects(RDF.type, SHEX.Shape):
# # add_shape_to_graph(shapes_graph, rdf_file_path, github_file_url, repo_url, shape, SHEX.schema)
# Parse SPARQL query files
elif shape_format == 'sparql':
# TODO: sparql+queries search failing might be due to a test SPARQL query hanging for long time
shape_found = True
shapes_graph.add((file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, SH.SPARQLFunction))
shapes_graph.add((file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add((file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
try:
with open(rdf_file_path.absolute()) as file:
sparql_query = file.read()
# Parse SPARQL query (added fix for some malformed queries with =+ instead of #+)
sparql_query = "\n".join([
'#+' + row.lstrip('=+') for row in sparql_query.split('\n')
if row.startswith('=+')
])
yaml_string = "\n".join([
row.lstrip('#+') for row in sparql_query.split('\n')
if row.startswith('#+')
])
query_string = "\n".join([
row for row in sparql_query.split('\n')
if not row.startswith('#+')
])
shapes_graph.add(
(file_uri, SCHEMA['query'], Literal(sparql_query)))
grlc_metadata = {}
try: # Invalid YAMLs will produce empty metadata
grlc_metadata = yaml.load(yaml_string,
Loader=yaml.FullLoader)
except:
pass
# Get grlc query metadata
if grlc_metadata:
file_descriptions = []
if 'endpoint' in grlc_metadata:
sparql_endpoint = grlc_metadata['endpoint']
try:
shapes_graph.add((file_uri, VOID.sparqlEndpoint,
URIRef(sparql_endpoint)))
test_sparql_endpoint(sparql_endpoint)
except Exception as e:
logging.debug(
'Issue parsing SPARQL endpoint from .rq file')
logging.debug(e)
if 'summary' in grlc_metadata and grlc_metadata['summary']:
file_descriptions.append(grlc_metadata['summary'])
if 'description' in grlc_metadata and grlc_metadata[
'description']:
file_descriptions.append(grlc_metadata['description'])
# Add the query description to the graph
if len(file_descriptions) > 0:
shapes_graph.add(
(file_uri, RDFS.comment,
Literal(' - '.join(file_descriptions))))
# If default params described for grlc SPARQL query we add them as shapes
if 'defaults' in grlc_metadata:
for args in grlc_metadata['defaults']:
for arg, default_label in args.items():
shapes_graph.add(
(file_uri, DCTERMS.hasPart, Literal(arg)))
try:
# Parse the query to get its operation (select, construct..)
parsed_query = translateQuery(
Query.parseString(query_string, parseAll=True))
query_operation = re.sub(r"(\w)([A-Z])", r"\1 \2",
parsed_query.algebra.name)
shapes_graph.add(
(file_uri, DCTERMS.hasPart, Literal(query_operation)))
except:
shapes_graph.add(
(file_uri, DCTERMS.hasPart, Literal('SPARQL Query')))
except:
logging.error('❌️ Issue opening file: ' + str(rdf_file_path))
# Parse RDF files
else:
try:
if shape_format == 'trig':
# Different graph required for trig to work
g = ConjunctiveGraph()
g.parse(str(rdf_file_path.absolute()), format=shape_format)
except Exception as e:
if shape_format == 'xml' and (str(rdf_file_path).endswith('.owl')
or
str(rdf_file_path).endswith('.rdf')):
# Try parsing with turtle for .owl and .rdf files
try:
g.parse(str(rdf_file_path.absolute()), format='ttl')
except:
add_to_report(
'RDF parsed as ' + shape_format + ', in repository: ' +
repo_url + "\n> " + str(e), github_file_url)
else:
add_to_report(
'RDF parsed as ' + shape_format + ', in repository: ' +
repo_url + "\n> " + str(e), github_file_url)
# Search for SHACL shapes
for shape in g.subjects(RDF.type, SH.NodeShape):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, SH.Shape))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
shape_label = shape
for label in g.objects(shape, RDFS.label):
# Try to get the label of the shape
shape_label = label
# Fixing
shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label)))
# Search for CSV on the Web RDF (csvw)
# https://medium.swirrl.com/how-to-publish-csv-on-the-web-csvw-4ea6cbb603b4
# https://www.w3.org/ns/csvw
for shape_file in g.subjects(RDF.type, CSVW.Schema):
# for shape_file in g.objects(None, CSVW.tableSchema):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, CSVW.Schema))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
# Get file label
# for file_label in g.objects(shape_file, RDFS.label):
# shapes_graph.add((file_uri, RDFS.comment, Literal(str(file_label))))
# break
# Get columns label
for col_label in g.objects(shape_file, CSVW.column):
shapes_graph.add(
(file_uri, DCTERMS.hasPart, Literal(str(col_label))))
# Search for DCAT Datasets
for shape_file in g.subjects(RDF.type, DCAT.Dataset):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, DCAT.Dataset))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
# Get file label
for file_label in g.objects(shape_file, RDFS.label):
shapes_graph.add(
(file_uri, RDFS.comment, Literal(str(file_label))))
break
# shape_label = shape_file
# for label in g.objects(shape_file, RDFS.label):
# # Try to get the label of the shape
# shape_label = label
# # Fixing
# shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label)))
# Search for nanopublication templates
for shape_file in g.subjects(RDF.type, NP_TEMPLATE.AssertionTemplate):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add(
(file_uri, RDF.type, NP_TEMPLATE.AssertionTemplate))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
# Get template label
for template_label in g.objects(shape_file, RDFS.label):
shapes_graph.add(
(file_uri, RDFS.comment, Literal(str(template_label))))
break
# TODO: get the shapes inside
nanopub_inputs = [
NP_TEMPLATE.GuidedChoicePlaceholder,
NP_TEMPLATE.LiteralPlaceholder,
NP_TEMPLATE.RestrictedChoicePlaceholder,
NP_TEMPLATE.UriPlaceholder
]
for np_input in nanopub_inputs:
for shape in g.subjects(RDF.type, np_input):
shape_label = shape
for label in g.objects(shape, RDFS.label):
# Try to get the label of the shape
shape_label = label
# Fixing
shapes_graph.add(
(file_uri, DCTERMS.hasPart, Literal(shape_label)))
# Search for RML and R2RML mappings
for shape in g.subjects(RDF.type, R2RML.SubjectMap):
shape_found = True
is_rml_mappings = False
# Differenciate RML and R2RML mappings
if (None, RML.logicalSource, None) in g:
shapes_graph.add((file_uri, RDF.type, RML.LogicalSource))
else:
shapes_graph.add((file_uri, RDF.type, R2RML.TriplesMap))
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
shape_label = shape
# Try to get the label or URI of the subjectMap
for label in g.objects(shape, R2RML.template):
shape_label = label
for label in g.objects(shape, RDFS.label):
shape_label = label
shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label)))
# Search for OWL classes
for shape in g.subjects(RDF.type, OWL.Class):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, OWL.Ontology))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
shape_label = shape
for label in g.objects(shape, RDFS.label):
# Try to get the label of the class
shape_label = label
shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label)))
# Get rdfs:label of owl:Ontology and shaclTest:Validate for file description
file_descriptions = []
for shape in g.subjects(RDF.type, OWL.Ontology):
# Get one of the labels
for ontology_label in g.objects(shape, RDFS.label):
if len(file_descriptions) < 1:
file_descriptions.append(str(ontology_label))
if len(file_descriptions) == 0:
for label in g.objects(shape, DC.title):
file_descriptions.append(str(label))
if len(file_descriptions) == 0:
for label in g.objects(shape, DCTERMS.title):
file_descriptions.append(str(label))
# Now add the description
for comment in g.objects(shape, RDFS.comment):
file_descriptions.append(str(comment))
for label in g.objects(shape, RDFS.comment):
file_descriptions.append(str(label))
for description in g.objects(shape, DCTERMS.description):
file_descriptions.append(str(description))
for shape in g.subjects(
RDF.type, URIRef('http://www.w3.org/ns/shacl-test#Validate')):
for ontology_label in g.objects(shape, RDFS.label):
file_descriptions.append(str(ontology_label))
if len(file_descriptions) > 0:
shapes_graph.add((file_uri, RDFS.comment,
Literal(' - '.join(file_descriptions))))
# Get SKOS concepts and concept scheme
for shape in g.subjects(RDF.type, SKOS.Concept):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, SKOS.ConceptScheme))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
shape_label = shape
for label in g.objects(shape, SKOS.prefLabel):
# Try to get the label of the class
shape_label = label
shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label)))
for shape in g.subjects(RDF.type, SKOS.ConceptScheme):
# Get one of the labels
for ontology_label in g.objects(shape, RDFS.label):
if len(file_descriptions) < 1:
file_descriptions.append(str(ontology_label))
if len(file_descriptions) == 0:
for label in g.objects(shape, DC.title):
file_descriptions.append(str(label))
if len(file_descriptions) == 0:
for label in g.objects(shape, DCTERMS.title):
file_descriptions.append(str(label))
# Now add the description
for comment in g.objects(shape, RDFS.comment):
file_descriptions.append(str(comment))
for label in g.objects(shape, RDFS.comment):
file_descriptions.append(str(label))
for description in g.objects(shape, DCTERMS.description):
file_descriptions.append(str(description))
# Search for ShEx Shapes and ShapeAnd
# TODO: Improve
for shape in g.subjects(RDF.type, SHEX.ShapeAnd):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, SHEX.Schema))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
shape_label = shape
for label in g.objects(shape, RDFS.label):
# Try to get the label of the shape
shape_label = label
shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label)))
for shape in g.subjects(RDF.type, SHEX.Shape):
shape_found = True
shapes_graph.add(
(file_uri, RDF.type, SCHEMA['SoftwareSourceCode']))
shapes_graph.add((file_uri, RDF.type, SHEX.Schema))
shapes_graph.add(
(file_uri, RDFS.label, Literal(rdf_file_path.name)))
shapes_graph.add(
(file_uri, SCHEMA.codeRepository, URIRef(repo_url)))
shape_label = shape
for label in g.objects(shape, RDFS.label):
# Try to get the label of the shape
shape_label = label
shapes_graph.add((file_uri, DCTERMS.hasPart, Literal(shape_label)))
# Add the git repo to the graph
if shape_found:
logging.debug('[' + datetime.now().strftime("%m/%d/%Y, %H:%M:%S") +
'] ' + "✔️ Shape found in file " + github_file_url)
shapes_graph.add((URIRef(repo_url), RDF.type, SCHEMA['DataCatalog']))
shapes_graph.add(
(URIRef(repo_url), RDFS.label, Literal(repo_url.rsplit('/',
1)[1])))
if (repo_description):
shapes_graph.add(
(URIRef(repo_url), RDFS.comment, Literal(repo_description)))
return shapes_graph
def __init__(self, fname):
self.parser = ResolvingParser(fname)
def __load_swagger_spec(self, file_path):
return ResolvingParser(url=file_path)
def issue_1_parser():
return ResolvingParser('tests/specs/issue_1.json')
def petstore_parser_from_string():
yaml = None
with open('tests/specs/petstore.yaml', 'rb') as f:
x = f.read()
yaml = x.decode('utf8')
return ResolvingParser(spec_string=yaml)
def petstore_parser():
return ResolvingParser('tests/specs/petstore.yaml')
def resolve_schema_refs(schema) -> dict:
"""Replaces the $refs within an openapi3.0 schema with the referenced components"""
parser = ResolvingParser(spec_string=schema,
backend="openapi-spec-validator")
return parser.specification
def create_app(runtime_environment):
connexion_options = {"swagger_ui": True}
# This feels like a hack but it is needed. The logging configuration
# needs to be setup before the flask app is initialized.
configure_logging()
app_config = Config(runtime_environment)
app_config.log_configuration()
connexion_app = connexion.App("inventory",
specification_dir="./swagger/",
options=connexion_options)
# Read the swagger.yml file to configure the endpoints
parser = ResolvingParser(SPECIFICATION_FILE, resolve_types=RESOLVE_FILES)
parser.parse()
for api_url in app_config.api_urls:
if api_url:
connexion_app.add_api(
parser.specification,
arguments={"title": "RestyResolver Example"},
resolver=RestyResolver("api"),
validate_responses=True,
strict_validation=True,
base_path=api_url,
)
logger.info("Listening on API: %s", api_url)
# Add an error handler that will convert our top level exceptions
# into error responses
connexion_app.add_error_handler(InventoryException, render_exception)
flask_app = connexion_app.app
flask_app.config["SQLALCHEMY_ECHO"] = False
flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
flask_app.config["SQLALCHEMY_DATABASE_URI"] = app_config.db_uri
flask_app.config["SQLALCHEMY_POOL_SIZE"] = app_config.db_pool_size
flask_app.config["SQLALCHEMY_POOL_TIMEOUT"] = app_config.db_pool_timeout
flask_app.config["INVENTORY_CONFIG"] = app_config
db.init_app(flask_app)
register_shutdown(db.get_engine(flask_app).dispose, "Closing database")
flask_app.register_blueprint(monitoring_blueprint,
url_prefix=app_config.mgmt_url_path_prefix)
@flask_app.before_request
def set_request_id():
threadctx.request_id = request.headers.get(REQUEST_ID_HEADER,
UNKNOWN_REQUEST_ID_VALUE)
if runtime_environment.event_producer_enabled:
flask_app.event_producer = EventProducer(app_config)
register_shutdown(flask_app.event_producer.close,
"Closing EventProducer")
else:
logger.warning(
"WARNING: The event producer has been disabled. "
"The message queue based event notifications have been disabled.")
payload_tracker_producer = None
if not runtime_environment.payload_tracker_enabled:
# If we are running in "testing" mode, then inject the NullProducer.
payload_tracker_producer = payload_tracker.NullProducer()
logger.warning(
"WARNING: Using the NullProducer for the payload tracker producer. "
"No payload tracker events will be sent to to payload tracker.")
payload_tracker.init_payload_tracker(app_config,
producer=payload_tracker_producer)
# HTTP request metrics
if runtime_environment.metrics_endpoint_enabled:
PrometheusMetrics(
flask_app,
defaults_prefix="inventory",
group_by="url_rule",
path=None,
excluded_paths=[
"^/metrics$", "^/health$", "^/version$", r"^/favicon\.ico$"
],
)
# initialize metrics to zero
initialize_metrics(app_config)
return flask_app
'string':
{'default': DataType(type_hint='str'),
'byte': DataType(type_hint='str'),
'binary': DataType(type_hint='bytes')
},
# 'data': date,}, #As defined by full-date - RFC3339
'boolean': {'default': DataType(type_hint='bool')}
}
def get_data_type(_type, format=None) -> DataType:
_format: str = format or 'default'
return data_types[_type][_format]
resolving_parser = ResolvingParser('api.yaml', backend='openapi-spec-validator')
base_parser = BaseParser('api.yaml', backend='openapi-spec-validator')
class Parser:
def __init__(self, data_model_type: Type[DataModel], data_model_field_type: Type[DataModelField]):
self.data_model_type: Type[DataModel] = data_model_type
self.data_model_field_type: Type[DataModelField] = data_model_field_type
self.models = []
def parse_object(self, name: str, obj: Dict):
requires: Set[str] = set(obj.get('required', []))
d_list: List[DataModelField] = []
for field_name, filed in obj['properties'].items():
# object
d_list.append(self.data_model_field_type(
from prance import ResolvingParser
import json
parser = ResolvingParser('swag.json')
swag = parser.specification # contains fully resolved specs as a dict
open('resolved.json', 'w').close()
with open('resolved.json', 'w') as outfile:
json.dump(swag, outfile, indent=4)
def with_externals_parser(mock_get):
mock_get.side_effect = mock_get_petstore
return ResolvingParser('tests/specs/with_externals.yaml')
