def validate_links(self, document):
if isinstance(document, list):
iterator = enumerate(document)
elif isinstance(document, dict):
for d in self.url_fields:
if d in document:
if isinstance(document[d], basestring):
if document[d] not in self.idx:
raise validate.ValidationException("Invalid link `%s` in field `%s`" % (document[d], d))
elif isinstance(document[d], list):
for i in document[d]:
if isinstance(i, basestring) and i not in self.idx:
raise validate.ValidationException("Invalid link `%s` in field `%s`" % (i, d))
iterator = document.iteritems()
else:
return
try:
for key, val in iterator:
self.validate_links(val)
except validate.ValidationException as v:
if isinstance(key, basestring):
raise validate.ValidationException("At field `%s`\n%s" % (key, validate.indent(str(v))))
else:
raise validate.ValidationException("At position %s\n%s" % (key, validate.indent(str(v))))
return
def validate_links(self, document):
if isinstance(document, list):
iterator = enumerate(document)
elif isinstance(document, dict):
for d in self.url_fields:
if d in document:
if isinstance(document[d], basestring):
if document[d] not in self.idx:
raise validate.ValidationException("Invalid link `%s` in field `%s`" % (document[d], d))
elif isinstance(document[d], list):
for i in document[d]:
if isinstance(i, basestring) and i not in self.idx:
raise validate.ValidationException("Invalid link `%s` in field `%s`" % (i, d))
iterator = document.iteritems()
else:
return
try:
for key, val in iterator:
self.validate_links(val)
except validate.ValidationException as v:
if isinstance(key, basestring):
raise validate.ValidationException("At field `%s`\n%s" % (key, validate.indent(str(v))))
else:
raise validate.ValidationException("At position %s\n%s" % (key, validate.indent(str(v))))
return
def validate_links(self, document):
docid = self.getid(document)
if docid is None:
docid = ""
errors = []
if isinstance(document, list):
iterator = enumerate(document)
elif isinstance(document, dict):
try:
for d in self.url_fields:
if d not in self.identity_links and d in document:
self.validate_link(d, document[d])
except validate.ValidationException as v:
errors.append(v)
iterator = document.iteritems()
else:
return
for key, val in iterator:
try:
self.validate_links(val)
except validate.ValidationException as v:
if key not in self.nolinkcheck:
docid = self.getid(val)
if docid:
errors.append(
validate.ValidationException(
"While checking object `%s`\n%s" % (docid, validate.indent(str(v)))
)
)
else:
if isinstance(key, basestring):
errors.append(
validate.ValidationException(
"While checking field `%s`\n%s" % (key, validate.indent(str(v)))
)
)
else:
errors.append(
validate.ValidationException(
"While checking position %s\n%s" % (key, validate.indent(str(v)))
)
)
if errors:
if len(errors) > 1:
raise validate.ValidationException("\n".join([str(e) for e in errors]))
else:
raise errors[0]
return
def validate_links(self, document):
docid = self.getid(document)
if docid is None:
docid = ""
errors = []
if isinstance(document, list):
iterator = enumerate(document)
elif isinstance(document, dict):
try:
for d in self.url_fields:
if d not in self.identity_links and d in document:
self.validate_link(d, document[d])
except validate.ValidationException as v:
errors.append(v)
iterator = document.iteritems()
else:
return
for key, val in iterator:
try:
self.validate_links(val)
except validate.ValidationException as v:
if key not in self.nolinkcheck:
docid = self.getid(val)
if docid:
errors.append(
validate.ValidationException(
"While checking object `%s`\n%s" %
(docid, validate.indent(str(v)))))
else:
if isinstance(key, basestring):
errors.append(
validate.ValidationException(
"While checking field `%s`\n%s" %
(key, validate.indent(str(v)))))
else:
errors.append(
validate.ValidationException(
"While checking position %s\n%s" %
(key, validate.indent(str(v)))))
if errors:
if len(errors) > 1:
raise validate.ValidationException("\n".join(
[str(e) for e in errors]))
else:
raise errors[0]
return
def validate_doc(schema_names, validate_doc, loader, strict):
has_root = False
for r in schema_names.names.values():
if r.get_prop("documentRoot"):
has_root = True
break
if not has_root:
raise validate.ValidationException(
"No document roots defined in the schema")
if isinstance(validate_doc, list):
pass
elif isinstance(validate_doc, dict):
validate_doc = [validate_doc]
else:
raise validate.ValidationException("Document must be dict or list")
anyerrors = []
for pos, item in enumerate(validate_doc):
errors = []
success = False
for r in schema_names.names.values():
if r.get_prop("documentRoot"):
try:
validate.validate_ex(
r,
item,
loader.identifiers,
strict,
foreign_properties=loader.foreign_properties)
success = True
break
except validate.ValidationException as e:
errors.append("Could not validate as `%s` because\n%s" %
(r.get_prop("name"),
validate.indent(str(e), nolead=False)))
if not success:
objerr = "Validation error at position %i" % pos
for ident in loader.identifiers:
if ident in item:
objerr = "Validation error in object %s" % (item[ident])
break
anyerrors.append("%s\n%s" %
(objerr, validate.indent("\n".join(errors))))
if anyerrors:
raise validate.ValidationException("\n".join(anyerrors))
def resolve_all(self, document, base_url):
if isinstance(document, list):
iterator = enumerate(document)
elif isinstance(document, dict):
inc = 'include' in document
if 'id' in document or 'import' in document or 'include' in document:
document = self.resolve_ref(document, base_url)
if inc:
return document
for d in self.url_fields:
if d in document:
if isinstance(document[d], basestring):
document[d] = expand_url(document[d], base_url)
elif isinstance(document[d], list):
document[d] = [
expand_url(url, base_url) if isinstance(
url, basestring) else url
for url in document[d]
]
iterator = document.iteritems()
else:
return document
for key, val in iterator:
try:
document[key] = self.resolve_all(val, base_url)
except validate.ValidationException as v:
if isinstance(key, basestring):
raise validate.ValidationException(
"Validation error in field %s:\n%s" %
(key, validate.indent(str(v))))
else:
raise validate.ValidationException(
"Validation error in position %i:\n%s" %
(key, validate.indent(str(v))))
return document
def validate_requirements(self, tool, field):
for r in tool.get(field, []):
try:
if self.names.get_name(r["class"], "") is None:
raise validate.ValidationException("Unknown requirement %s" % (r["class"]))
validate.validate_ex(self.names.get_name(r["class"], ""), r)
if "requirements" in r:
self.validate_requirements(r, "requirements")
if "hints" in r:
self.validate_requirements(r, "hints")
except validate.ValidationException as v:
err = "While validating %s %s\n%s" % (field, r["class"], validate.indent(str(v)))
if field == "hints":
_logger.warn(err)
else:
raise validate.ValidationException(err)
def __init__(self, toolpath_object, docpath):
self.impl = toolpath_object["impl"]
try:
self.embedded_tool = makeTool(
from_url(os.path.join(docpath, self.impl)), docpath)
except validate.ValidationException as v:
raise WorkflowException(
"Tool definition %s failed validation:\n%s" %
(os.path.join(docpath, self.impl), validate.indent(str(v))))
if "id" in toolpath_object:
self.id = toolpath_object["id"]
else:
self.id = "#step_" + str(random.randint(1, 1000000000))
for i in toolpath_object["inputs"]:
d = i["def"][len(self.impl):]
toolid = i.get("id", self.id + "." + idk(d))
found = False
for a in self.embedded_tool.tool["inputs"]:
if a["id"] == d:
i.update(a)
found = True
if not found:
raise WorkflowException(
"Did not find input '%s' in external process" % (i["def"]))
i["id"] = toolid
for i in toolpath_object["outputs"]:
d = i["def"][len(self.impl):]
toolid = i["id"]
found = False
for a in self.embedded_tool.tool["outputs"]:
if a["id"] == d:
i.update(a)
found = True
if not found:
raise WorkflowException(
"Did not find output '%s' in external process" %
(i["def"]))
i["id"] = toolid
super(External, self).__init__(toolpath_object, "Process", docpath)
def validate_doc(schema_names, validate_doc, loader, strict):
has_root = False
for r in schema_names.names.values():
if r.get_prop("documentRoot"):
has_root = True
break
if not has_root:
raise validate.ValidationException("No document roots defined in the schema")
if isinstance(validate_doc, list):
pass
elif isinstance(validate_doc, dict):
validate_doc = [validate_doc]
else:
raise validate.ValidationException("Document must be dict or list")
anyerrors = []
for pos, item in enumerate(validate_doc):
errors = []
success = False
for r in schema_names.names.values():
if r.get_prop("documentRoot"):
try:
validate.validate_ex(r, item, loader.identifiers, strict, foreign_properties=loader.foreign_properties)
success = True
break
except validate.ValidationException as e:
errors.append("Could not validate as `%s` because\n%s" % (r.get_prop("name"), validate.indent(str(e), nolead=False)))
if not success:
objerr = "Validation error at position %i" % pos
for ident in loader.identifiers:
if ident in item:
objerr = "Validation error in object %s" % (item[ident])
break
anyerrors.append("%s\n%s" % (objerr, validate.indent("\n".join(errors))))
if anyerrors:
raise validate.ValidationException("\n".join(anyerrors))
def resolve_all(self, document, base_url, file_base=None):
loader = self
metadata = {}
if file_base is None:
file_base = base_url
if isinstance(document, dict):
# Handle $import and $include
if ('$import' in document or '$include' in document):
return self.resolve_ref(document, file_base)
elif isinstance(document, list):
pass
else:
return document, metadata
newctx = None
if isinstance(document, dict):
# Handle $base, $profile, $namespaces, $schemas and $graph
if "$base" in document:
base_url = document["$base"]
if "$profile" in document:
if not newctx:
newctx = SubLoader(self)
prof = self.fetch(document["$profile"])
newctx.add_namespaces(document.get("$namespaces", {}),
document["$profile"])
newctx.add_schemas(document.get("$schemas", []),
document["$profile"])
if "$namespaces" in document:
if not newctx:
newctx = SubLoader(self)
newctx.add_namespaces(document["$namespaces"])
if "$schemas" in document:
if not newctx:
newctx = SubLoader(self)
newctx.add_schemas(document["$schemas"], file_base)
if newctx:
loader = newctx
if "$graph" in document:
metadata = {k: v for k, v in document.items() if k != "$graph"}
document = document["$graph"]
metadata, _ = loader.resolve_all(metadata, base_url, file_base)
if isinstance(document, dict):
for identifer in loader.identity_links:
if identifer in document:
if isinstance(document[identifer], basestring):
document[identifer] = loader.expand_url(
document[identifer], base_url, scoped=True)
if document[identifer] not in loader.idx or isinstance(
loader.idx[document[identifer]], basestring):
loader.idx[document[identifer]] = document
base_url = document[identifer]
elif isinstance(document[identifer], list):
for n, v in enumerate(document[identifer]):
document[identifer][n] = loader.expand_url(
document[identifer][n], base_url, scoped=True)
if document[identifer][n] not in loader.idx:
loader.idx[document[identifer]
[n]] = document[identifer][n]
for d in document:
d2 = loader.expand_url(d, "", scoped=False, vocab_term=True)
if d != d2:
document[d2] = document[d]
del document[d]
for d in loader.url_fields:
if d in document:
if isinstance(document[d], basestring):
document[d] = loader.expand_url(
document[d],
base_url,
scoped=False,
vocab_term=(d in loader.vocab_fields))
elif isinstance(document[d], list):
document[d] = [
loader.expand_url(
url,
base_url,
scoped=False,
vocab_term=(d in loader.vocab_fields))
if isinstance(url, basestring) else url
for url in document[d]
]
try:
for key, val in document.items():
document[key], _ = loader.resolve_all(
val, base_url, file_base)
except validate.ValidationException as v:
_logger.debug("loader is %s", id(loader))
raise validate.ValidationException(
"(%s) (%s) Validation error in field %s:\n%s" %
(id(loader), file_base, key, validate.indent(str(v))))
elif isinstance(document, list):
i = 0
try:
while i < len(document):
val = document[i]
if isinstance(val, dict) and "$import" in val:
l, _ = loader.resolve_ref(val, file_base)
if isinstance(l, list):
del document[i]
for item in aslist(l):
document.insert(i, item)
i += 1
else:
document[i] = l
i += 1
else:
document[i], _ = loader.resolve_all(
val, base_url, file_base)
i += 1
except validate.ValidationException as v:
raise validate.ValidationException(
"(%s) (%s) Validation error in position %i:\n%s" %
(id(loader), file_base, i, validate.indent(str(v))))
for identifer in loader.identity_links:
if identifer in metadata:
if isinstance(metadata[identifer], basestring):
metadata[identifer] = loader.expand_url(
metadata[identifer], base_url, scoped=True)
loader.idx[metadata[identifer]] = document
return document, metadata
def resolve_all(self, document, base_url):
loader = self
metadata = {}
if isinstance(document, dict):
inc = '@include' in document
if '@import' in document or '@include' in document:
document, _ = self.resolve_ref(document, base_url)
else:
for identifer in self.identity_links:
if identifer in document:
if isinstance(document[identifer], basestring):
document[identifer] = self.expand_url(document[identifer], base_url, scoped=True)
if document[identifer] not in self.idx or isinstance(self.idx[document[identifer]], basestring):
self.idx[document[identifer]] = document
base_url = document[identifer]
elif isinstance(document[identifer], list):
for n, v in enumerate(document[identifer]):
document[identifer][n] = self.expand_url(document[identifer][n], base_url, scoped=True)
if document[identifer][n] not in self.idx:
self.idx[document[identifer][n]] = document[identifer][n]
if inc:
return document, {}
if isinstance(document, dict) and "@context" in document:
loader = Loader(self.ctx)
loader.idx = self.idx
loader.add_context(document["@context"])
if "@base" in loader.ctx:
base_url = loader.ctx["@base"]
if "@graph" in document:
metadata = {k: v for k,v in document.items() if k != "@graph"}
document = document["@graph"]
metadata, _ = self.resolve_all(metadata, base_url)
elif isinstance(document, list):
pass
else:
return document, metadata
try:
if isinstance(document, dict):
for d in document:
d2 = self.expand_url(d, "", scoped=False, vocab_term=True)
if d != d2:
document[d2] = document[d]
del document[d]
for d in loader.url_fields:
if d in document:
if isinstance(document[d], basestring):
document[d] = loader.expand_url(document[d], base_url, scoped=False, vocab_term=(d in loader.vocab_fields))
elif isinstance(document[d], list):
document[d] = [loader.expand_url(url, base_url, scoped=False, vocab_term=(d in loader.vocab_fields)) if isinstance(url, basestring) else url for url in document[d] ]
for key, val in document.items():
document[key], _ = loader.resolve_all(val, base_url)
elif isinstance(document, list):
i = 0
while i < len(document):
val = document[i]
if isinstance(val, dict) and "@import" in val and val.get("inline"):
l, _ = loader.resolve_all(val, base_url)
del document[i]
for item in aslist(l):
document.insert(i, item)
i += 1
else:
document[i], _ = loader.resolve_all(val, base_url)
i += 1
except validate.ValidationException as v:
if isinstance(key, basestring):
raise validate.ValidationException("Validation error in field %s:\n%s" % (key, validate.indent(str(v))))
else:
raise validate.ValidationException("Validation error in position %i:\n%s" % (key, validate.indent(str(v))))
return document, metadata
def resolve_all(self, document, base_url):
if isinstance(document, list):
iterator = enumerate(document)
elif isinstance(document, dict):
inc = 'include' in document
if 'id' in document or 'import' in document or 'include' in document:
document = self.resolve_ref(document, base_url)
if inc:
return document
for d in self.url_fields:
if d in document:
if isinstance(document[d], basestring):
document[d] = expand_url(document[d], base_url)
elif isinstance(document[d], list):
document[d] = [expand_url(url, base_url) if isinstance(url, basestring) else url for url in document[d] ]
iterator = document.iteritems()
else:
return document
for key, val in iterator:
try:
document[key] = self.resolve_all(val, base_url)
except validate.ValidationException as v:
if isinstance(key, basestring):
raise validate.ValidationException("Validation error in field %s:\n%s" % (key, validate.indent(str(v))))
else:
raise validate.ValidationException("Validation error in position %i:\n%s" % (key, validate.indent(str(v))))
return document
def resolve_all(self, document, base_url, file_base=None):
loader = self
metadata = {}
if file_base is None:
file_base = base_url
if isinstance(document, dict):
# Handle $import and $include
if ('$import' in document or '$include' in document):
return self.resolve_ref(document, file_base)
elif isinstance(document, list):
pass
else:
return document, metadata
newctx = None
if isinstance(document, dict):
# Handle $base, $profile, $namespaces, $schemas and $graph
if "$base" in document:
base_url = document["$base"]
if "$profile" in document:
if not newctx:
newctx = SubLoader(self)
prof = self.fetch(document["$profile"])
newctx.add_namespaces(document.get("$namespaces", {}), document["$profile"])
newctx.add_schemas(document.get("$schemas", []), document["$profile"])
if "$namespaces" in document:
if not newctx:
newctx = SubLoader(self)
newctx.add_namespaces(document["$namespaces"])
if "$schemas" in document:
if not newctx:
newctx = SubLoader(self)
newctx.add_schemas(document["$schemas"], file_base)
if newctx:
loader = newctx
if "$graph" in document:
metadata = {k: v for k,v in document.items() if k != "$graph"}
document = document["$graph"]
metadata, _ = loader.resolve_all(metadata, base_url, file_base)
if isinstance(document, dict):
for identifer in loader.identity_links:
if identifer in document:
if isinstance(document[identifer], basestring):
document[identifer] = loader.expand_url(document[identifer], base_url, scoped=True)
if document[identifer] not in loader.idx or isinstance(loader.idx[document[identifer]], basestring):
loader.idx[document[identifer]] = document
base_url = document[identifer]
elif isinstance(document[identifer], list):
for n, v in enumerate(document[identifer]):
document[identifer][n] = loader.expand_url(document[identifer][n], base_url, scoped=True)
if document[identifer][n] not in loader.idx:
loader.idx[document[identifer][n]] = document[identifer][n]
for d in document:
d2 = loader.expand_url(d, "", scoped=False, vocab_term=True)
if d != d2:
document[d2] = document[d]
del document[d]
for d in loader.url_fields:
if d in document:
if isinstance(document[d], basestring):
document[d] = loader.expand_url(document[d], base_url, scoped=False, vocab_term=(d in loader.vocab_fields))
elif isinstance(document[d], list):
document[d] = [loader.expand_url(url, base_url, scoped=False, vocab_term=(d in loader.vocab_fields)) if isinstance(url, basestring) else url for url in document[d] ]
try:
for key, val in document.items():
document[key], _ = loader.resolve_all(val, base_url, file_base)
except validate.ValidationException as v:
_logger.debug("loader is %s", id(loader))
raise validate.ValidationException("(%s) (%s) Validation error in field %s:\n%s" % (id(loader), file_base, key, validate.indent(str(v))))
elif isinstance(document, list):
i = 0
try:
while i < len(document):
val = document[i]
if isinstance(val, dict) and "$import" in val:
l, _ = loader.resolve_ref(val, file_base)
if isinstance(l, list):
del document[i]
for item in aslist(l):
document.insert(i, item)
i += 1
else:
document[i] = l
i += 1
else:
document[i], _ = loader.resolve_all(val, base_url, file_base)
i += 1
except validate.ValidationException as v:
raise validate.ValidationException("(%s) (%s) Validation error in position %i:\n%s" % (id(loader), file_base, i, validate.indent(str(v))))
return document, metadata
def __init__(self, toolpath_object, docpath):
self.impl = toolpath_object["impl"]
try:
self.embedded_tool = makeTool(from_url(os.path.join(docpath, self.impl)), docpath)
except validate.ValidationException as v:
raise WorkflowException("Tool definition %s failed validation:\n%s" % (os.path.join(docpath, self.impl), validate.indent(str(v))))
if "id" in toolpath_object:
self.id = toolpath_object["id"]
else:
self.id = "#step_" + str(random.randint(1, 1000000000))
for i in toolpath_object["inputs"]:
d = i["def"][len(self.impl):]
toolid = i.get("id", self.id + "." + idk(d))
found = False
for a in self.embedded_tool.tool["inputs"]:
if a["id"] == d:
i.update(a)
found = True
if not found:
raise WorkflowException("Did not find input '%s' in external process" % (i["def"]))
i["id"] = toolid
for i in toolpath_object["outputs"]:
d = i["def"][len(self.impl):]
toolid = i["id"]
found = False
for a in self.embedded_tool.tool["outputs"]:
if a["id"] == d:
i.update(a)
found = True
if not found:
raise WorkflowException("Did not find output '%s' in external process" % (i["def"]))
i["id"] = toolid
super(External, self).__init__(toolpath_object, "External", docpath)