def addLocation(d):
if "location" not in d:
if d["class"] == "File" and ("contents" not in d):
raise validate.ValidationException("Anonymous file object must have 'contents' and 'basename' fields.")
if d["class"] == "Directory" and ("listing" not in d or "basename" not in d):
raise validate.ValidationException(
"Anonymous directory object must have 'listing' and 'basename' fields.")
d["location"] = "_:" + Text(uuid.uuid4())
if "basename" not in d:
d["basename"] = d["location"][2:]
parse = urllib.parse.urlparse(d["location"])
path = parse.path
# strip trailing slash
if path.endswith("/"):
if d["class"] != "Directory":
raise validate.ValidationException(
"location '%s' ends with '/' but is not a Directory" % d["location"])
path = path.rstrip("/")
d["location"] = urllib.parse.urlunparse((parse.scheme, parse.netloc, path, parse.params, parse.query, parse.fragment))
if not d.get("basename"):
if path.startswith("_:"):
d["basename"] = Text(path[2:])
else:
d["basename"] = Text(os.path.basename(urllib.request.url2pathname(path)))
if d["class"] == "File":
nr, ne = os.path.splitext(d["basename"])
if d.get("nameroot") != nr:
d["nameroot"] = Text(nr)
if d.get("nameext") != ne:
d["nameext"] = Text(ne)
def get_from_requirements(
self,
r, # type: Dict[Text, Text]
req, # type: bool
pull_image, # type: bool
force_pull=False, # type: bool
tmp_outdir_prefix=DEFAULT_TMP_PREFIX # type: Text
): # type: (...) -> Optional[Text]
if r:
errmsg = None
try:
subprocess.check_output(["docker", "version"])
except subprocess.CalledProcessError as err:
errmsg = "Cannot communicate with docker daemon: " + Text(err)
except OSError as err:
errmsg = "'docker' executable not found: " + Text(err)
if errmsg:
if req:
raise WorkflowException(errmsg)
else:
return None
if self.get_image(r, pull_image, force_pull, tmp_outdir_prefix):
return r["dockerImageId"]
if req:
raise WorkflowException(u"Docker image %s not found" %
r["dockerImageId"])
return None
def check_adjust(builder, file_o):
# type: (Builder, Dict[Text, Any]) -> Dict[Text, Any]
"""
Map files to assigned path inside a container.
We need to also explicitly walk over input, as implicit reassignment
doesn't reach everything in builder.bindings
"""
if not builder.pathmapper:
raise ValueError(
"Do not call check_adjust using a builder that doesn't have a pathmapper."
)
file_o["path"] = docker_windows_path_adjust(
builder.pathmapper.mapper(file_o["location"])[1])
dn, bn = os.path.split(file_o["path"])
if file_o.get("dirname") != dn:
file_o["dirname"] = Text(dn)
if file_o.get("basename") != bn:
file_o["basename"] = Text(bn)
if file_o["class"] == "File":
nr, ne = os.path.splitext(file_o["basename"])
if file_o.get("nameroot") != nr:
file_o["nameroot"] = Text(nr)
if file_o.get("nameext") != ne:
file_o["nameext"] = Text(ne)
if not ACCEPTLIST_RE.match(file_o["basename"]):
raise WorkflowException(
"Invalid filename: '{}' contains illegal characters".format(
file_o["basename"]))
return file_o
def do_output_callback(self, final_output_callback):
# type: (Callable[[Any, Any], Any]) -> None
supportsMultipleInput = bool(self.workflow.get_requirement("MultipleInputFeatureRequirement")[0])
wo = None # type: Optional[Dict[Text, Text]]
try:
wo = object_from_state(
self.state, self.tool["outputs"], True, supportsMultipleInput,
"outputSource", incomplete=True)
except WorkflowException as err:
_logger.error(
u"[%s] Cannot collect workflow output: %s", self.name, Text(err))
self.processStatus = "permanentFail"
if self.prov_obj and self.parent_wf \
and self.prov_obj.workflow_run_uri != self.parent_wf.workflow_run_uri:
process_run_id = None
self.prov_obj.generate_output_prov(wo or {}, process_run_id, self.name)
self.prov_obj.document.wasEndedBy(
self.prov_obj.workflow_run_uri, None, self.prov_obj.engine_uuid,
datetime.datetime.now())
prov_ids = self.prov_obj.finalize_prov_profile(self.name)
# Tell parent to associate our provenance files with our wf run
self.parent_wf.activity_has_provenance(self.prov_obj.workflow_run_uri, prov_ids)
_logger.info(u"[%s] completed %s", self.name, self.processStatus)
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"[%s] %s", self.name, json_dumps(wo, indent=4))
self.did_callback = True
final_output_callback(wo, self.processStatus)
def job(
self,
job_order, # type: Mapping[Text, Text]
output_callbacks, # type: Callable[[Any, Any], Any]
runtimeContext, # type: RuntimeContext
): # type: (...) -> Generator[Union[ExpressionTool.ExpressionJob, JobBase, CallbackJob], None, None]
#initialize sub-workflow as a step in the parent profile
if self.embedded_tool.tool["class"] == "Workflow" \
and runtimeContext.research_obj and self.prov_obj \
and self.embedded_tool.provenance_object:
self.embedded_tool.parent_wf = self.prov_obj
process_name = self.tool["id"].split("#")[1]
self.prov_obj.start_process(
process_name, datetime.datetime.now(),
self.embedded_tool.provenance_object.workflow_run_uri)
step_input = {}
for inp in self.tool["inputs"]:
field = shortname(inp["id"])
if not inp.get("not_connected"):
step_input[field] = job_order[inp["id"]]
try:
for tool in self.embedded_tool.job(
step_input,
functools.partial(self.receive_output, output_callbacks),
runtimeContext):
yield tool
except WorkflowException:
_logger.error(u"Exception on step '%s'", runtimeContext.name)
raise
except Exception as exc:
_logger.exception("Unexpected exception")
raise_from(WorkflowException(Text(exc)), exc)
def parallel_steps(steps, rc, runtimeContext):
# type: (List[Optional[Generator[Union[ExpressionTool.ExpressionJob, JobBase, CallbackJob, None], None, None]]], ReceiveScatterOutput, RuntimeContext) -> Generator[Union[ExpressionTool.ExpressionJob, JobBase, CallbackJob, None], None, None]
while rc.completed < rc.total:
made_progress = False
for index, step in enumerate(steps):
if getdefault(runtimeContext.on_error,
"stop") == "stop" and rc.processStatus != "success":
break
if step is None:
continue
try:
for j in step:
if getdefault(
runtimeContext.on_error, "stop"
) == "stop" and rc.processStatus != "success":
break
if j is not None:
made_progress = True
yield j
else:
break
if made_progress:
break
except WorkflowException as exc:
_logger.error(u"Cannot make scatter job: %s", Text(exc))
_logger.debug("", exc_info=True)
rc.receive_scatter_output(index, {}, "permanentFail")
if not made_progress and rc.completed < rc.total:
yield None
def fetch_document(
argsworkflow, # type: Union[Text, Dict[Text, Any]]
loadingContext=None # type: Optional[LoadingContext]
): # type: (...) -> Tuple[LoadingContext, CommentedMap, Text]
"""Retrieve a CWL document."""
if loadingContext is None:
loadingContext = LoadingContext()
loadingContext.loader = default_loader()
else:
loadingContext = loadingContext.copy()
if loadingContext.loader is None:
loadingContext.loader = default_loader(
loadingContext.fetcher_constructor)
uri = None # type: Optional[Text]
workflowobj = None # type: Optional[CommentedMap]
if isinstance(argsworkflow, string_types):
uri, fileuri = resolve_tool_uri(argsworkflow,
resolver=loadingContext.resolver,
document_loader=loadingContext.loader)
workflowobj = loadingContext.loader.fetch(fileuri)
elif isinstance(argsworkflow, dict):
uri = argsworkflow["id"] if argsworkflow.get("id") else "_:" + Text(
uuid.uuid4())
workflowobj = cast(CommentedMap, cmap(argsworkflow, fn=uri))
loadingContext.loader.idx[uri] = workflowobj
else:
raise ValidationException("Must be URI or object: '%s'" % argsworkflow)
assert workflowobj is not None
return loadingContext, workflowobj, uri
def job(
self,
job_order, # type: Mapping[Text, Any]
output_callbacks, # type: Callable[[Any, Any], Any]
runtimeContext # type: RuntimeContext
): # type: (...) -> Generator[Any, None, None]
# FIXME: Declare base type for what Generator yields
try:
for tool in self.procgenerator.embedded_tool.job(
job_order, self.receive_output, runtimeContext):
yield tool
while self.processStatus is None:
yield None
if self.processStatus != "success":
output_callbacks(self.jobout, self.processStatus)
return
if self.jobout is None:
raise WorkflowException("jobout should not be None")
created_tool, runinputs = self.procgenerator.result(
job_order, self.jobout, runtimeContext)
for tool in created_tool.job(runinputs, output_callbacks,
runtimeContext):
yield tool
except WorkflowException:
raise
except Exception as exc:
_logger.exception("Unexpected exception")
raise WorkflowException(Text(exc))
def collect_output_ports(self,
ports, # type: Set[Dict[Text, Any]]
builder, # type: Builder
outdir, # type: Text
rcode, # type: int
compute_checksum=True, # type: bool
jobname="", # type: Text
readers=None # type: Dict[Text, Any]
): # type: (...) -> OutputPorts
ret = {} # type: OutputPorts
debug = _logger.isEnabledFor(logging.DEBUG)
cwl_version = self.metadata.get(
"http://commonwl.org/cwltool#original_cwlVersion", None)
if cwl_version != "v1.0":
builder.resources["exitCode"] = rcode
try:
fs_access = builder.make_fs_access(outdir)
custom_output = fs_access.join(outdir, "cwl.output.json")
if fs_access.exists(custom_output):
with fs_access.open(custom_output, "r") as f:
ret = json.load(f)
if debug:
_logger.debug(u"Raw output from %s: %s", custom_output,
json_dumps(ret, indent=4))
else:
for i, port in enumerate(ports):
def makeWorkflowException(msg):
return WorkflowException(
u"Error collecting output for parameter '%s':\n%s"
% (shortname(port["id"]), msg))
with SourceLine(ports, i, makeWorkflowException, debug):
fragment = shortname(port["id"])
ret[fragment] = self.collect_output(port, builder, outdir, fs_access,
compute_checksum=compute_checksum)
if ret:
revmap = partial(revmap_file, builder, outdir)
adjustDirObjs(ret, trim_listing)
visit_class(ret, ("File", "Directory"), cast(Callable[[Any], Any], revmap))
visit_class(ret, ("File", "Directory"), remove_path)
normalizeFilesDirs(ret)
visit_class(ret, ("File", "Directory"), partial(check_valid_locations, fs_access))
if compute_checksum:
adjustFileObjs(ret, partial(compute_checksums, fs_access))
expected_schema = cast(Schema, self.names.get_name(
"outputs_record_schema", ""))
validate.validate_ex(expected_schema, ret,
strict=False, logger=_logger_validation_warnings)
if ret is not None and builder.mutation_manager is not None:
adjustFileObjs(ret, builder.mutation_manager.set_generation)
return ret if ret is not None else {}
except validate.ValidationException as e:
raise WorkflowException(
"Error validating output record. " + Text(e) + "\n in "
+ json_dumps(ret, indent=4))
finally:
if builder.mutation_manager and readers:
for r in readers.values():
builder.mutation_manager.release_reader(jobname, r)
def test_cid_file_dir_arg_is_file_instead_of_dir(tmpdir):
test_file = "cache_test_workflow.cwl"
bad_cidfile_dir = Text(tmpdir.ensure("cidfile-dir-actually-a-file"))
error_code, _, stderr = get_main_output(
["--cidfile-dir", bad_cidfile_dir,
get_data("tests/wf/" + test_file)])
assert "is not a directory, please check it first" in stderr, stderr
assert error_code == 2
tmpdir.remove(ignore_errors=True)
def run_jobs(self,
process, # type: Process
job_order_object, # type: Dict[Text, Any]
logger, # type: logging.Logger
runtime_context # type: RuntimeContext
): # type: (...) -> None
process_run_id = None # type: Optional[str]
# define provenance profile for single commandline tool
if not isinstance(process, Workflow) \
and runtime_context.research_obj is not None:
process.provenance_object = ProvenanceProfile(
runtime_context.research_obj,
full_name=runtime_context.cwl_full_name,
host_provenance=False,
user_provenance=False,
orcid=runtime_context.orcid,
# single tool execution, so RO UUID = wf UUID = tool UUID
run_uuid=runtime_context.research_obj.ro_uuid,
fsaccess=runtime_context.make_fs_access(''))
process.parent_wf = process.provenance_object
jobiter = process.job(job_order_object, self.output_callback,
runtime_context)
try:
for job in jobiter:
if job is not None:
if runtime_context.builder is not None:
job.builder = runtime_context.builder
if job.outdir is not None:
self.output_dirs.add(job.outdir)
if runtime_context.research_obj is not None:
if not isinstance(process, Workflow):
prov_obj = process.provenance_object
else:
prov_obj = job.prov_obj
if prov_obj:
runtime_context.prov_obj = prov_obj
prov_obj.fsaccess = runtime_context.make_fs_access('')
prov_obj.evaluate(
process, job, job_order_object,
runtime_context.research_obj)
process_run_id =\
prov_obj.record_process_start(process, job)
runtime_context = runtime_context.copy()
runtime_context.process_run_id = process_run_id
job.run(runtime_context)
else:
logger.error("Workflow cannot make any more progress.")
break
except (ValidationException, WorkflowException): # pylint: disable=try-except-raise
raise
except Exception as err:
logger.exception("Got workflow error")
raise_from(WorkflowException(Text(err)), err)
def add(self, value):
# type: (Text) -> Text
if not isinstance(value, string_types):
raise Exception("Secret store only accepts strings")
if value not in self.secrets:
placeholder = "(secret-%s)" % Text(uuid.uuid4())
self.secrets[placeholder] = value
return placeholder
return value
def test_cid_file_non_existing_dir(tmpdir):
test_file = "cache_test_workflow.cwl"
bad_cidfile_dir = Text(tmpdir.join("cidfile-dir-badpath"))
error_code, _, stderr = get_main_output([
'--record-container-id', "--cidfile-dir", bad_cidfile_dir,
get_data("tests/wf/" + test_file)
])
assert "directory doesn't exist, please create it first" in stderr, stderr
assert error_code == 2
tmpdir.remove(ignore_errors=True)
def abspath(src, basedir): # type: (Text, Text) -> Text
if src.startswith(u"file://"):
abpath = Text(uri_file_path(str(src)))
elif urllib.parse.urlsplit(src).scheme in ['http', 'https']:
return src
else:
if basedir.startswith(u"file://"):
abpath = src if os.path.isabs(src) else basedir + '/' + src
else:
abpath = src if os.path.isabs(src) else os.path.join(basedir, src)
return abpath
def _convert_stdstreams_to_files(workflowobj):
# type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None
if isinstance(workflowobj, MutableMapping):
if workflowobj.get('class') == 'CommandLineTool':
with SourceLine(workflowobj, "outputs", ValidationException,
_logger.isEnabledFor(logging.DEBUG)):
outputs = workflowobj.get('outputs', [])
if not isinstance(outputs, CommentedSeq):
raise ValidationException('"outputs" section is not '
'valid.')
for out in workflowobj.get('outputs', []):
if not isinstance(out, CommentedMap):
raise ValidationException(
"Output '{}' is not a valid "
"OutputParameter.".format(out))
for streamtype in ['stdout', 'stderr']:
if out.get('type') == streamtype:
if 'outputBinding' in out:
raise ValidationException(
"Not allowed to specify outputBinding when"
" using %s shortcut." % streamtype)
if streamtype in workflowobj:
filename = workflowobj[streamtype]
else:
filename = Text(
hashlib.sha1(
json_dumps(workflowobj,
sort_keys=True).encode(
'utf-8')).hexdigest())
workflowobj[streamtype] = filename
out['type'] = 'File'
out['outputBinding'] = cmap({'glob': filename})
for inp in workflowobj.get('inputs', []):
if inp.get('type') == 'stdin':
if 'inputBinding' in inp:
raise ValidationException(
"Not allowed to specify inputBinding when"
" using stdin shortcut.")
if 'stdin' in workflowobj:
raise ValidationException(
"Not allowed to specify stdin path when"
" using stdin type shortcut.")
else:
workflowobj['stdin'] = \
"$(inputs.%s.path)" % \
inp['id'].rpartition('#')[2]
inp['type'] = 'File'
else:
for entry in itervalues(workflowobj):
_convert_stdstreams_to_files(entry)
if isinstance(workflowobj, MutableSequence):
for entry in workflowobj:
_convert_stdstreams_to_files(entry)
def run_jobs(
self,
process, # type: Process
job_order_object, # type: Dict[Text, Any]
logger,
runtime_context # type: RuntimeContext
): # type: (...) -> None
process_run_id = None # type: Optional[str]
reference_locations = {} # type: Dict[Text,Text]
# define provenance profile for single commandline tool
if not isinstance(process, Workflow) \
and runtime_context.research_obj is not None:
orcid = runtime_context.orcid
full_name = runtime_context.cwl_full_name
process.provenance_object = CreateProvProfile(
runtime_context.research_obj, orcid, full_name)
process.parent_wf = process.provenance_object
jobiter = process.job(job_order_object, self.output_callback,
runtime_context)
try:
for job in jobiter:
if job:
if runtime_context.builder is not None:
job.builder = runtime_context.builder
if job.outdir:
self.output_dirs.add(job.outdir)
if runtime_context.research_obj is not None:
if not isinstance(process, Workflow):
runtime_context.prov_obj = process.provenance_object
else:
runtime_context.prov_obj = job.prov_obj
assert runtime_context.prov_obj
process_run_id, reference_locations = \
runtime_context.prov_obj.evaluate(
process, job, job_order_object,
runtime_context.make_fs_access,
runtime_context)
runtime_context = runtime_context.copy()
runtime_context.process_run_id = process_run_id
runtime_context.reference_locations = \
reference_locations
job.run(runtime_context)
else:
logger.error("Workflow cannot make any more progress.")
break
except (ValidationException, WorkflowException): # pylint: disable=try-except-raise
raise
except Exception as err:
logger.exception("Got workflow error")
raise WorkflowException(Text(err))
def do_eval(
ex, # type: Union[Text, Dict[Text, Text]]
jobinput, # type: Dict[Text, JSON]
requirements, # type: List[Dict[Text, Any]]
outdir, # type: Optional[Text]
tmpdir, # type: Optional[Text]
resources, # type: Dict[str, int]
context=None, # type: Any
timeout=default_timeout, # type: float
force_docker_pull=False, # type: bool
debug=False, # type: bool
js_console=False, # type: bool
strip_whitespace=True # type: bool
): # type: (...) -> Any
runtime = copy.deepcopy(resources) # type: Dict[str, Any]
runtime["tmpdir"] = docker_windows_path_adjust(tmpdir) if tmpdir else None
runtime["outdir"] = docker_windows_path_adjust(outdir) if outdir else None
rootvars = {u"inputs": jobinput, u"self": context, u"runtime": runtime}
# TODO: need to make sure the `rootvars dict`
# contains no bytes type in the first place.
if six.PY3:
rootvars = bytes2str_in_dicts(rootvars) # type: ignore
if isinstance(ex, string_types) and needs_parsing(ex):
fullJS = False
jslib = u""
for r in reversed(requirements):
if r["class"] == "InlineJavascriptRequirement":
fullJS = True
jslib = jshead(r.get("expressionLib", []), rootvars)
break
try:
return interpolate(ex,
rootvars,
timeout=timeout,
fullJS=fullJS,
jslib=jslib,
force_docker_pull=force_docker_pull,
debug=debug,
js_console=js_console,
strip_whitespace=strip_whitespace)
except Exception as e:
raise_from(
WorkflowException("Expression evaluation error:\n%s" %
Text(e)), e)
else:
return ex
def _add_blank_ids(workflowobj):
# type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None
if isinstance(workflowobj, MutableMapping):
if ("run" in workflowobj
and isinstance(workflowobj["run"], MutableMapping)
and "id" not in workflowobj["run"]
and "$import" not in workflowobj["run"]):
workflowobj["run"]["id"] = Text(uuid.uuid4())
for entry in itervalues(workflowobj):
_add_blank_ids(entry)
if isinstance(workflowobj, MutableSequence):
for entry in workflowobj:
_add_blank_ids(entry)
def add(self, value): # type: (Text) -> Text
"""
Add the given value to the store.
Returns a placeholder value to use until the real value is needed.
"""
if not isinstance(value, string_types):
raise Exception("Secret store only accepts strings")
if value not in self.secrets:
placeholder = "(secret-%s)" % Text(uuid.uuid4())
self.secrets[placeholder] = value
return placeholder
return value
def run(
self,
runtimeContext, # type: RuntimeContext
tmpdir_lock=None # type: Optional[threading.Lock]
): # type: (...) -> None
try:
normalizeFilesDirs(self.builder.job)
ev = self.builder.do_eval(self.script)
normalizeFilesDirs(ev)
self.output_callback(ev, "success")
except Exception as err:
_logger.warning(u"Failed to evaluate expression:\n%s",
Text(err),
exc_info=runtimeContext.debug)
self.output_callback({}, "permanentFail")
def validate_hints(self, avsc_names, hints, strict):
# type: (Any, List[Dict[Text, Any]], bool) -> None
for i, r in enumerate(hints):
sl = SourceLine(hints, i, validate.ValidationException)
with sl:
if avsc_names.get_name(r["class"], "") is not None and self.doc_loader is not None:
plain_hint = dict((key, r[key]) for key in r if key not in
self.doc_loader.identifiers) # strip identifiers
validate.validate_ex(
avsc_names.get_name(plain_hint["class"], ""),
plain_hint, strict=strict)
elif r["class"] in ("NetworkAccess", "LoadListingRequirement"):
pass
else:
_logger.info(Text(sl.makeError(u"Unknown hint %s" % (r["class"]))))
def tostr(self, value): # type: (Any) -> Text
if isinstance(value, MutableMapping) and value.get("class") in ("File", "Directory"):
if "path" not in value:
raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value))
# Path adjust for windows file path when passing to docker, docker accepts unix like path only
(docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
if onWindows() and docker_req is not None:
# docker_req is none only when there is no dockerRequirement
# mentioned in hints and Requirement
path = docker_windows_path_adjust(value["path"])
return path
return value["path"]
else:
return Text(value)
def check_working_directories(runtimeContext # type: RuntimeContext
): # type: (...) -> Optional[int]
for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"):
if getattr(runtimeContext, dirprefix) and getattr(runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX:
sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \
else ""
setattr(runtimeContext, dirprefix,
os.path.abspath(getattr(runtimeContext, dirprefix)) + sl)
if not os.path.exists(os.path.dirname(getattr(runtimeContext, dirprefix))):
try:
os.makedirs(os.path.dirname(getattr(runtimeContext, dirprefix)))
except Exception as e:
_logger.error("Failed to create directory: %s", Text(e))
return 1
return None
def runner():
""" Job running thread. """
try:
job.run(runtime_context)
except WorkflowException as err:
_logger.exception("Got workflow error")
self.exceptions.append(err)
except Exception as err: # pylint: disable=broad-except
_logger.exception("Got workflow error")
self.exceptions.append(WorkflowException(Text(err)))
finally:
with runtime_context.workflow_eval_lock:
self.threads.remove(threading.current_thread())
if isinstance(job, JobBase):
self.allocated_ram -= job.builder.resources["ram"]
self.allocated_cores -= job.builder.resources["cores"]
runtime_context.workflow_eval_lock.notifyAll()
def avroize_type(field_type, name_prefix=""):
# type: (Union[List[Dict[Text, Any]], Dict[Text, Any]], Text) -> Any
"""
adds missing information to a type so that CWL types are valid in schema_salad.
"""
if isinstance(field_type, MutableSequence):
for f in field_type:
avroize_type(f, name_prefix)
elif isinstance(field_type, MutableMapping):
if field_type["type"] in ("enum", "record"):
if "name" not in field_type:
field_type["name"] = name_prefix + Text(uuid.uuid4())
if field_type["type"] == "record":
avroize_type(field_type["fields"], name_prefix)
if field_type["type"] == "array":
avroize_type(field_type["items"], name_prefix)
return field_type
def _updateDev2Script(ent): # type: (Any) -> Any
if isinstance(ent, dict) and "engine" in ent:
if ent["engine"] == "https://w3id.org/cwl/cwl#JsonPointer":
sp = ent["script"].split("/")
if sp[0] in ("tmpdir", "outdir"):
return u"$(runtime.%s)" % sp[0]
if not sp[0]:
sp.pop(0)
front = sp.pop(0)
sp = [Text(i) if digits.match(i) else "'" + i + "'" for i in sp]
if front == "job":
return u"$(inputs[%s])" % ']['.join(sp)
if front == "context":
return u"$(self[%s])" % ']['.join(sp)
else:
sc = updateScript(ent["script"])
if sc[0] == "{":
return "$" + sc
return u"$(%s)" % sc
return ent
def _runner(self, job, runtime_context, TMPDIR_LOCK):
# type: (Union[JobBase, WorkflowJob, CallbackJob], RuntimeContext, threading.Lock) -> None
"""Job running thread."""
try:
_logger.debug("job: {}, runtime_context: {}, TMPDIR_LOCK: {}".format(job, runtime_context, TMPDIR_LOCK))
job.run(runtime_context, TMPDIR_LOCK)
except WorkflowException as err:
_logger.exception("Got workflow error")
self.exceptions.append(err)
except Exception as err: # pylint: disable=broad-except
_logger.exception("Got workflow error")
self.exceptions.append(WorkflowException(Text(err)))
finally:
if runtime_context.workflow_eval_lock:
with runtime_context.workflow_eval_lock:
self.threads.remove(threading.current_thread())
if isinstance(job, JobBase):
self.allocated_ram -= job.builder.resources["ram"]
self.allocated_cores -= job.builder.resources["cores"]
runtime_context.workflow_eval_lock.notifyAll()
def generate_example_input(inptype):
# type: (Union[Text, Dict[Text, Any]]) -> Any
defaults = {u'null': 'null',
u'Any': 'null',
u'boolean': False,
u'int': 0,
u'long': 0,
u'float': 0.1,
u'double': 0.1,
u'string': 'default_string',
u'File': {'class': 'File',
'path': 'default/file/path'},
u'Directory': {'class': 'Directory',
'path': 'default/directory/path'}
} # type: Dict[Text, Any]
if (not isinstance(inptype, string_types) and
not isinstance(inptype, collections.Mapping)
and isinstance(inptype, collections.MutableSet)):
if len(inptype) == 2 and 'null' in inptype:
inptype.remove('null')
return generate_example_input(inptype[0])
# TODO: indicate that this input is optional
raise Exception("multi-types other than optional not yet supported"
" for generating example input objects: "
"{}".format(inptype))
if isinstance(inptype, collections.Mapping) and 'type' in inptype:
if inptype['type'] == 'array':
return [generate_example_input(inptype['items'])]
if inptype['type'] == 'enum':
return 'valid_enum_value'
# TODO: list valid values in a comment
if inptype['type'] == 'record':
record = {}
for field in inptype['fields']:
record[shortname(field['name'])] = generate_example_input(
field['type'])
return record
elif isinstance(inptype, string_types):
return defaults.get(Text(inptype), 'custom_type')
# TODO: support custom types, complex arrays
return None
def fetch_document(argsworkflow, # type: Union[Text, Dict[Text, Any]]
resolver=None, # type: Callable[[Loader, Union[Text, Dict[Text, Any]]], Text]
fetcher_constructor=None # type: FetcherConstructorType
): # type: (...) -> Tuple[Loader, CommentedMap, Text]
"""Retrieve a CWL document."""
document_loader = default_loader(fetcher_constructor) # type: ignore
uri = None # type: Optional[Text]
workflowobj = None # type: Optional[CommentedMap]
if isinstance(argsworkflow, string_types):
uri, fileuri = resolve_tool_uri(argsworkflow, resolver=resolver,
document_loader=document_loader)
workflowobj = document_loader.fetch(fileuri)
elif isinstance(argsworkflow, MutableMapping):
uri = "#" + Text(id(argsworkflow))
workflowobj = cast(CommentedMap, cmap(argsworkflow, fn=uri))
else:
raise ValidationException("Must be URI or object: '%s'" % argsworkflow)
assert workflowobj is not None
return document_loader, workflowobj, uri
def generate_example_input(inptype, # type: Any
default # type: Optional[Any]
): # type: (...) -> Tuple[Any, Text]
"""Converts a single input schema into an example."""
example = None
comment = u""
defaults = {u'null': 'null',
u'Any': 'null',
u'boolean': False,
u'int': 0,
u'long': 0,
u'float': 0.1,
u'double': 0.1,
u'string': 'a_string',
u'File': yaml.comments.CommentedMap([
('class', 'File'), ('path', 'a/file/path')]),
u'Directory': yaml.comments.CommentedMap([
('class', 'Directory'), ('path', 'a/directory/path')])
} # type: Dict[Text, Any]
if isinstance(inptype, collections.MutableSequence):
optional = False
if 'null' in inptype:
inptype.remove('null')
optional = True
if len(inptype) == 1:
example, comment = generate_example_input(inptype[0], default)
if optional:
if comment:
comment = u"{} (optional)".format(comment)
else:
comment = u"optional"
else:
example = yaml.comments.CommentedSeq()
for index, entry in enumerate(inptype):
value, e_comment = generate_example_input(entry, default)
example.append(value)
example.yaml_add_eol_comment(e_comment, index)
if optional:
comment = u"optional"
elif isinstance(inptype, collections.Mapping) and 'type' in inptype:
if inptype['type'] == 'array':
if len(inptype['items']) == 1 and 'type' in inptype['items'][0] \
and inptype['items'][0]['type'] == 'enum':
# array of just an enum then list all the options
example = inptype['items'][0]['symbols']
if 'name' in inptype['items'][0]:
comment = u'array of type "{}".'.format(inptype['items'][0]['name'])
else:
value, comment = generate_example_input(inptype['items'], None)
comment = u"array of " + comment
if len(inptype['items']) == 1:
example = [value]
else:
example = value
if default is not None:
example = default
elif inptype['type'] == 'enum':
if default is not None:
example = default
elif 'default' in inptype:
example = inptype['default']
elif len(inptype['symbols']) == 1:
example = inptype['symbols'][0]
else:
example = '{}_enum_value'.format(inptype.get('name', 'valid'))
comment = u'enum; valid values: "{}"'.format(
'", "'.join(inptype['symbols']))
elif inptype['type'] == 'record':
example = yaml.comments.CommentedMap()
if 'name' in inptype:
comment = u'"{}" record type.'.format(inptype['name'])
for field in inptype['fields']:
value, f_comment = generate_example_input(field['type'], None)
example.insert(0, shortname(field['name']), value, f_comment)
elif 'default' in inptype:
example = inptype['default']
comment = u'default value of type "{}".'.format(inptype['type'])
else:
example = defaults.get(inptype['type'], Text(inptype))
comment = u'type "{}".'.format(inptype['type'])
else:
if not default:
example = defaults.get(Text(inptype), Text(inptype))
comment = u'type "{}"'.format(inptype)
else:
example = default
comment = u'default value of type "{}".'.format(inptype)
return example, comment
