def collect_output_ports(self, ports, builder, outdir):
try:
ret = {}
custom_output = os.path.join(outdir, "cwl.output.json")
if builder.fs_access.exists(custom_output):
with builder.fs_access.open(custom_output, "r") as f:
ret = yaml.load(f)
_logger.debug("Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
adjustFileObjs(ret, remove_hostfs)
adjustFileObjs(ret, functools.partial(revmap_file, builder, outdir))
adjustFileObjs(ret, remove_hostfs)
validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret)
return ret
for port in ports:
fragment = shortname(port["id"])
try:
ret[fragment] = self.collect_output(port, builder, outdir)
except Exception as e:
raise WorkflowException("Error collecting output for parameter '%s': %s" % (shortname(port["id"]), e))
if ret:
adjustFileObjs(ret, remove_hostfs)
validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret)
return ret if ret is not None else {}
except validate.ValidationException as e:
raise WorkflowException("Error validating output record, " + str(e) + "\n in " + json.dumps(ret, indent=4))
def object_from_state(state, parms, frag_only, supportsMultipleInput):
inputobj = {}
for inp in parms:
iid = inp["id"]
if frag_only:
iid = shortname(iid)
if "source" in inp:
if isinstance(inp["source"], list) and not supportsMultipleInput:
raise WorkflowException("Workflow contains multiple inbound links to a single parameter but MultipleInputFeatureRequirement is not declared.")
connections = aslist(inp["source"])
for src in connections:
if src in state and state[src] is not None:
if not match_types(inp["type"], state[src], iid, inputobj,
inp.get("linkMerge", ("merge_nested" if len(connections) > 1 else None)),
valueFrom=inp.get("valueFrom")):
raise WorkflowException("Type mismatch between source '%s' (%s) and sink '%s' (%s)" % (src, state[src].parameter["type"], inp["id"], inp["type"]))
elif src not in state:
raise WorkflowException("Connect source '%s' on parameter '%s' does not exist" % (src, inp["id"]))
else:
return None
elif "default" in inp:
inputobj[iid] = inp["default"]
elif "valueFrom" in inp:
inputobj[iid] = None
else:
raise WorkflowException("Value for %s not specified" % (inp["id"]))
return inputobj
def exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image):
if ex["engine"] == "https://w3id.org/cwl/cwl#JsonPointer":
try:
obj = {"job": jobinput, "context": context, "outdir": outdir, "tmpdir": tmpdir}
return schema_salad.ref_resolver.resolve_json_pointer(obj, ex["script"])
except ValueError as v:
raise WorkflowException("%s in %s" % (v, obj))
if ex["engine"] == "https://w3id.org/cwl/cwl#JavascriptEngine":
engineConfig = []
for r in reversed(requirements):
if r["class"] == "ExpressionEngineRequirement" and r["id"] == "https://w3id.org/cwl/cwl#JavascriptEngine":
engineConfig = r.get("engineConfig", [])
break
return sandboxjs.execjs(ex["script"], jshead(engineConfig, jobinput, context, tmpdir, outdir))
for r in reversed(requirements):
if r["class"] == "ExpressionEngineRequirement" and r["id"] == ex["engine"]:
runtime = []
class DR(object):
pass
dr = DR()
dr.requirements = r.get("requirements", [])
dr.hints = r.get("hints", [])
(docker_req, docker_is_req) = process.get_feature(dr, "DockerRequirement")
img_id = None
if docker_req:
img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image)
if img_id:
runtime = ["docker", "run", "-i", "--rm", img_id]
inp = {
"script": ex["script"],
"engineConfig": r.get("engineConfig", []),
"job": jobinput,
"context": context,
"outdir": outdir,
"tmpdir": tmpdir,
}
_logger.debug("Invoking expression engine %s with %s",
runtime + aslist(r["engineCommand"]),
json.dumps(inp, indent=4))
sp = subprocess.Popen(runtime + aslist(r["engineCommand"]),
shell=False,
close_fds=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
(stdoutdata, stderrdata) = sp.communicate(json.dumps(inp) + "\n\n")
if sp.returncode != 0:
raise WorkflowException("Expression engine returned non-zero exit code on evaluation of\n%s" % json.dumps(inp, indent=4))
return json.loads(stdoutdata)
raise WorkflowException("Unknown expression engine '%s'" % ex["engine"])
def _init_job(self, joborder, input_basedir, **kwargs):
builder = Builder()
builder.job = copy.deepcopy(joborder)
for i in self.tool["inputs"]:
d = shortname(i["id"])
if d not in builder.job and "default" in i:
builder.job[d] = i["default"]
for r in self.requirements:
if r["class"] not in supportedProcessRequirements:
raise WorkflowException("Unsupported process requirement %s" %
(r["class"]))
# Validate job order
try:
validate.validate_ex(
self.names.get_name("input_record_schema", ""), builder.job)
except validate.ValidationException as e:
raise WorkflowException("Error validating input record, " + str(e))
builder.files = []
builder.bindings = []
builder.schemaDefs = self.schemaDefs
builder.names = self.names
builder.requirements = self.requirements
builder.resources = {}
builder.timeout = kwargs.get("eval_timeout")
dockerReq, _ = self.get_requirement("DockerRequirement")
if dockerReq and kwargs.get("use_container"):
builder.outdir = kwargs.get("docker_outdir") or "/var/spool/cwl"
builder.tmpdir = kwargs.get("docker_tmpdir") or "/tmp"
else:
builder.outdir = kwargs.get("outdir") or tempfile.mkdtemp()
builder.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp()
builder.fs_access = kwargs.get("fs_access") or StdFsAccess(
input_basedir)
if self.formatgraph:
for i in self.tool["inputs"]:
d = shortname(i["id"])
if d in builder.job and i.get("format"):
checkFormat(builder.job[d], builder.do_eval(i["format"]),
self.requirements, self.formatgraph)
builder.bindings.extend(
builder.bind_input(self.inputs_record_schema, builder.job))
builder.resources = self.evalResources(builder, kwargs)
return builder
def defaultMakeTool(toolpath_object, **kwargs):
if not isinstance(toolpath_object, dict):
raise WorkflowException("Not a dict: `%s`" % toolpath_object)
if "class" in toolpath_object:
if toolpath_object["class"] == "CommandLineTool":
return draft2tool.CommandLineTool(toolpath_object, **kwargs)
elif toolpath_object["class"] == "ExpressionTool":
return draft2tool.ExpressionTool(toolpath_object, **kwargs)
elif toolpath_object["class"] == "Workflow":
return Workflow(toolpath_object, **kwargs)
raise WorkflowException("Missing or invalid 'class' field in %s, expecting one of: CommandLineTool, ExpressionTool, Workflow" % toolpath_object["id"])
def dotproduct_scatter(process, joborder, basedir, scatter_keys,
output_callback, **kwargs):
l = None
for s in scatter_keys:
if l is None:
l = len(joborder[s])
elif l != len(joborder[s]):
raise WorkflowException(
"Length of input arrays must be equal when performing dotproduct scatter."
)
output = {}
for i in process.tool["outputs"]:
output[i["id"]] = [None] * l
rc = ReceiveScatterOutput(output_callback, output)
for n in range(0, l):
jo = copy.copy(joborder)
for s in scatter_keys:
jo[s] = kwargs["valueFrom"](s, joborder[s][n])
for j in process.job(jo, basedir,
functools.partial(rc.receive_scatter_output, n),
**kwargs):
yield j
rc.setTotal(l)
def match_types(sinktype, src, iid, inputobj, linkMerge, valueFrom):
if isinstance(sinktype, list):
# Sink is union type
for st in sinktype:
if match_types(st, src, iid, inputobj, linkMerge, valueFrom):
return True
elif isinstance(src.parameter["type"], list):
# Source is union type
# Check that every source type is compatible with the sink.
for st in src.parameter["type"]:
srccopy = copy.deepcopy(src)
srccopy.parameter["type"] = st
if not match_types(st, srccopy, iid, inputobj, linkMerge, valueFrom):
return False
return True
elif linkMerge:
if iid not in inputobj:
inputobj[iid] = []
if linkMerge == "merge_nested":
inputobj[iid].append(src.value)
elif linkMerge == "merge_flattened":
if isinstance(src.value, list):
inputobj[iid].extend(src.value)
else:
inputobj[iid].append(src.value)
else:
raise WorkflowException("Unrecognized linkMerge enum '%s'" % linkMerge)
return True
elif valueFrom is not None or are_same_type(src.parameter["type"], sinktype) or sinktype == "Any":
# simply assign the value from state to input
inputobj[iid] = copy.deepcopy(src.value)
return True
return False
def makePathMapper(self, reffiles, input_basedir, **kwargs):
dockerReq, _ = self.get_requirement("DockerRequirement")
try:
if dockerReq and kwargs.get("use_container"):
return DockerPathMapper(reffiles, input_basedir)
else:
return PathMapper(reffiles, input_basedir)
except OSError as e:
if e.errno == errno.ENOENT:
raise WorkflowException("Missing input file %s" % e)
def initialize_socket(self, timeout):
self.sock = socket(AF_INET, SOCK_DGRAM)
self.sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self.sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
self.sock.settimeout(timeout)
try:
self.sock.bind(
("", 0)) # allow OS to assign next available source port
except Exception as err:
raise WorkflowException(
"WorkflowException: error {} while trying to open socket".
format(str(err)))
def job(self, joborder, basedir, output_callback, **kwargs):
for i in self.tool["inputs"]:
p = i["id"]
field = shortname(p)
joborder[field] = joborder[i["id"]]
del joborder[i["id"]]
kwargs["requirements"] = kwargs.get("requirements", []) + self.tool.get("requirements", [])
kwargs["hints"] = kwargs.get("hints", []) + self.tool.get("hints", [])
try:
for t in self.embedded_tool.job(joborder, basedir,
functools.partial(self.receive_output, output_callback),
**kwargs):
yield t
except WorkflowException:
_logger.error("Exception on step '%s'", kwargs.get("name"))
raise
except Exception as e:
_logger.exception("Unexpected exception")
raise WorkflowException(str(e))
def revmap_file(builder, outdir, f):
"""Remap a file back to original path. For Docker, this is outside the container.
Uses either files in the pathmapper or remaps internal output directories
to the external directory.
"""
if f.get("hostfs"):
return
revmap_f = builder.pathmapper.reversemap(f["path"])
if revmap_f:
f["path"] = revmap_f[1]
f["hostfs"] = True
return f
elif f["path"].startswith(builder.outdir):
f["path"] = os.path.join(outdir, f["path"][len(builder.outdir)+1:])
f["hostfs"] = True
return f
else:
raise WorkflowException("Output file path %s must be within designated output directory (%s) or an input file pass through." % (f["path"], builder.outdir))
def collect_output_ports(self, ports, builder, outdir):
try:
custom_output = os.path.join(outdir, "cwl.output.json")
if builder.fs_access.exists(custom_output):
outputdoc = yaml.load(custom_output)
validate.validate_ex(
self.names.get_name("outputs_record_schema", ""),
outputdoc)
return outputdoc
ret = {}
for port in ports:
fragment = shortname(port["id"])
ret[fragment] = self.collect_output(port, builder, outdir)
validate.validate_ex(
self.names.get_name("outputs_record_schema", ""), ret)
return ret if ret is not None else {}
except validate.ValidationException as e:
raise WorkflowException("Error validating output record, " +
str(e) + "\n in " +
json.dumps(ret, indent=4))
def __init__(self, toolpath_object, pos, **kwargs):
if "id" in toolpath_object:
self.id = toolpath_object["id"]
else:
self.id = "#step" + str(pos)
try:
makeTool = kwargs.get("makeTool")
runobj = None
if isinstance(toolpath_object["run"], basestring):
runobj, _ = schema_salad.schema.load_and_validate(
kwargs["loader"], kwargs["avsc_names"],
toolpath_object["run"], True)
else:
runobj = toolpath_object["run"]
self.embedded_tool = makeTool(runobj, **kwargs)
except validate.ValidationException as v:
raise WorkflowException(
"Tool definition %s failed validation:\n%s" %
(toolpath_object["run"], validate.indent(str(v))))
for field in ("inputs", "outputs"):
for i in toolpath_object[field]:
inputid = i["id"]
p = shortname(inputid)
found = False
for a in self.embedded_tool.tool[field]:
frag = shortname(a["id"])
if frag == p:
i.update(a)
found = True
if not found:
i["type"] = "Any"
#raise WorkflowException("Parameter '%s' of %s in workflow step %s does not correspond to parameter in %s" % (p, field, self.id, self.embedded_tool.tool.get("id")))
i["id"] = inputid
super(WorkflowStep, self).__init__(toolpath_object, **kwargs)
if self.embedded_tool.tool["class"] == "Workflow":
(feature,
_) = self.get_requirement("SubworkflowFeatureRequirement")
if not feature:
raise WorkflowException(
"Workflow contains embedded workflow but SubworkflowFeatureRequirement not in requirements"
)
if "scatter" in self.tool:
(feature, _) = self.get_requirement("ScatterFeatureRequirement")
if not feature:
raise WorkflowException(
"Workflow contains scatter but ScatterFeatureRequirement not in requirements"
)
inputparms = copy.deepcopy(self.tool["inputs"])
outputparms = copy.deepcopy(self.tool["outputs"])
scatter = aslist(self.tool["scatter"])
method = self.tool.get("scatterMethod")
if method is None and len(scatter) != 1:
raise WorkflowException(
"Must specify scatterMethod when scattering over multiple inputs"
)
inp_map = {i["id"]: i for i in inputparms}
for s in scatter:
if s not in inp_map:
raise WorkflowException("Invalid Scatter parameter '%s'" %
s)
inp_map[s]["type"] = {
"type": "array",
"items": inp_map[s]["type"]
}
if self.tool.get("scatterMethod") == "nested_crossproduct":
nesting = len(scatter)
else:
nesting = 1
for r in xrange(0, nesting):
for i in outputparms:
i["type"] = {"type": "array", "items": i["type"]}
self.tool["inputs"] = inputparms
self.tool["outputs"] = outputparms
def job(self,
joborder,
basedir,
output_callback,
move_outputs=True,
**kwargs):
self.state = {}
self.processStatus = "success"
if "outdir" in kwargs:
del kwargs["outdir"]
for i in self.tool["inputs"]:
iid = shortname(i["id"])
if iid in joborder:
self.state[i["id"]] = WorkflowStateItem(
i, copy.deepcopy(joborder[iid]))
elif "default" in i:
self.state[i["id"]] = WorkflowStateItem(
i, copy.deepcopy(i["default"]))
else:
raise WorkflowException(
"Input '%s' not in input object and does not have a default value."
% (i["id"]))
for s in self.steps:
for out in s.tool["outputs"]:
self.state[out["id"]] = None
output_dirs = set()
completed = 0
while completed < len(self.steps) and self.processStatus == "success":
made_progress = False
completed = 0
for step in self.steps:
if step.completed:
completed += 1
else:
for newjob in self.try_make_job(step, basedir, **kwargs):
if newjob:
made_progress = True
if newjob.outdir:
output_dirs.add(newjob.outdir)
yield newjob
if not made_progress and completed < len(self.steps):
yield None
supportsMultipleInput = bool(
self.workflow.get_requirement("MultipleInputFeatureRequirement")
[0])
wo = object_from_state(self.state, self.tool["outputs"], True,
supportsMultipleInput)
if wo is None:
raise WorkflowException("Output for workflow not available")
if move_outputs:
targets = set()
conflicts = set()
outfiles = findfiles(wo)
for f in outfiles:
for a in output_dirs:
if f["path"].startswith(a):
src = f["path"]
dst = os.path.join(self.outdir, src[len(a) + 1:])
if dst in targets:
conflicts.add(dst)
else:
targets.add(dst)
for f in outfiles:
for a in output_dirs:
if f["path"].startswith(a):
src = f["path"]
dst = os.path.join(self.outdir, src[len(a) + 1:])
if dst in conflicts:
sp = os.path.splitext(dst)
dst = "%s-%s%s" % (
sp[0], str(random.randint(1,
1000000000)), sp[1])
dirname = os.path.dirname(dst)
if not os.path.exists(dirname):
os.makedirs(dirname)
_logger.debug("[workflow %s] Moving '%s' to '%s'",
self.name, src, dst)
shutil.move(src, dst)
f["path"] = dst
for a in output_dirs:
if os.path.exists(a) and empty_subtree(a):
if kwargs.get("rm_tmpdir", True):
_logger.debug(
"[workflow %s] Removing intermediate output directory %s",
self.name, a)
shutil.rmtree(a, True)
_logger.info("[workflow %s] outdir is %s", self.name, self.outdir)
output_callback(wo, self.processStatus)
def try_make_job(self, step, basedir, **kwargs):
inputparms = step.tool["inputs"]
outputparms = step.tool["outputs"]
supportsMultipleInput = bool(
self.workflow.get_requirement("MultipleInputFeatureRequirement")
[0])
try:
inputobj = object_from_state(self.state, inputparms, False,
supportsMultipleInput)
if inputobj is None:
_logger.debug("[workflow %s] job step %s not ready", self.name,
step.id)
return
_logger.debug("[step %s] starting job step %s of workflow %s",
id(step), step.id, id(self))
if step.submitted:
return
callback = functools.partial(self.receive_output, step,
outputparms)
valueFrom = {
i["id"]: i["valueFrom"]
for i in step.tool["inputs"] if "valueFrom" in i
}
if len(valueFrom) > 0 and not bool(
self.workflow.get_requirement(
"StepInputExpressionRequirement")[0]):
raise WorkflowException(
"Workflow step contains valueFrom but StepInputExpressionRequirement not in requirements"
)
vfinputs = {shortname(k): v for k, v in inputobj.iteritems()}
def valueFromFunc(k, v):
if k in valueFrom:
return expression.do_eval(valueFrom[k],
vfinputs,
self.workflow.requirements,
None,
None, {},
context=v)
else:
return v
if "scatter" in step.tool:
scatter = aslist(step.tool["scatter"])
method = step.tool.get("scatterMethod")
if method is None and len(scatter) != 1:
raise WorkflowException(
"Must specify scatterMethod when scattering over multiple inputs"
)
if "valueFrom" not in kwargs:
kwargs["valueFrom"] = valueFromFunc
if method == "dotproduct" or method is None:
jobs = dotproduct_scatter(step, inputobj, basedir, scatter,
callback, **kwargs)
elif method == "nested_crossproduct":
jobs = nested_crossproduct_scatter(step, inputobj, basedir,
scatter, callback,
**kwargs)
elif method == "flat_crossproduct":
jobs = flat_crossproduct_scatter(step, inputobj, basedir,
scatter, callback, 0,
**kwargs)
else:
_logger.debug("[workflow %s] Job is input %s", self.name,
json.dumps(inputobj, indent=4))
inputobj = {
k: valueFromFunc(k, v)
for k, v in inputobj.items()
}
_logger.debug("[workflow %s] Evaluated job input to %s",
self.name, json.dumps(inputobj, indent=4))
jobs = step.job(inputobj, basedir, callback, **kwargs)
step.submitted = True
for j in jobs:
yield j
except WorkflowException:
raise
except Exception as e:
_logger.exception("Unhandled exception")
self.processStatus = "permanentFail"
step.completed = True
def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True, move_outputs=True, **kwargs):
if not os.path.exists(self.outdir):
os.makedirs(self.outdir)
#with open(os.path.join(outdir, "cwl.input.json"), "w") as fp:
# json.dump(self.joborder, fp)
runtime = []
env = {"TMPDIR": self.tmpdir}
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
for f in self.pathmapper.files():
if not os.path.exists(self.pathmapper.mapper(f)[0]):
raise WorkflowException("Required input file %s not found" % self.pathmapper.mapper(f)[0])
img_id = None
if docker_req and kwargs.get("use_container") is not False:
env = os.environ
img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image)
if docker_is_req and img_id is None:
raise WorkflowException("Docker is required for running this tool.")
if img_id:
runtime = ["docker", "run", "-i"]
for src in self.pathmapper.files():
vol = self.pathmapper.mapper(src)
runtime.append("--volume=%s:%s:ro" % vol)
runtime.append("--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/var/spool/cwl"))
runtime.append("--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp"))
runtime.append("--workdir=%s" % ("/var/spool/cwl"))
runtime.append("--read-only=true")
runtime.append("--net=none")
euid = docker_vm_uid() or os.geteuid()
runtime.append("--user=%s" % (euid))
if rm_container:
runtime.append("--rm")
runtime.append("--env=TMPDIR=/tmp")
for t,v in self.environment.items():
runtime.append("--env=%s=%s" % (t, v))
runtime.append(img_id)
else:
env = self.environment
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
env["TMPDIR"] = self.tmpdir
vars_to_preserve = kwargs.get("preserve_environment")
if vars_to_preserve is not None:
for key, value in os.environ.items():
if key in vars_to_preserve and key not in env:
env[key] = value
stdin = None
stdout = None
scr, _ = get_feature(self, "ShellCommandRequirement")
if scr:
shouldquote = lambda x: False
else:
shouldquote = needs_shell_quoting_re.search
_logger.info("[job %s] %s$ %s%s%s",
self.name,
self.outdir,
" ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
' < %s' % (self.stdin) if self.stdin else '',
' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '')
if dry_run:
return (self.outdir, {})
outputs = {}
try:
for t in self.generatefiles:
if isinstance(self.generatefiles[t], dict):
src = self.generatefiles[t]["path"]
dst = os.path.join(self.outdir, t)
if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir:
_logger.debug("symlinking %s to %s", dst, src)
os.symlink(src, dst)
else:
with open(os.path.join(self.outdir, t), "w") as f:
f.write(self.generatefiles[t])
if self.stdin:
stdin = open(self.pathmapper.mapper(self.stdin)[0], "rb")
else:
stdin = subprocess.PIPE
if self.stdout:
absout = os.path.join(self.outdir, self.stdout)
dn = os.path.dirname(absout)
if dn and not os.path.exists(dn):
os.makedirs(dn)
stdout = open(absout, "wb")
else:
stdout = sys.stderr
sp = subprocess.Popen([str(x) for x in runtime + self.command_line],
shell=False,
close_fds=True,
stdin=stdin,
stdout=stdout,
env=env,
cwd=self.outdir)
if stdin == subprocess.PIPE:
sp.stdin.close()
rcode = sp.wait()
if stdin != subprocess.PIPE:
stdin.close()
if stdout is not sys.stderr:
stdout.close()
if self.successCodes and rcode in self.successCodes:
processStatus = "success"
elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
processStatus = "temporaryFail"
elif self.permanentFailCodes and rcode in self.permanentFailCodes:
processStatus = "permanentFail"
elif rcode == 0:
processStatus = "success"
else:
processStatus = "permanentFail"
for t in self.generatefiles:
if isinstance(self.generatefiles[t], dict):
src = self.generatefiles[t]["path"]
dst = os.path.join(self.outdir, t)
if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir:
os.remove(dst)
os.symlink(self.pathmapper.reversemap(src)[1], dst)
outputs = self.collect_outputs(self.outdir)
except OSError as e:
if e.errno == 2:
if runtime:
_logger.error("'%s' not found", runtime[0])
else:
_logger.error("'%s' not found", self.command_line[0])
else:
_logger.exception("Exception while running job")
processStatus = "permanentFail"
except WorkflowException as e:
_logger.error("Error while running job: %s" % e)
processStatus = "permanentFail"
except Exception as e:
_logger.exception("Exception while running job")
processStatus = "permanentFail"
if processStatus != "success":
_logger.warn("[job %s] completed %s", self.name, processStatus)
else:
_logger.debug("[job %s] completed %s", self.name, processStatus)
_logger.debug("[job %s] %s", self.name, json.dumps(outputs, indent=4))
self.output_callback(outputs, processStatus)
if rm_tmpdir:
_logger.debug("[job %s] Removing temporary directory %s", self.name, self.tmpdir)
shutil.rmtree(self.tmpdir, True)
if move_outputs and empty_subtree(self.outdir):
_logger.debug("[job %s] Removing empty output directory %s", self.name, self.outdir)
shutil.rmtree(self.outdir, True)
def req_with_resp(self,
msg_type,
response_type,
payload={},
timeout_secs=DEFAULT_TIMEOUT,
max_attempts=DEFAULT_ATTEMPTS):
# Need to put error checking here for aguments
if type(response_type) != type([]):
response_type = [response_type]
success = False
device_response = None
socket_id = self.initialize_socket(timeout_secs)
sock = self.socket_table[socket_id]
if len(response_type) == 1 and Acknowledgement in response_type:
msg = msg_type(self.mac_addr,
self.source_id,
seq_num=0,
payload=payload,
ack_requested=True,
response_requested=False)
else:
msg = msg_type(self.mac_addr,
self.source_id,
seq_num=0,
payload=payload,
ack_requested=False,
response_requested=True)
response_seen = False
attempts = 0
while not response_seen and attempts < max_attempts:
sent = False
start_time = time()
timedout = False
while not response_seen and not timedout:
if not sent:
if self.ip_addr:
sock.sendto(msg.packed_message,
(self.ip_addr, self.port))
else:
for ip_addr in UDP_BROADCAST_IP_ADDRS:
sock.sendto(msg.packed_message,
(ip_addr, self.port))
sent = True
if self.verbose:
print("SEND: " + str(msg))
try:
data, (ip_addr, port) = sock.recvfrom(1024)
response = unpack_lifx_message(data)
if self.verbose:
print("RECV: " + str(response))
if type(response) in response_type:
if response.source_id == self.source_id and (
response.target_addr == self.mac_addr
or response.target_addr == BROADCAST_MAC):
response_seen = True
device_response = response
self.ip_addr = ip_addr
success = True
except timeout:
pass
elapsed_time = time() - start_time
timedout = True if elapsed_time > timeout_secs else False
attempts += 1
if not success:
self.close_socket(socket_id)
raise WorkflowException(
"WorkflowException: Did not receive {} from {} (Name: {}) in response to {}"
.format(str(response_type), str(self.mac_addr),
str(self.label), str(msg_type)))
else:
self.close_socket(socket_id)
return device_response
def collect_output(self, schema, builder, outdir):
r = None
if "outputBinding" in schema:
binding = schema["outputBinding"]
globpatterns = []
revmap = functools.partial(revmap_file, builder, outdir)
if "glob" in binding:
r = []
for gb in aslist(binding["glob"]):
gb = builder.do_eval(gb)
if gb:
globpatterns.extend(aslist(gb))
for gb in globpatterns:
if gb.startswith("/"):
raise WorkflowError("glob patterns must not start with '/'")
try:
r.extend([{"path": g, "class": "File", "hostfs": True}
for g in builder.fs_access.glob(os.path.join(outdir, gb))])
except (OSError, IOError) as e:
_logger.warn(str(e))
for files in r:
checksum = hashlib.sha1()
with builder.fs_access.open(files["path"], "rb") as f:
contents = f.read(CONTENT_LIMIT)
if binding.get("loadContents"):
files["contents"] = contents
filesize = 0
while contents != "":
checksum.update(contents)
filesize += len(contents)
contents = f.read(1024*1024)
files["checksum"] = "sha1$%s" % checksum.hexdigest()
files["size"] = filesize
if "format" in schema:
files["format"] = builder.do_eval(schema["format"], context=files)
optional = False
singlefile = False
if isinstance(schema["type"], list):
if "null" in schema["type"]:
optional = True
if "File" in schema["type"]:
singlefile = True
elif schema["type"] == "File":
singlefile = True
if "outputEval" in binding:
r = builder.do_eval(binding["outputEval"], context=r)
if singlefile:
# Handle single file outputs not wrapped in a list
if r is not None and not isinstance(r, (list, tuple)):
r = [r]
if optional and r is None:
pass
elif (r is None or len(r) != 1 or not isinstance(r[0], dict) or "path" not in r[0]):
raise WorkflowException("Expression must return a file object for %s." % schema["id"])
if singlefile:
if not r and not optional:
raise WorkflowException("Did not find output file with glob pattern: '{}'".format(globpatterns))
elif not r and optional:
pass
elif isinstance(r, list):
if len(r) > 1:
raise WorkflowException("Multiple matches for output item that is a single file.")
else:
r = r[0]
# Ensure files point to local references outside of the run environment
adjustFileObjs(r, revmap)
if "secondaryFiles" in schema:
for primary in aslist(r):
if isinstance(primary, dict):
primary["secondaryFiles"] = []
for sf in aslist(schema["secondaryFiles"]):
if isinstance(sf, dict) or "$(" in sf or "${" in sf:
sfpath = builder.do_eval(sf, context=r)
if isinstance(sfpath, basestring):
sfpath = revmap({"path": sfpath, "class": "File"})
else:
sfpath = {"path": substitute(primary["path"], sf), "class": "File", "hostfs": True}
for sfitem in aslist(sfpath):
if builder.fs_access.exists(sfitem["path"]):
primary["secondaryFiles"].append(sfitem)
if not r and optional:
r = None
if not r and isinstance(schema["type"], dict) and schema["type"]["type"] == "record":
r = {}
for f in schema["type"]["fields"]:
r[shortname(f["name"])] = self.collect_output(f, builder, outdir)
return r