def parse_listing(self, listing, inputs):
for item in listing:
if "writable" in item:
raise UnsupportedRequirement(
"The TES spec does not allow for writable inputs")
if "contents" in item:
loc = self.fs_access.join(self.tmpdir, item["basename"])
with self.fs_access.open(loc, "wb") as gen:
gen.write(item["contents"])
else:
loc = item["location"]
parameter = tes.Input(
name=item["basename"],
description="InitialWorkDirRequirement:cwl_input:%s" %
(item["basename"]),
url=file_uri(loc),
path=self.fs_access.join(self.docker_workdir,
item["basename"]),
type=item["class"].upper())
inputs.append(parameter)
return inputs
def check_requirements(self):
for feature in self.supported_features:
requirement, is_required = self.get_requirement(feature)
if requirement and is_required:
for field in requirement:
if not field in self.supported_features[feature]:
raise UnsupportedRequirement('Error: feature {}.{} is not supported'.format(feature, field))
def create(self):
self.cwlwf = self.load_cwl(self.cwl_workflow)
if type(self.cwlwf) == int or check_unsupported_feature(self.cwlwf.tool)[0]:
raise UnsupportedRequirement(check_unsupported_feature(self.cwlwf.tool)[1])
if self.cwlwf.tool["class"] == "CommandLineTool" or self.cwlwf.tool["class"] == "ExpressionTool":
dirname = os.path.dirname(self.default_args["cwl_workflow"])
filename, ext = os.path.splitext(os.path.basename(self.cwl_workflow))
new_workflow_name = os.path.join(dirname, filename + '_workflow' + ext)
generated_workflow = gen_workflow(self.cwlwf.tool, self.cwl_workflow)
with open(new_workflow_name, 'w') as generated_workflow_stream:
generated_workflow_stream.write(json.dumps(generated_workflow, indent=4))
self.cwlwf = self.load_cwl(new_workflow_name)
self.requirements = self.cwlwf.tool.get("requirements", [])
outputs = {}
for step in self.cwlwf.steps:
cwl_task = CWLStepOperator(cwl_step=step,
dag=self,
ui_color='#5C6BC0')
outputs[shortname(step.tool["id"])] = cwl_task
for out in step.tool["outputs"]:
outputs[shortname(out["id"])] = cwl_task
for step in self.cwlwf.steps:
current_task = outputs[shortname(step.tool["id"])]
for inp in step.tool["inputs"]:
step_input_sources = inp.get("source", '') \
if isinstance(inp.get("source", ''), list) \
else [inp.get("source", '')]
for source in step_input_sources:
parent_task = outputs.get(shortname(source), None)
if parent_task and parent_task not in current_task.upstream_list:
current_task.set_upstream(parent_task)
# https://material.io/guidelines/style/color.html#color-color-palette
for t in self.tasks:
if not t.downstream_list and t.upstream_list:
t.ui_color = '#4527A0'
elif not t.upstream_list:
t.ui_color = '#303F9F'
def create_task_msg(self):
input_parameters = self.get_inputs()
output_parameters = []
if self.stdout is not None:
parameter = tes.Output(name="stdout",
url=self.output2url(self.stdout),
path=self.output2path(self.stdout))
output_parameters.append(parameter)
if self.stderr is not None:
parameter = tes.Output(name="stderr",
url=self.output2url(self.stderr),
path=self.output2path(self.stderr))
output_parameters.append(parameter)
output_parameters.append(
tes.Output(name="workdir",
url=self.output2url(""),
path=self.docker_workdir,
type="DIRECTORY"))
container = self.get_container()
cpus = None
ram = None
disk = None
for i in self.builder.requirements:
if i.get("class", "NA") == "ResourceRequirement":
cpus = i.get("coresMin", i.get("coresMax", None))
ram = i.get("ramMin", i.get("ramMax", None))
disk = i.get("outdirMin", i.get("outdirMax", None))
if (cpus is None or isinstance(cpus, str)) or \
(ram is None or isinstance(ram, str)) or \
(disk is None or isinstance(disk, str)):
raise UnsupportedRequirement(
"cwl-tes does not support dynamic resource requests")
ram = ram / 953.674 if ram is not None else None
disk = disk / 953.674 if disk is not None else None
elif i.get("class", "NA") == "DockerRequirement":
if i.get("dockerOutputDirectory", None) is not None:
output_parameters.append(
tes.Output(name="dockerOutputDirectory",
url=self.output2url(""),
path=i.get("dockerOutputDirectory"),
type="DIRECTORY"))
create_body = tes.Task(name=self.name,
description=self.spec.get("doc", ""),
executors=[
tes.Executor(
command=self.command_line,
image=container,
workdir=self.docker_workdir,
stdout=self.output2path(self.stdout),
stderr=self.output2path(self.stderr),
stdin=self.stdin,
env=self.get_envvars())
],
inputs=input_parameters,
outputs=output_parameters,
resources=tes.Resources(cpu_cores=cpus,
ram_gb=ram,
disk_gb=disk),
tags={"CWLDocumentId": self.spec.get("id")})
return create_body