def __init__(
self,
cwl_step,
ui_color=None,
op_args=None,
op_kwargs=None,
*args, **kwargs):
# self.cwl_base = os.path.abspath(cwl_base) if cwl_base else os.path.abspath('.')
self.working_dir = None
self.outdir = None
self.cwl_step = cwl_step
step_id = shortname(cwl_step.tool["id"])
super(self.__class__, self).__init__(task_id=step_id, *args, **kwargs)
self.op_args = op_args or []
self.op_kwargs = op_kwargs or {}
if ui_color:
self.ui_color = ui_color
def __init__(self,
cwl_step,
ui_color=None,
op_args=None,
op_kwargs=None,
*args,
**kwargs):
# self.cwl_base = os.path.abspath(cwl_base) if cwl_base else os.path.abspath('.')
self.working_dir = None
self.outdir = None
self.cwl_step = cwl_step
step_id = shortname(cwl_step.tool["id"])
super(self.__class__, self).__init__(task_id=step_id, *args, **kwargs)
self.op_args = op_args or []
self.op_kwargs = op_kwargs or {}
if ui_color:
self.ui_color = ui_color
def create(self):
outputs = {}
promises = {}
for inp in self.cwlwf.tool["inputs"]:
promises[inp["id"]] = None
#
# Fullfill all the Vertecies of the DAG
#
_tasks_added = []
while len(self.cwlwf.steps) != len(_tasks_added):
for step in self.cwlwf.steps:
step_id = shortname(step.tool["id"])
if step_id in _tasks_added:
continue
stepinputs_fufilled = True
for inp in step.tool["inputs"]:
if "source" in inp and inp["source"] not in promises and inp["source"] not in outputs:
stepinputs_fufilled = False
break
if stepinputs_fufilled:
cwl_task = CWLStepOperator(
cwl_step=step,
dag=self)
_tasks_added.append(step_id)
for inp in step.tool["inputs"]:
if "default" in inp:
promises[inp["id"]] = None
if "source" in inp and inp["source"] in outputs:
cwl_task.set_upstream(outputs[inp["source"]])
for out in step.tool["outputs"]:
outputs[out["id"]] = cwl_task
def create(self):
outputs = {}
promises = {}
for inp in self.cwlwf.tool["inputs"]:
promises[inp["id"]] = None
#
# Fullfill all the Vertecies of the DAG
#
_tasks_added = []
while len(self.cwlwf.steps) != len(_tasks_added):
for step in self.cwlwf.steps:
step_id = shortname(step.tool["id"])
if step_id in _tasks_added:
continue
stepinputs_fufilled = True
for inp in step.tool["inputs"]:
if "source" in inp and inp[
"source"] not in promises and inp[
"source"] not in outputs:
stepinputs_fufilled = False
break
if stepinputs_fufilled:
cwl_task = CWLStepOperator(cwl_step=step, dag=self)
_tasks_added.append(step_id)
for inp in step.tool["inputs"]:
if "default" in inp:
promises[inp["id"]] = None
if "source" in inp and inp["source"] in outputs:
cwl_task.set_upstream(outputs[inp["source"]])
for out in step.tool["outputs"]:
outputs[out["id"]] = cwl_task
def execute(self, context):
# logging.info("Options {0}:{1}".format(self.task_id, str(sys.argv)))
cur_dir = os.getcwd()
loader = schema_salad.ref_resolver.Loader({"id": "@id"})
upstream_task_ids = [t.task_id for t in self.upstream_list]
upstream_data = self.xcom_pull(context=context,
task_ids=upstream_task_ids)
promises = {}
for j in upstream_data:
data = j
promises = merge(promises, data["promises"])
if "outdir" in data:
self.outdir = data["outdir"]
if "working_folder" in promises:
self.working_dir = promises["working_folder"]
else:
raise cwltool.errors.WorkflowException(
"working_folder is required")
if self.outdir:
if not os.path.exists(self.outdir):
os.makedirs(self.outdir)
os.chdir(self.outdir)
else:
raise cwltool.errors.WorkflowException(
"Outdir is not provided, please use job dispatcher")
jobobj = {}
for inp in self.cwl_step.tool["inputs"]:
inp_id = shortname(inp["id"])
jobobj_id = inp_id.split(".")[-1]
if "source" in inp:
source_id = shortname(inp["source"])
if source_id in promises:
jobobj[jobobj_id] = promises[source_id]
elif "default" in inp:
d = copy.copy(inp["default"])
jobobj[jobobj_id] = d
promises[inp_id] = d
logging.info("Job {self.task_id}: {jobobj}".format(**locals()))
job, _ = loader.resolve_all(jobobj, 'file://%s' % os.path.abspath('.'))
output = cwltool.main.single_job_executor(self.cwl_step.embedded_tool,
job,
self.working_dir,
None,
outdir=self.outdir)
for out in self.cwl_step.tool["outputs"]:
out_id = shortname(out["id"])
jobout_id = out_id.split(".")[-1]
promises[out_id] = output[jobout_id]
os.chdir(cur_dir)
data = {}
data["promises"] = promises
data["outdir"] = self.outdir
logging.info("Data: {0}".format(data))
return data
def execute(self, context):
# logging.info("Options {0}:{1}".format(self.task_id, str(sys.argv)))
cur_dir = os.getcwd()
loader = schema_salad.ref_resolver.Loader({"id": "@id"})
upstream_task_ids = [t.task_id for t in self.upstream_list]
upstream_data = self.xcom_pull(context=context, task_ids=upstream_task_ids)
promises = {}
for j in upstream_data:
data = j
promises = merge(promises, data["promises"])
if "outdir" in data:
self.outdir = data["outdir"]
if "working_folder" in promises:
self.working_dir = promises["working_folder"]
else:
raise cwltool.errors.WorkflowException("working_folder is required")
if self.outdir:
if not os.path.exists(self.outdir):
os.makedirs(self.outdir)
os.chdir(self.outdir)
else:
raise cwltool.errors.WorkflowException("Outdir is not provided, please use job dispatcher")
jobobj = {}
for inp in self.cwl_step.tool["inputs"]:
inp_id = shortname(inp["id"])
jobobj_id = inp_id.split(".")[-1]
if "source" in inp:
source_id = shortname(inp["source"])
if source_id in promises:
jobobj[jobobj_id] = promises[source_id]
elif "default" in inp:
d = copy.copy(inp["default"])
jobobj[jobobj_id] = d
promises[inp_id] = d
logging.info("Job {self.task_id}: {jobobj}".format(**locals()))
job, _ = loader.resolve_all(jobobj, 'file://%s' % os.path.abspath('.'))
output = cwltool.main.single_job_executor(self.cwl_step.embedded_tool, job, self.working_dir, None,
outdir=self.outdir)
for out in self.cwl_step.tool["outputs"]:
out_id = shortname(out["id"])
jobout_id = out_id.split(".")[-1]
promises[out_id] = output[jobout_id]
os.chdir(cur_dir)
data = {}
data["promises"] = promises
data["outdir"] = self.outdir
logging.info("Data: {0}".format(data))
return data
def get_output_list(self):
# return [shortname(o) for o in self.cwlwf.tool["outputs"] ]
outputs = {}
for out in self.cwlwf.tool["outputs"]:
outputs[shortname(out["source"])] = shortname(out["id"])
return outputs
def get_output_list(self):
# return [shortname(o) for o in self.cwlwf.tool["outputs"] ]
outputs = {}
for out in self.cwlwf.tool["outputs"]:
outputs[shortname(out["source"])] = shortname(out["id"])
return outputs
