def remove_stage(args):
# get workflow
workflow_id, project = get_workflow_id_and_project(args.workflow)
try:
args.stage = int(args.stage)
except:
pass
dxworkflow = dxpy.DXWorkflow(workflow_id, project=project)
stage_id = try_call(dxworkflow.remove_stage, args.stage)
if args.brief:
print(stage_id)
else:
print("Removed stage " + stage_id)
def once():
try:
desc = test_files[tname]
inputs = read_json_file_maybe_empty(desc.dx_input)
#print("inputs={}".format(inputs))
workflow = dxpy.DXWorkflow(project=project.get_id(), dxid=wfId)
project.new_folder(test_folder, parents=True)
analysis = workflow.run(
inputs,
project=project.get_id(),
folder=test_folder,
name="{} {}".format(desc.wf_name, git_revision),
delay_workspace_destruction=delay_workspace_destruction)
return analysis
except Exception, e:
print("exception message={}".format(e))
return None
def dxWDL(filename, project, folder, reorg=True, inputs=None, quiet=False):
dxWDL_path = ensure_dxWDL()
cmd = [
"java", "-jar", dxWDL_path, "compile",
os.path.join(here, filename), "-project",
project.get_id(), "-folder", folder, "-imports", here + "/wdl"
]
if inputs:
cmd = cmd + ["--inputs", inputs]
if reorg:
cmd = cmd + ["--reorg"]
if quiet:
cmd = cmd + ["--quiet"]
buf = subprocess.check_output(cmd)
wfid = buf.strip()
wf = dxpy.DXWorkflow(wfid, project.get_id())
wf.set_properties({"git_revision": git_revision})
return wf
def copy_wf_test(tname, src_proj, src_folder, debug_flag):
alt_proj_name = "dxWDL_playground_2"
trg_proj = util.get_project(alt_proj_name)
if trg_proj is None:
raise RuntimeError("Could not find project {}".format(alt_proj_name))
test_folder = "/test"
# clean up target
print("cleaning up target")
trg_proj.remove_folder(test_folder, recurse=True)
trg_proj.new_folder(test_folder, parents=True)
# copy to alternate project
src_wf_id = lookup_dataobj(tname, src_proj, src_folder)
print("copy {} to alternate project".format(src_wf_id))
wf = dxpy.DXWorkflow(dxid=src_wf_id)
wf2 = wf.clone(trg_proj.get_id(), folder=test_folder)
# Run the workflow, and wait for completion
runnable = {tname: wf2.get_id()}
run_test_subset(trg_proj, runnable, test_folder, debug_flag)
def once():
try:
desc = test_files[tname]
if tname in test_defaults:
inputs = {}
elif desc.dx_input is None:
inputs = {}
else:
inputs = read_json_file(desc.dx_input)
project.new_folder(test_folder, parents=True)
if desc.kind == "workflow":
exec_obj = dxpy.DXWorkflow(project=project.get_id(), dxid=oid)
elif desc.kind == "applet":
exec_obj = dxpy.DXApplet(project=project.get_id(), dxid=oid)
else:
raise RuntimeError("Unknown kind {}".format(desc.kind))
run_kwargs = {}
if debug_flag:
run_kwargs = {
"debug": {
"debugOn":
['AppError', 'AppInternalError', 'ExecutionError']
},
"allow_ssh": ["*"]
}
if delay_workspace_destruction:
run_kwargs["delay_workspace_destruction"] = True
return exec_obj.run(inputs,
project=project.get_id(),
folder=test_folder,
name="{} {}".format(desc.name, git_revision),
instance_type="mem1_ssd1_x4",
**run_kwargs)
except Exception as e:
print("exception message={}".format(e))
return None
def list_stages(args):
# get workflow
workflow_id, project = get_workflow_id_and_project(args.workflow)
dxworkflow = dxpy.DXWorkflow(workflow_id, project=project)
desc = dxworkflow.describe()
print((printing.BOLD() + printing.GREEN() + '{name}' + printing.ENDC() +
' ({id})').format(**desc))
print()
print('Title: ' + desc['title'])
print('Output Folder: ' + (desc.get('outputFolder') if desc.
get('outputFolder') is not None else '-'))
if len(desc['stages']) == 0:
print()
print(' No stages; add stages with the command "dx add stage"')
for i, stage in enumerate(desc['stages']):
stage['i'] = i
print()
if stage['name'] is None:
stage['name'] = '<no name>'
print((printing.UNDERLINE() + 'Stage {i}' + printing.ENDC() +
': {name} ({id})').format(**stage))
print('Executable {executable}'.format(**stage) + \
(" (" + printing.RED() + "inaccessible" + printing.ENDC() + ")" \
if stage.get('accessible') is False else ""))
if stage['folder'] is not None and stage['folder'].startswith('/'):
stage_output_folder = stage['folder']
else:
stage_output_folder = '<workflow output folder>/' + (
stage['folder'] if stage['folder'] is not None else "")
print('Output Folder {folder}'.format(folder=stage_output_folder))
if "input" in stage and stage["input"]:
print('Bound input ' + \
('\n' + ' '*16).join([
'{key}={value}'.format(key=key, value=io_val_to_str(stage["input"][key])) for
key in stage['input']
]))
def once():
try:
desc = test_files[tname]
if tname in test_defaults:
inputs = {}
else:
inputs = read_json_file_maybe_empty(desc.dx_input)
project.new_folder(test_folder, parents=True)
if desc.kind == "workflow":
exec_obj = dxpy.DXWorkflow(project=project.get_id(), dxid=oid)
elif desc.kind == "applet":
exec_obj = dxpy.DXApplet(project=project.get_id(), dxid=oid)
else:
raise RuntimeError("Unknown kind {}".format(desc.kind))
return exec_obj.run(
inputs,
project=project.get_id(),
folder=test_folder,
name="{} {}".format(desc.name, git_revision),
delay_workspace_destruction=delay_workspace_destruction,
instance_type="mem1_ssd1_x4")
except Exception, e:
print("exception message={}".format(e))
return None
def add_stage(args):
# get workflow
workflow_id, project = get_workflow_id_and_project(args.workflow)
# get executable
exec_handler = try_call(dxpy.utils.resolver.get_exec_handler,
args.executable,
args.alias)
exec_inputs = dxpy.cli.exec_io.ExecutableInputs(exec_handler)
try_call(exec_inputs.update_from_args, args, require_all_inputs=False)
# get folder path
folderpath = None
if args.output_folder is not None:
try:
_ignore, folderpath, _none = resolve_path(args.output_folder, expected='folder')
except:
folderpath = args.output_folder
elif args.relative_output_folder is not None:
folderpath = args.relative_output_folder
# process instance type
try_call(process_instance_type_arg, args)
dxworkflow = dxpy.DXWorkflow(workflow_id, project=project)
stage_id = try_call(dxworkflow.add_stage,
exec_handler,
name=args.name,
stage_id=args.stage_id,
folder=folderpath,
stage_input=exec_inputs.inputs,
instance_type=args.instance_type)
if args.brief:
print(stage_id)
else:
dxpy.utils.describe.print_desc(dxworkflow.describe())
def update_stage(args):
# get workflow
workflow_id, project = get_workflow_id_and_project(args.workflow)
dxworkflow = dxpy.DXWorkflow(workflow_id, project=project)
# process instance type
try_call(process_instance_type_arg, args)
initial_edit_version = dxworkflow.editVersion
try:
args.stage = int(args.stage)
except:
pass
if not any([
args.executable, args.name, args.no_name, args.output_folder,
args.relative_output_folder, args.input, args.input_json,
args.filename, args.instance_type
]):
print('No updates requested; none made')
return
new_exec_handler = None
if args.executable is not None:
# get executable
new_exec_handler = try_call(dxpy.utils.resolver.get_exec_handler,
args.executable, args.alias)
exec_inputs = dxpy.cli.exec_io.ExecutableInputs(new_exec_handler)
try_call(exec_inputs.update_from_args, args, require_all_inputs=False)
stage_input = exec_inputs.inputs
elif args.input or args.input_json or args.filename:
# input is updated, so look up the existing one
existing_exec_handler = dxpy.utils.resolver.get_exec_handler(
dxworkflow.get_stage(args.stage)['executable'])
exec_inputs = dxpy.cli.exec_io.ExecutableInputs(existing_exec_handler)
try_call(exec_inputs.update_from_args, args, require_all_inputs=False)
stage_input = exec_inputs.inputs
else:
stage_input = None
# get folder path
folderpath = None
if args.output_folder is not None:
try:
_ignore, folderpath, _none = resolve_path(args.output_folder,
expected='folder')
except:
folderpath = args.output_folder
elif args.relative_output_folder is not None:
folderpath = args.relative_output_folder
try:
dxworkflow.update_stage(args.stage,
executable=new_exec_handler,
force=args.force,
name=args.name,
unset_name=args.no_name,
folder=folderpath,
stage_input=stage_input,
instance_type=args.instance_type,
edit_version=initial_edit_version)
except InvalidState as e:
if "compatible" in str(e):
err_msg = 'The requested executable could not be verified as a compatible replacement'
if 'incompatibilities' in e.details and e.details[
'incompatibilities']:
err_msg += ' for the following reasons:\n'
err_msg += '\n'.join([
printing.fill(incompat,
initial_indent='- ',
subsequent_indent=' ')
for incompat in e.details['incompatibilities']
])
else:
err_msg += '.'
err_msg += '\nRerun with --force to replace the executable anyway'
err_exit(expected_exceptions=DXCLIError,
exception=DXCLIError(err_msg))
else:
try_call_err_exit()
except:
try_call_err_exit()
def run_workflow(dx_proj, test_folder, oid):
dx_proj.new_folder(test_folder, parents=True)
wf = dxpy.DXWorkflow(project=dx_proj.get_id(), dxid=oid.decode("utf-8"))
return wf.run({}, project=dx_proj.get_id(), folder=test_folder)
def _resolve_workflow(
self,
wdl_path: Path,
workflow_name: str,
kwargs: dict,
) -> dxpy.DXWorkflow:
if wdl_path in DxWdlExecutor._workflow_cache:
return DxWdlExecutor._workflow_cache[wdl_path]
project_id = (
kwargs.get("workflow_project_id") or
kwargs.get("project_id", dxpy.PROJECT_CONTEXT_ID)
)
folder = kwargs.get("workflow_folder") or kwargs.get("folder", "/")
# # This probably isn't necessary, since (I think) dxWDL creates the folder
# # if it doesn't exist
# if not folder:
# folder = "/"
# else:
# # Check that the project exists and create the folder (any any missing
# # parents) if it doesn't exist. May also fail if the user does not have
# # write access to the project.
# project = dxpy.DXProject(project_id)
# project.new_folder(folder, parents=True)
build_workflow = kwargs.get("force", False)
workflow_id = None
if not build_workflow:
existing_workflow = list(dxpy.find_data_objects(
classname="workflow",
name=workflow_name,
project=project_id,
folder=folder,
describe={
"created": True
}
))
if not existing_workflow:
build_workflow = True
else:
created = existing_workflow[0]["describe"]["created"]
if wdl_path.stat().st_mtime > created:
build_workflow = True
elif self._import_dirs:
for import_dir in self._import_dirs:
for imp in import_dir.glob("*.wdl"):
if imp.stat().st_mtime > created:
build_workflow = True
break
else:
workflow_id = existing_workflow[0]["id"]
if build_workflow:
java_args = kwargs.get("java_args", self.java_args) or ""
imports_args = " ".join(f"-imports {d}" for d in self._import_dirs)
extras = kwargs.get("extras")
extras_arg = f"-extras {extras}" if extras else ""
archive = kwargs.get("archive")
archive_arg = "-a" if archive else "-f"
cmd = (
f"{self.java_bin} {java_args} -jar {self._dxwdl_jar_file} compile "
f"{wdl_path} -destination {project_id}:{folder} {imports_args} "
f"{extras_arg} {archive_arg}"
)
LOG.info(f"Building workflow with command '{cmd}'")
try:
workflow_id = subby.sub(cmd).splitlines(False)[-1]
except subby.core.CalledProcessError as perr:
raise ExecutorError(
"dxwdl",
f"Error building DNAnexus workflow with dxWDL; "
f"stdout={perr.stdout}; stderr={perr.stderr}"
) from perr
workflow = dxpy.DXWorkflow(workflow_id)
DxWdlExecutor._workflow_cache[wdl_path] = workflow
return workflow
