def remove_stage(args): # get workflow workflow_id, project = get_workflow_id_and_project(args.workflow) try: args.stage = int(args.stage) except: pass dxworkflow = dxpy.DXWorkflow(workflow_id, project=project) stage_id = try_call(dxworkflow.remove_stage, args.stage) if args.brief: print(stage_id) else: print("Removed stage " + stage_id)
def once(): try: desc = test_files[tname] inputs = read_json_file_maybe_empty(desc.dx_input) #print("inputs={}".format(inputs)) workflow = dxpy.DXWorkflow(project=project.get_id(), dxid=wfId) project.new_folder(test_folder, parents=True) analysis = workflow.run( inputs, project=project.get_id(), folder=test_folder, name="{} {}".format(desc.wf_name, git_revision), delay_workspace_destruction=delay_workspace_destruction) return analysis except Exception, e: print("exception message={}".format(e)) return None
def dxWDL(filename, project, folder, reorg=True, inputs=None, quiet=False): dxWDL_path = ensure_dxWDL() cmd = [ "java", "-jar", dxWDL_path, "compile", os.path.join(here, filename), "-project", project.get_id(), "-folder", folder, "-imports", here + "/wdl" ] if inputs: cmd = cmd + ["--inputs", inputs] if reorg: cmd = cmd + ["--reorg"] if quiet: cmd = cmd + ["--quiet"] buf = subprocess.check_output(cmd) wfid = buf.strip() wf = dxpy.DXWorkflow(wfid, project.get_id()) wf.set_properties({"git_revision": git_revision}) return wf
def copy_wf_test(tname, src_proj, src_folder, debug_flag): alt_proj_name = "dxWDL_playground_2" trg_proj = util.get_project(alt_proj_name) if trg_proj is None: raise RuntimeError("Could not find project {}".format(alt_proj_name)) test_folder = "/test" # clean up target print("cleaning up target") trg_proj.remove_folder(test_folder, recurse=True) trg_proj.new_folder(test_folder, parents=True) # copy to alternate project src_wf_id = lookup_dataobj(tname, src_proj, src_folder) print("copy {} to alternate project".format(src_wf_id)) wf = dxpy.DXWorkflow(dxid=src_wf_id) wf2 = wf.clone(trg_proj.get_id(), folder=test_folder) # Run the workflow, and wait for completion runnable = {tname: wf2.get_id()} run_test_subset(trg_proj, runnable, test_folder, debug_flag)
def once(): try: desc = test_files[tname] if tname in test_defaults: inputs = {} elif desc.dx_input is None: inputs = {} else: inputs = read_json_file(desc.dx_input) project.new_folder(test_folder, parents=True) if desc.kind == "workflow": exec_obj = dxpy.DXWorkflow(project=project.get_id(), dxid=oid) elif desc.kind == "applet": exec_obj = dxpy.DXApplet(project=project.get_id(), dxid=oid) else: raise RuntimeError("Unknown kind {}".format(desc.kind)) run_kwargs = {} if debug_flag: run_kwargs = { "debug": { "debugOn": ['AppError', 'AppInternalError', 'ExecutionError'] }, "allow_ssh": ["*"] } if delay_workspace_destruction: run_kwargs["delay_workspace_destruction"] = True return exec_obj.run(inputs, project=project.get_id(), folder=test_folder, name="{} {}".format(desc.name, git_revision), instance_type="mem1_ssd1_x4", **run_kwargs) except Exception as e: print("exception message={}".format(e)) return None
def list_stages(args): # get workflow workflow_id, project = get_workflow_id_and_project(args.workflow) dxworkflow = dxpy.DXWorkflow(workflow_id, project=project) desc = dxworkflow.describe() print((printing.BOLD() + printing.GREEN() + '{name}' + printing.ENDC() + ' ({id})').format(**desc)) print() print('Title: ' + desc['title']) print('Output Folder: ' + (desc.get('outputFolder') if desc. get('outputFolder') is not None else '-')) if len(desc['stages']) == 0: print() print(' No stages; add stages with the command "dx add stage"') for i, stage in enumerate(desc['stages']): stage['i'] = i print() if stage['name'] is None: stage['name'] = '<no name>' print((printing.UNDERLINE() + 'Stage {i}' + printing.ENDC() + ': {name} ({id})').format(**stage)) print('Executable {executable}'.format(**stage) + \ (" (" + printing.RED() + "inaccessible" + printing.ENDC() + ")" \ if stage.get('accessible') is False else "")) if stage['folder'] is not None and stage['folder'].startswith('/'): stage_output_folder = stage['folder'] else: stage_output_folder = '<workflow output folder>/' + ( stage['folder'] if stage['folder'] is not None else "") print('Output Folder {folder}'.format(folder=stage_output_folder)) if "input" in stage and stage["input"]: print('Bound input ' + \ ('\n' + ' '*16).join([ '{key}={value}'.format(key=key, value=io_val_to_str(stage["input"][key])) for key in stage['input'] ]))
def once(): try: desc = test_files[tname] if tname in test_defaults: inputs = {} else: inputs = read_json_file_maybe_empty(desc.dx_input) project.new_folder(test_folder, parents=True) if desc.kind == "workflow": exec_obj = dxpy.DXWorkflow(project=project.get_id(), dxid=oid) elif desc.kind == "applet": exec_obj = dxpy.DXApplet(project=project.get_id(), dxid=oid) else: raise RuntimeError("Unknown kind {}".format(desc.kind)) return exec_obj.run( inputs, project=project.get_id(), folder=test_folder, name="{} {}".format(desc.name, git_revision), delay_workspace_destruction=delay_workspace_destruction, instance_type="mem1_ssd1_x4") except Exception, e: print("exception message={}".format(e)) return None
def add_stage(args): # get workflow workflow_id, project = get_workflow_id_and_project(args.workflow) # get executable exec_handler = try_call(dxpy.utils.resolver.get_exec_handler, args.executable, args.alias) exec_inputs = dxpy.cli.exec_io.ExecutableInputs(exec_handler) try_call(exec_inputs.update_from_args, args, require_all_inputs=False) # get folder path folderpath = None if args.output_folder is not None: try: _ignore, folderpath, _none = resolve_path(args.output_folder, expected='folder') except: folderpath = args.output_folder elif args.relative_output_folder is not None: folderpath = args.relative_output_folder # process instance type try_call(process_instance_type_arg, args) dxworkflow = dxpy.DXWorkflow(workflow_id, project=project) stage_id = try_call(dxworkflow.add_stage, exec_handler, name=args.name, stage_id=args.stage_id, folder=folderpath, stage_input=exec_inputs.inputs, instance_type=args.instance_type) if args.brief: print(stage_id) else: dxpy.utils.describe.print_desc(dxworkflow.describe())
def update_stage(args): # get workflow workflow_id, project = get_workflow_id_and_project(args.workflow) dxworkflow = dxpy.DXWorkflow(workflow_id, project=project) # process instance type try_call(process_instance_type_arg, args) initial_edit_version = dxworkflow.editVersion try: args.stage = int(args.stage) except: pass if not any([ args.executable, args.name, args.no_name, args.output_folder, args.relative_output_folder, args.input, args.input_json, args.filename, args.instance_type ]): print('No updates requested; none made') return new_exec_handler = None if args.executable is not None: # get executable new_exec_handler = try_call(dxpy.utils.resolver.get_exec_handler, args.executable, args.alias) exec_inputs = dxpy.cli.exec_io.ExecutableInputs(new_exec_handler) try_call(exec_inputs.update_from_args, args, require_all_inputs=False) stage_input = exec_inputs.inputs elif args.input or args.input_json or args.filename: # input is updated, so look up the existing one existing_exec_handler = dxpy.utils.resolver.get_exec_handler( dxworkflow.get_stage(args.stage)['executable']) exec_inputs = dxpy.cli.exec_io.ExecutableInputs(existing_exec_handler) try_call(exec_inputs.update_from_args, args, require_all_inputs=False) stage_input = exec_inputs.inputs else: stage_input = None # get folder path folderpath = None if args.output_folder is not None: try: _ignore, folderpath, _none = resolve_path(args.output_folder, expected='folder') except: folderpath = args.output_folder elif args.relative_output_folder is not None: folderpath = args.relative_output_folder try: dxworkflow.update_stage(args.stage, executable=new_exec_handler, force=args.force, name=args.name, unset_name=args.no_name, folder=folderpath, stage_input=stage_input, instance_type=args.instance_type, edit_version=initial_edit_version) except InvalidState as e: if "compatible" in str(e): err_msg = 'The requested executable could not be verified as a compatible replacement' if 'incompatibilities' in e.details and e.details[ 'incompatibilities']: err_msg += ' for the following reasons:\n' err_msg += '\n'.join([ printing.fill(incompat, initial_indent='- ', subsequent_indent=' ') for incompat in e.details['incompatibilities'] ]) else: err_msg += '.' err_msg += '\nRerun with --force to replace the executable anyway' err_exit(expected_exceptions=DXCLIError, exception=DXCLIError(err_msg)) else: try_call_err_exit() except: try_call_err_exit()
def run_workflow(dx_proj, test_folder, oid): dx_proj.new_folder(test_folder, parents=True) wf = dxpy.DXWorkflow(project=dx_proj.get_id(), dxid=oid.decode("utf-8")) return wf.run({}, project=dx_proj.get_id(), folder=test_folder)
def _resolve_workflow( self, wdl_path: Path, workflow_name: str, kwargs: dict, ) -> dxpy.DXWorkflow: if wdl_path in DxWdlExecutor._workflow_cache: return DxWdlExecutor._workflow_cache[wdl_path] project_id = ( kwargs.get("workflow_project_id") or kwargs.get("project_id", dxpy.PROJECT_CONTEXT_ID) ) folder = kwargs.get("workflow_folder") or kwargs.get("folder", "/") # # This probably isn't necessary, since (I think) dxWDL creates the folder # # if it doesn't exist # if not folder: # folder = "/" # else: # # Check that the project exists and create the folder (any any missing # # parents) if it doesn't exist. May also fail if the user does not have # # write access to the project. # project = dxpy.DXProject(project_id) # project.new_folder(folder, parents=True) build_workflow = kwargs.get("force", False) workflow_id = None if not build_workflow: existing_workflow = list(dxpy.find_data_objects( classname="workflow", name=workflow_name, project=project_id, folder=folder, describe={ "created": True } )) if not existing_workflow: build_workflow = True else: created = existing_workflow[0]["describe"]["created"] if wdl_path.stat().st_mtime > created: build_workflow = True elif self._import_dirs: for import_dir in self._import_dirs: for imp in import_dir.glob("*.wdl"): if imp.stat().st_mtime > created: build_workflow = True break else: workflow_id = existing_workflow[0]["id"] if build_workflow: java_args = kwargs.get("java_args", self.java_args) or "" imports_args = " ".join(f"-imports {d}" for d in self._import_dirs) extras = kwargs.get("extras") extras_arg = f"-extras {extras}" if extras else "" archive = kwargs.get("archive") archive_arg = "-a" if archive else "-f" cmd = ( f"{self.java_bin} {java_args} -jar {self._dxwdl_jar_file} compile " f"{wdl_path} -destination {project_id}:{folder} {imports_args} " f"{extras_arg} {archive_arg}" ) LOG.info(f"Building workflow with command '{cmd}'") try: workflow_id = subby.sub(cmd).splitlines(False)[-1] except subby.core.CalledProcessError as perr: raise ExecutorError( "dxwdl", f"Error building DNAnexus workflow with dxWDL; " f"stdout={perr.stdout}; stderr={perr.stderr}" ) from perr workflow = dxpy.DXWorkflow(workflow_id) DxWdlExecutor._workflow_cache[wdl_path] = workflow return workflow