def _add_disk_estimates(cwl_res, inputs, file_estimates, disk): """Add disk usage estimates to CWL ResourceRequirement. Based on inputs (which need to be staged) and disk specifications (which estimate outputs). """ if not disk: disk = {} if file_estimates: total_estimate = 0 for key, multiplier in disk.items(): if key in file_estimates: total_estimate += int(multiplier * file_estimates[key]) for inp in inputs: scale = 2.0 if inp.get("type") == "array" else 1.0 if workflow.is_cwl_record(inp): for f in _get_record_fields(inp): if f["name"] in file_estimates: total_estimate += file_estimates[f["name"]] * scale elif inp["id"] in file_estimates: total_estimate += file_estimates[inp["id"]] * scale if total_estimate: # Round total estimates to integer, assign extra half to temp space # It's not entirely clear how different runners interpret this cwl_res["tmpdirMin"] = int(math.ceil(total_estimate * 0.5)) cwl_res["outdirMin"] += int(math.ceil(total_estimate)) return cwl_res
def _add_disk_estimates(cwl_res, inputs, file_estimates, disk): """Add disk usage estimates to CWL ResourceRequirement. Based on inputs (which need to be staged) and disk specifications (which estimate outputs). """ if not disk: disk = {} if file_estimates: total_estimate = 0 for key, multiplier in disk.items(): if key in file_estimates: total_estimate += int(multiplier * file_estimates[key]) for inp in inputs: scale = 2.0 if inp.get("type") == "array" else 1.0 if workflow.is_cwl_record(inp): for f in _get_record_fields(inp): if f["name"] in file_estimates: total_estimate += file_estimates[f["name"]] * scale elif inp["id"] in file_estimates: total_estimate += file_estimates[inp["id"]] * scale if total_estimate: # scale total estimate to allow extra room, round to integer total_estimate = int(math.ceil(total_estimate * 1.5)) cwl_res["tmpdirMin"] = total_estimate cwl_res["outdirMin"] += total_estimate return cwl_res
def _clean_record(rec): """Remove secondary files from record fields, which are currently not supported. To be removed later when secondaryFiles added to records. """ if workflow.is_cwl_record(rec): def _clean_fields(d): if isinstance(d, dict): if "fields" in d: out = [] for f in d["fields"]: f = utils.deepish_copy(f) f.pop("secondaryFiles", None) out.append(f) d["fields"] = out return d else: out = {} for k, v in d.items(): out[k] = _clean_fields(v) return out else: return d return _clean_fields(rec) else: return rec
def _add_inputs_to_tool(inputs, tool, parallel, use_commandline_args=False): for i, inp in enumerate(inputs): base_id = workflow.get_base_id(inp["id"]) inp_tool = copy.deepcopy(inp) inp_tool["id"] = base_id if inp.get("wf_duplicate"): inp_tool["id"] += "_toolinput" for attr in ["source", "valueFrom", "wf_duplicate"]: inp_tool.pop(attr, None) # Ensure records and workflow inputs get scattered if (_is_scatter_parallel(parallel) and _do_scatter_var(inp, parallel) and (workflow.is_cwl_record(inp) or inp["wf_duplicate"])): inp_tool = workflow._flatten_nested_input(inp_tool) if use_commandline_args: inp_binding = { "prefix": "%s=" % base_id, "separate": False, "itemSeparator": ";;", "position": i } inp_tool = _place_input_binding(inp_tool, inp_binding, parallel) else: inp_binding = None inp_tool = _place_secondary_files(inp_tool, inp_binding) inp_tool = _clean_record(inp_tool) tool["inputs"].append(inp_tool) return tool
def _get_sentinel_val(v): """Retrieve expected sentinel value for an output, expanding records. """ out = workflow.get_base_id(v["id"]) if workflow.is_cwl_record(v): out += ":%s" % ";".join([x["name"] for x in _get_record_fields(v)]) return out
def _do_scatter_var(v, parallel): """Logic for scattering a variable. """ # For batches, scatter records only at the top level (double nested) if parallel.startswith("batch") and workflow.is_cwl_record(v): return (tz.get_in(["type", "type"], v) == "array" and tz.get_in(["type", "type", "type"], v) == "array") # Otherwise, scatter arrays else: return (tz.get_in(["type", "type"], v) == "array")
def _get_disk_estimates(name, parallel, inputs, file_estimates, samples, disk, cur_remotes, no_files): """Retrieve disk usage estimates as CWL ResourceRequirement and hint. Disk specification for temporary files and outputs. Also optionally includes disk input estimates as a custom hint for platforms which need to stage these and don't pre-estimate these when allocating machine sizes. """ tmp_disk, out_disk, in_disk = 0, 0, 0 if file_estimates: if disk: for key, multiplier in disk.items(): if key in file_estimates: out_disk += int(multiplier * file_estimates[key]) for inp in inputs: scale = 2.0 if inp.get("type") == "array" else 1.0 # Allocating all samples, could remove for `to_rec` when we ensure we # don't have to stage. Currently dnanexus stages everything so need to consider if parallel in ["multi-combined", "multi-batch" ] and "dnanexus" in cur_remotes: scale *= (len(samples)) if workflow.is_cwl_record(inp): for f in _get_record_fields(inp): if f["name"] in file_estimates: in_disk += file_estimates[f["name"]] * scale elif inp["id"] in file_estimates: in_disk += file_estimates[inp["id"]] * scale # Round total estimates to integer, assign extra half to temp space # It's not entirely clear how different runners interpret this tmp_disk = int(math.ceil(out_disk * 0.5)) out_disk = int(math.ceil(out_disk)) bcbio_docker_disk = ( 10 if cur_remotes else 1) * 1024 # Minimum requirements for bcbio Docker image disk_hint = { "outdirMin": bcbio_docker_disk + out_disk, "tmpdirMin": tmp_disk } # Skip input disk for steps which require only transformation (and thus no staging) if no_files: in_disk = 0 # Avoid accidentally flagging as no staging if we don't know sizes of expected inputs elif in_disk == 0: in_disk = 1 input_hint = { "class": "dx:InputResourceRequirement", "indirMin": int(math.ceil(in_disk)) } return disk_hint, input_hint
def _get_sentinel_val(v): """Retrieve expected sentinel value for an output, expanding records. """ out = workflow.get_base_id(v["id"]) if workflow.is_cwl_record(v): def _get_fields(d): if isinstance(d, dict): if "fields" in d: return d["fields"] else: for v in d.values(): fields = _get_fields(v) if fields: return fields out += ":%s" % ";".join([x["name"] for x in _get_fields(v)]) return out
def _get_disk_estimates(name, parallel, inputs, file_estimates, samples, disk, cur_remotes, no_files): """Retrieve disk usage estimates as CWL ResourceRequirement and hint. Disk specification for temporary files and outputs. Also optionally includes disk input estimates as a custom hint for platforms which need to stage these and don't pre-estimate these when allocating machine sizes. """ tmp_disk, out_disk, in_disk = 0, 0, 0 if file_estimates: if disk: for key, multiplier in disk.items(): if key in file_estimates: out_disk += int(multiplier * file_estimates[key]) for inp in inputs: scale = 2.0 if inp.get("type") == "array" else 1.0 # Allocating all samples, could remove for `to_rec` when we ensure we # don't have to stage. Currently dnanexus stages everything so need to consider if parallel in ["multi-combined", "multi-batch"] and "dnanexus" in cur_remotes: scale *= (len(samples)) if workflow.is_cwl_record(inp): for f in _get_record_fields(inp): if f["name"] in file_estimates: in_disk += file_estimates[f["name"]] * scale elif inp["id"] in file_estimates: in_disk += file_estimates[inp["id"]] * scale # Round total estimates to integer, assign extra half to temp space # It's not entirely clear how different runners interpret this tmp_disk = int(math.ceil(out_disk * 0.5)) out_disk = int(math.ceil(out_disk)) bcbio_docker_disk = (10 if cur_remotes else 1) * 1024 # Minimum requirements for bcbio Docker image disk_hint = {"outdirMin": bcbio_docker_disk + out_disk, "tmpdirMin": tmp_disk} # Skip input disk for steps which require only transformation (and thus no staging) if no_files: in_disk = 0 # Avoid accidentally flagging as no staging if we don't know sizes of expected inputs elif in_disk == 0: in_disk = 1 input_hint = {"class": "dx:InputResourceRequirement", "indirMin": int(math.ceil(in_disk))} return disk_hint, input_hint
def _add_inputs_to_tool(inputs, tool, parallel, use_commandline_args=False): for i, inp in enumerate(inputs): base_id = workflow.get_base_id(inp["id"]) inp_tool = copy.deepcopy(inp) inp_tool["id"] = base_id if inp.get("wf_duplicate"): inp_tool["id"] += "_toolinput" for attr in ["source", "valueFrom", "wf_duplicate"]: inp_tool.pop(attr, None) # Ensure records and workflow inputs get scattered if (_is_scatter_parallel(parallel) and _do_scatter_var(inp, parallel) and (workflow.is_cwl_record(inp) or inp["wf_duplicate"])): inp_tool = workflow._flatten_nested_input(inp_tool) if use_commandline_args: inp_binding = {"prefix": "%s=" % base_id, "separate": False, "itemSeparator": ";;", "position": i} inp_tool = _place_input_binding(inp_tool, inp_binding, parallel) else: inp_binding = None inp_tool = _place_secondary_files(inp_tool, inp_binding) inp_tool = _clean_record(inp_tool) tool["inputs"].append(inp_tool) return tool
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs, file_estimates, disk, step_cores, samples): out_file = os.path.join(step_dir, "%s.cwl" % name) resource_cores, mem_gb_per_core = resources.cpu_and_memory((programs or []) + ["default"], samples) cores = step_cores if step_cores else resource_cores mem_mb_total = int(mem_gb_per_core * cores * 1024) bcbio_docker_disk = 1 * 1024 # Minimum requirements for bcbio Docker image cwl_res = {"class": "ResourceRequirement", "coresMin": cores, "ramMin": mem_mb_total, "outdirMin": bcbio_docker_disk} docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image docker = {"class": "DockerRequirement", "dockerPull": docker_image, "dockerImageId": docker_image} if file_estimates and disk: total_estimate = 0 for key, multiplier in disk.items(): if key in file_estimates: total_estimate += int(multiplier * file_estimates[key]) if total_estimate: cwl_res["tmpdirMin"] = total_estimate cwl_res["outdirMin"] += total_estimate out = {"class": "CommandLineTool", "cwlVersion": "v1.0", "baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"], "requirements": [], "hints": [docker, cwl_res], "arguments": [], "inputs": [], "outputs": []} if programs: def resolve_package(p): out = {} parts = p.split("=") if len(parts) == 2: out["package"] = parts[0] out["version"] = [parts[1]] else: out["package"] = p out["specs"] = ["https://anaconda.org/bioconda/%s" % out["package"]] return out out["hints"].append({"class": "SoftwareRequirement", "packages": [resolve_package(p) for p in programs]}) # Use JSON for inputs, rather than command line arguments # Correctly handles multiple values and batching across CWL runners use_commandline_args = False out["requirements"] += [{"class": "InlineJavascriptRequirement"}, {"class": "InitialWorkDirRequirement", "listing": [{"entryname": "cwl.inputs.json", "entry": "$(JSON.stringify(inputs))"}]}] out["arguments"] += [{"position": 0, "valueFrom": "sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])"}, "sentinel_parallel=%s" % parallel, "sentinel_outputs=%s" % ",".join([_get_sentinel_val(v) for v in outputs]), "sentinel_inputs=%s" % ",".join(["%s:%s" % (workflow.get_base_id(v["id"]), "record" if workflow.is_cwl_record(v) else "var") for v in inputs])] for i, inp in enumerate(inputs): base_id = workflow.get_base_id(inp["id"]) inp_tool = copy.deepcopy(inp) inp_tool["id"] = base_id if inp.get("wf_duplicate"): inp_tool["id"] += "_toolinput" for attr in ["source", "valueFrom", "wf_duplicate"]: inp_tool.pop(attr, None) if _is_scatter_parallel(parallel) and _do_scatter_var(inp, parallel): inp_tool = workflow._flatten_nested_input(inp_tool) if use_commandline_args: inp_binding = {"prefix": "%s=" % base_id, "separate": False, "itemSeparator": ";;", "position": i} inp_tool = _place_input_binding(inp_tool, inp_binding, parallel) else: inp_binding = None inp_tool = _place_secondary_files(inp_tool, inp_binding) out["inputs"].append(inp_tool) for outp in outputs: outp_tool = copy.deepcopy(outp) outp_tool["id"] = workflow.get_base_id(outp["id"]) out["outputs"].append(outp_tool) with open(out_file, "w") as out_handle: def str_presenter(dumper, data): if len(data.splitlines()) > 1: # check for multiline string return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') return dumper.represent_scalar('tag:yaml.org,2002:str', data) yaml.add_representer(str, str_presenter) yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False) return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs, file_estimates, disk, step_cores, samples, cur_remotes): out_file = os.path.join(step_dir, "%s.cwl" % name) resource_cores, mem_gb_per_core = resources.cpu_and_memory( (programs or []) + ["default"], samples) cores = min([step_cores, resource_cores]) if step_cores else resource_cores mem_mb_total = int(mem_gb_per_core * cores * 1024) bcbio_docker_disk = 1 * 1024 # Minimum requirements for bcbio Docker image cwl_res = { "class": "ResourceRequirement", "coresMin": cores, "ramMin": mem_mb_total, "outdirMin": bcbio_docker_disk } cwl_res = _add_disk_estimates(cwl_res, inputs, file_estimates, disk) docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image docker = { "class": "DockerRequirement", "dockerPull": docker_image, "dockerImageId": docker_image } out = { "class": "CommandLineTool", "cwlVersion": "v1.0", "baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"], "requirements": [], "hints": [docker, cwl_res], "arguments": [], "inputs": [], "outputs": [] } if programs: def resolve_package(p): out = {} parts = p.split("=") if len(parts) == 2: out["package"] = parts[0] out["version"] = [parts[1]] else: out["package"] = p out["specs"] = [ "https://anaconda.org/bioconda/%s" % out["package"] ] return out out["hints"].append({ "class": "SoftwareRequirement", "packages": [resolve_package(p) for p in programs] }) # GATK requires networking for setting up log4j logging, use arvados extension if any(p.startswith(("gatk", "sentieon")) for p in programs): out["hints"] += [{"class": "arv:APIRequirement"}] # Multi-process methods that read heavily from BAM files need extra keep cache for Arvados if name in ["pipeline_summary", "variantcall_batch_region"]: out["hints"] += [{ "class": "arv:RuntimeConstraints", "keep_cache": 4096 }] if any(h.get("class", "").startswith("arv:") for h in out["hints"]): out["$namespaces"] = {"arv": "http://arvados.org/cwl#"} # Use JSON for inputs, rather than command line arguments # Correctly handles multiple values and batching across CWL runners use_commandline_args = False out["requirements"] += [{ "class": "InlineJavascriptRequirement" }, { "class": "InitialWorkDirRequirement", "listing": [{ "entryname": "cwl.inputs.json", "entry": "$(JSON.stringify(inputs))" }] }] out["arguments"] += [{ "position": 0, "valueFrom": "sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])" }, "sentinel_parallel=%s" % parallel, "sentinel_outputs=%s" % ",".join([_get_sentinel_val(v) for v in outputs]), "sentinel_inputs=%s" % ",".join([ "%s:%s" % (workflow.get_base_id(v["id"]), "record" if workflow.is_cwl_record(v) else "var") for v in inputs ])] for i, inp in enumerate(inputs): base_id = workflow.get_base_id(inp["id"]) inp_tool = copy.deepcopy(inp) inp_tool["id"] = base_id if inp.get("wf_duplicate"): inp_tool["id"] += "_toolinput" for attr in ["source", "valueFrom", "wf_duplicate"]: inp_tool.pop(attr, None) # Ensure records and workflow inputs get scattered if (_is_scatter_parallel(parallel) and _do_scatter_var(inp, parallel) and (workflow.is_cwl_record(inp) or inp["wf_duplicate"])): inp_tool = workflow._flatten_nested_input(inp_tool) if use_commandline_args: inp_binding = { "prefix": "%s=" % base_id, "separate": False, "itemSeparator": ";;", "position": i } inp_tool = _place_input_binding(inp_tool, inp_binding, parallel) else: inp_binding = None inp_tool = _place_secondary_files(inp_tool, inp_binding) inp_tool = _clean_record(inp_tool) out["inputs"].append(inp_tool) for outp in outputs: outp_tool = copy.deepcopy(outp) outp_tool = _clean_record(outp_tool) outp_tool["id"] = workflow.get_base_id(outp["id"]) out["outputs"].append(outp_tool) with open(out_file, "w") as out_handle: def str_presenter(dumper, data): if len(data.splitlines()) > 1: # check for multiline string return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') return dumper.represent_scalar('tag:yaml.org,2002:str', data) yaml.add_representer(str, str_presenter) yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False) return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs, file_estimates, disk, step_cores, samples, cur_remotes, no_files, container_tags=None): out_file = os.path.join(step_dir, "%s.cwl" % name) resource_cores, mem_gb_per_core = resources.cpu_and_memory( (programs or []) + ["default"], samples) cores = min([step_cores, resource_cores]) if step_cores else resource_cores mem_mb_total = int(mem_gb_per_core * cores * 1024) cwl_res = { "class": "ResourceRequirement", "coresMin": cores, "ramMin": mem_mb_total } disk_hint, input_hint = _get_disk_estimates(name, parallel, inputs, file_estimates, samples, disk, cur_remotes, no_files) cwl_res.update(disk_hint) docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image if container_tags is not None: docker_image, container_tags = _add_current_quay_tag( docker_image, container_tags) docker = { "class": "DockerRequirement", "dockerPull": docker_image, "dockerImageId": docker_image } out = { "class": "CommandLineTool", "cwlVersion": "v1.0", "baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"], "requirements": [], "hints": [docker, cwl_res, input_hint], "arguments": [], "inputs": [], "outputs": [] } if programs: def resolve_package(p): out = {} parts = p.split("=") if len(parts) == 2: out["package"] = parts[0] out["version"] = [parts[1]] else: out["package"] = p out["specs"] = [ "https://anaconda.org/bioconda/%s" % out["package"] ] return out out["hints"].append({ "class": "SoftwareRequirement", "packages": [resolve_package(p) for p in programs] }) # GATK requires networking for setting up log4j logging, use arvados extension if any(p.startswith(("gatk", "sentieon")) for p in programs): out["hints"] += [{"class": "arv:APIRequirement"}] # Multi-process methods that read heavily from BAM files need extra keep cache for Arvados if name in ["pipeline_summary", "variantcall_batch_region", "detect_sv"]: out["hints"] += [{ "class": "arv:RuntimeConstraints", "keep_cache": 4096 }] def add_to_namespaces(k, v, out): if "$namespaces" not in out: out["$namespaces"] = {} out["$namespaces"][k] = v return out if any(h.get("class", "").startswith("arv:") for h in out["hints"]): out = add_to_namespaces("arv", "http://arvados.org/cwl#", out) if any(h.get("class", "").startswith("dx") for h in out["hints"]): out = add_to_namespaces("dx", "https://www.dnanexus.com/cwl#", out) # Use JSON for inputs, rather than command line arguments # Correctly handles multiple values and batching across CWL runners use_commandline_args = False out["requirements"] += [{ "class": "InlineJavascriptRequirement" }, { "class": "InitialWorkDirRequirement", "listing": [{ "entryname": "cwl.inputs.json", "entry": "$(JSON.stringify(inputs))" }] }] out["arguments"] += [{ "position": 0, "valueFrom": "sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])" }, "sentinel_parallel=%s" % parallel, "sentinel_outputs=%s" % ",".join([_get_sentinel_val(v) for v in outputs]), "sentinel_inputs=%s" % ",".join([ "%s:%s" % (workflow.get_base_id(v["id"]), "record" if workflow.is_cwl_record(v) else "var") for v in inputs ]), "run_number=0"] out = _add_inputs_to_tool(inputs, out, parallel, use_commandline_args) out = _add_outputs_to_tool(outputs, out) _tool_to_file(out, out_file) return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs, file_estimates, disk, step_cores, samples): out_file = os.path.join(step_dir, "%s.cwl" % name) resource_cores, mem_gb_per_core = resources.cpu_and_memory( (programs or []) + ["default"], samples) cores = step_cores if step_cores else resource_cores mem_mb_total = int(mem_gb_per_core * cores * 1024) bcbio_docker_disk = 1 * 1024 # Minimum requirements for bcbio Docker image cwl_res = { "class": "ResourceRequirement", "coresMin": cores, "ramMin": mem_mb_total, "outdirMin": bcbio_docker_disk } docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image docker = { "class": "DockerRequirement", "dockerPull": docker_image, "dockerImageId": docker_image } if file_estimates and disk: total_estimate = 0 for key, multiplier in disk.items(): if key in file_estimates: total_estimate += int(multiplier * file_estimates[key]) if total_estimate: cwl_res["tmpdirMin"] = total_estimate cwl_res["outdirMin"] += total_estimate out = { "class": "CommandLineTool", "cwlVersion": "v1.0", "baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"], "requirements": [], "hints": [docker, cwl_res], "arguments": [], "inputs": [], "outputs": [] } if programs: def resolve_package(p): out = {} parts = p.split("=") if len(parts) == 2: out["package"] = parts[0] out["version"] = [parts[1]] else: out["package"] = p out["specs"] = [ "https://anaconda.org/bioconda/%s" % out["package"] ] return out out["hints"].append({ "class": "SoftwareRequirement", "packages": [resolve_package(p) for p in programs] }) # Use JSON for inputs, rather than command line arguments # Correctly handles multiple values and batching across CWL runners use_commandline_args = False out["requirements"] += [{ "class": "InlineJavascriptRequirement" }, { "class": "InitialWorkDirRequirement", "listing": [{ "entryname": "cwl.inputs.json", "entry": "$(JSON.stringify(inputs))" }] }] out["arguments"] += [{ "position": 0, "valueFrom": "sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])" }, "sentinel_parallel=%s" % parallel, "sentinel_outputs=%s" % ",".join([_get_sentinel_val(v) for v in outputs]), "sentinel_inputs=%s" % ",".join([ "%s:%s" % (workflow.get_base_id(v["id"]), "record" if workflow.is_cwl_record(v) else "var") for v in inputs ])] for i, inp in enumerate(inputs): base_id = workflow.get_base_id(inp["id"]) inp_tool = copy.deepcopy(inp) inp_tool["id"] = base_id if inp.get("wf_duplicate"): inp_tool["id"] += "_toolinput" for attr in ["source", "valueFrom", "wf_duplicate"]: inp_tool.pop(attr, None) if _is_scatter_parallel(parallel) and _do_scatter_var(inp, parallel): inp_tool = workflow._flatten_nested_input(inp_tool) if use_commandline_args: inp_binding = { "prefix": "%s=" % base_id, "separate": False, "itemSeparator": ";;", "position": i } inp_tool = _place_input_binding(inp_tool, inp_binding, parallel) else: inp_binding = None inp_tool = _place_secondary_files(inp_tool, inp_binding) inp_tool = _clean_record(inp_tool) out["inputs"].append(inp_tool) for outp in outputs: outp_tool = copy.deepcopy(outp) outp_tool = _clean_record(outp_tool) outp_tool["id"] = workflow.get_base_id(outp["id"]) out["outputs"].append(outp_tool) with open(out_file, "w") as out_handle: def str_presenter(dumper, data): if len(data.splitlines()) > 1: # check for multiline string return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') return dumper.represent_scalar('tag:yaml.org,2002:str', data) yaml.add_representer(str, str_presenter) yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False) return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs, file_estimates, disk, step_cores, samples, cur_remotes, no_files, container_tags=None): out_file = os.path.join(step_dir, "%s.cwl" % name) resource_cores, mem_gb_per_core = resources.cpu_and_memory((programs or []) + ["default"], samples) cores = min([step_cores, resource_cores]) if step_cores else resource_cores mem_mb_total = int(mem_gb_per_core * cores * 1024) cwl_res = {"class": "ResourceRequirement", "coresMin": cores, "ramMin": mem_mb_total} disk_hint, input_hint = _get_disk_estimates(name, parallel, inputs, file_estimates, samples, disk, cur_remotes, no_files) cwl_res.update(disk_hint) docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image if container_tags is not None: docker_image, container_tags = _add_current_quay_tag(docker_image, container_tags) docker = {"class": "DockerRequirement", "dockerPull": docker_image, "dockerImageId": docker_image} out = {"class": "CommandLineTool", "cwlVersion": "v1.0", "baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"], "requirements": [], "hints": [docker, cwl_res, input_hint], "arguments": [], "inputs": [], "outputs": []} if programs: def resolve_package(p): out = {} parts = p.split("=") if len(parts) == 2: out["package"] = parts[0] out["version"] = [parts[1]] else: out["package"] = p out["specs"] = ["https://anaconda.org/bioconda/%s" % out["package"]] return out out["hints"].append({"class": "SoftwareRequirement", "packages": [resolve_package(p) for p in programs]}) # GATK requires networking for setting up log4j logging, use arvados extension if any(p.startswith(("gatk", "sentieon")) for p in programs): out["hints"] += [{"class": "arv:APIRequirement"}] # Multi-process methods that read heavily from BAM files need extra keep cache for Arvados if name in ["pipeline_summary", "variantcall_batch_region", "detect_sv"]: out["hints"] += [{"class": "arv:RuntimeConstraints", "keep_cache": 4096}] def add_to_namespaces(k, v, out): if "$namespaces" not in out: out["$namespaces"] = {} out["$namespaces"][k] = v return out if any(h.get("class", "").startswith("arv:") for h in out["hints"]): out = add_to_namespaces("arv", "http://arvados.org/cwl#", out) if any(h.get("class", "").startswith("dx") for h in out["hints"]): out = add_to_namespaces("dx", "https://www.dnanexus.com/cwl#", out) # Use JSON for inputs, rather than command line arguments # Correctly handles multiple values and batching across CWL runners use_commandline_args = False out["requirements"] += [{"class": "InlineJavascriptRequirement"}, {"class": "InitialWorkDirRequirement", "listing": [{"entryname": "cwl.inputs.json", "entry": "$(JSON.stringify(inputs))"}]}] out["arguments"] += [{"position": 0, "valueFrom": "sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])"}, "sentinel_parallel=%s" % parallel, "sentinel_outputs=%s" % ",".join([_get_sentinel_val(v) for v in outputs]), "sentinel_inputs=%s" % ",".join(["%s:%s" % (workflow.get_base_id(v["id"]), "record" if workflow.is_cwl_record(v) else "var") for v in inputs]), "run_number=0"] out = _add_inputs_to_tool(inputs, out, parallel, use_commandline_args) out = _add_outputs_to_tool(outputs, out) _tool_to_file(out, out_file) return os.path.join("steps", os.path.basename(out_file))