def _write_tool(step_dir, name, inputs, outputs, parallel, programs, file_estimates, disk, samples):
out_file = os.path.join(step_dir, "%s.cwl" % name)
cores, mem_gb_per_core = resources.cpu_and_memory((programs or []) + ["default"], samples)
mem_mb_total = int(mem_gb_per_core * cores * 1024)
bcbio_docker_disk = 1 * 1024 # Minimum requirements for bcbio Docker image
cwl_res = {"class": "ResourceRequirement",
"coresMin": cores, "ramMin": mem_mb_total, "outdirMin": bcbio_docker_disk}
if file_estimates and disk:
total_estimate = 0
for key, multiplier in disk.items():
if key in file_estimates:
total_estimate += int(multiplier * file_estimates[key])
if total_estimate:
cwl_res["tmpdirMin"] = total_estimate
cwl_res["outdirMin"] += total_estimate
out = {"class": "CommandLineTool",
"cwlVersion": "v1.0",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"hints": [cwl_res],
"arguments": [],
"inputs": [],
"outputs": []}
out["arguments"].append({"position": 0, "valueFrom":
"sentinel-runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])"})
std_inputs = [{"id": "sentinel-parallel", "type": "string",
"default": parallel},
{"id": "sentinel-outputs", "type": "string",
"default": ",".join([workflow.get_base_id(x["id"]) for x in outputs])}]
inputs = std_inputs + inputs
for i, inp in enumerate(inputs):
base_id = workflow.get_base_id(inp["id"])
inp_tool = copy.deepcopy(inp)
inp_tool["id"] = base_id
for attr in ["source", "valueFrom"]:
inp_tool.pop(attr, None)
inp_binding = {"prefix": "%s=" % base_id, "separate": False,
"itemSeparator": ";;", "position": i}
inp_tool = _place_input_binding(inp_tool, inp_binding, parallel)
inp_tool = _place_secondary_files(inp_tool, inp_binding)
out["inputs"].append(inp_tool)
for outp in outputs:
outp_tool = copy.deepcopy(outp)
outp_tool["id"] = workflow.get_base_id(outp["id"])
out["outputs"].append(outp_tool)
with open(out_file, "w") as out_handle:
def str_presenter(dumper, data):
if len(data.splitlines()) > 1: # check for multiline string
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False)
return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, programs, file_estimates, disk, samples):
out_file = os.path.join(step_dir, "%s.cwl" % name)
cores, mem_gb_per_core = resources.cpu_and_memory((programs or []) + ["default"], samples)
mem_mb_total = int(mem_gb_per_core * cores * 1024)
cwl_res = {"class": "ResourceRequirement",
"coresMin": cores, "ramMin": mem_mb_total}
if file_estimates and disk:
total_estimate = 0
for key, multiplier in disk.items():
if key in file_estimates:
total_estimate += int(multiplier * file_estimates[key])
if total_estimate:
cwl_res["tmpdirMin"] = total_estimate
out = {"class": "CommandLineTool",
"cwlVersion": "cwl:draft-3",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"hints": [cwl_res],
"arguments": [],
"inputs": [],
"outputs": []}
out["arguments"].append({"position": 0, "valueFrom": "sentinel-runtime=$(runtime)"})
std_inputs = [{"id": "#sentinel-parallel", "type": "string",
"default": parallel},
{"id": "#sentinel-outputs", "type": "string",
"default": json.dumps([workflow.get_base_id(x["id"]) for x in outputs],
sort_keys=True, separators=(',', ':'))}]
inputs = std_inputs + inputs
for i, inp in enumerate(inputs):
base_id = workflow.get_base_id(inp["id"])
inp_tool = copy.deepcopy(inp)
inp_tool["id"] = "#%s" % base_id
for attr in ["source", "valueFrom"]:
inp_tool.pop(attr, None)
inp_binding = {"prefix": "%s=" % base_id, "separate": False,
"itemSeparator": ";;", "position": i}
inp_tool = _place_input_binding(inp_tool, inp_binding, parallel)
inp_tool = _place_secondary_files(inp_tool, inp_binding)
out["inputs"].append(inp_tool)
for outp in outputs:
outp_tool = copy.deepcopy(outp)
outp_tool["id"] = "#%s" % workflow.get_base_id(outp["id"])
out["outputs"].append(outp_tool)
with open(out_file, "w") as out_handle:
def str_presenter(dumper, data):
if len(data.splitlines()) > 1: # check for multiline string
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False)
return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, programs, samples):
out_file = os.path.join(step_dir, "%s.cwl" % name)
cores, mem_gb_per_core = resources.cpu_and_memory(programs if programs else ["default"], samples)
mem_mb_total = int(mem_gb_per_core * cores * 1024)
out = {"class": "CommandLineTool",
"cwlVersion": "cwl:draft-3",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"hints": [{"class": "ResourceRequirement",
"coresMin": cores, "ramMin": mem_mb_total}],
"arguments": [],
"inputs": [],
"outputs": []}
out["arguments"].append({"position": 0, "prefix": "sentinel-runtime=", "separate": False,
"valueFrom": "$(JSON.stringify(runtime))"})
std_inputs = [{"id": "#sentinel-parallel", "type": "string",
"default": parallel}]
inputs = std_inputs + inputs
for i, inp in enumerate(inputs):
base_id = workflow.get_base_id(inp["id"])
inp_tool = copy.deepcopy(inp)
inp_tool["id"] = "#%s" % base_id
for attr in ["source", "valueFrom"]:
inp_tool.pop(attr, None)
inp_binding = {"prefix": "%s=" % base_id, "separate": False,
"itemSeparator": ";;", "position": i}
inp_tool = _place_input_binding(inp_tool, inp_binding, parallel)
inp_tool = _place_secondary_files(inp_tool, inp_binding)
out["inputs"].append(inp_tool)
for outp in outputs:
outp_tool = copy.deepcopy(outp)
outp_tool["id"] = "#%s" % workflow.get_base_id(outp["id"])
out["outputs"].append(outp_tool)
with open(out_file, "w") as out_handle:
def str_presenter(dumper, data):
if len(data.splitlines()) > 1: # check for multiline string
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False)
return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, programs, samples):
out_file = os.path.join(step_dir, "%s.cwl" % name)
cores, mem_gb_per_core = resources.cpu_and_memory(programs if programs else ["default"], samples)
mem_mb_total = int(mem_gb_per_core * cores * 1024)
out = {"class": "CommandLineTool",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"hints": [{"class": "ResourceRequirement",
"coresMin": cores, "ramMin": mem_mb_total}],
"arguments": [],
"inputs": [],
"outputs": []}
out["arguments"].append({"position": 0, "prefix": "sentinel-runtime=", "separate": False,
"valueFrom": "$(JSON.stringify(runtime))"})
std_inputs = [{"id": "#sentinel-parallel", "type": "string",
"default": parallel}]
inputs = std_inputs + inputs
for i, inp in enumerate(inputs):
base_id = workflow.get_base_id(inp["id"])
inp_tool = copy.deepcopy(inp)
inp_tool["id"] = "#%s" % base_id
for attr in ["source", "valueFrom"]:
inp_tool.pop(attr, None)
inp_binding = {"prefix": "%s=" % base_id, "separate": False,
"itemSeparator": ";;", "position": i}
inp_tool = _place_input_binding(inp_tool, inp_binding, parallel)
inp_tool = _place_secondary_files(inp_tool, inp_binding)
out["inputs"].append(inp_tool)
for outp in outputs:
outp_tool = copy.deepcopy(outp)
outp_tool["id"] = "#%s" % workflow.get_base_id(outp["id"])
out["outputs"].append(outp_tool)
with open(out_file, "w") as out_handle:
def str_presenter(dumper, data):
if len(data.splitlines()) > 1: # check for multiline string
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False)
return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir,
name,
inputs,
outputs,
parallel,
image,
programs,
file_estimates,
disk,
step_cores,
samples,
cur_remotes,
no_files,
container_tags=None):
out_file = os.path.join(step_dir, "%s.cwl" % name)
resource_cores, mem_gb_per_core = resources.cpu_and_memory(
(programs or []) + ["default"], samples)
cores = min([step_cores, resource_cores]) if step_cores else resource_cores
mem_mb_total = int(mem_gb_per_core * cores * 1024)
cwl_res = {
"class": "ResourceRequirement",
"coresMin": cores,
"ramMin": mem_mb_total
}
disk_hint, input_hint = _get_disk_estimates(name, parallel, inputs,
file_estimates, samples, disk,
cur_remotes, no_files)
cwl_res.update(disk_hint)
docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image
if container_tags is not None:
docker_image, container_tags = _add_current_quay_tag(
docker_image, container_tags)
docker = {
"class": "DockerRequirement",
"dockerPull": docker_image,
"dockerImageId": docker_image
}
out = {
"class": "CommandLineTool",
"cwlVersion": "v1.0",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"requirements": [],
"hints": [docker, cwl_res, input_hint],
"arguments": [],
"inputs": [],
"outputs": []
}
if programs:
def resolve_package(p):
out = {}
parts = p.split("=")
if len(parts) == 2:
out["package"] = parts[0]
out["version"] = [parts[1]]
else:
out["package"] = p
out["specs"] = [
"https://anaconda.org/bioconda/%s" % out["package"]
]
return out
out["hints"].append({
"class": "SoftwareRequirement",
"packages": [resolve_package(p) for p in programs]
})
# GATK requires networking for setting up log4j logging, use arvados extension
if any(p.startswith(("gatk", "sentieon")) for p in programs):
out["hints"] += [{"class": "arv:APIRequirement"}]
# Multi-process methods that read heavily from BAM files need extra keep cache for Arvados
if name in ["pipeline_summary", "variantcall_batch_region", "detect_sv"]:
out["hints"] += [{
"class": "arv:RuntimeConstraints",
"keep_cache": 4096
}]
def add_to_namespaces(k, v, out):
if "$namespaces" not in out:
out["$namespaces"] = {}
out["$namespaces"][k] = v
return out
if any(h.get("class", "").startswith("arv:") for h in out["hints"]):
out = add_to_namespaces("arv", "http://arvados.org/cwl#", out)
if any(h.get("class", "").startswith("dx") for h in out["hints"]):
out = add_to_namespaces("dx", "https://www.dnanexus.com/cwl#", out)
# Use JSON for inputs, rather than command line arguments
# Correctly handles multiple values and batching across CWL runners
use_commandline_args = False
out["requirements"] += [{
"class": "InlineJavascriptRequirement"
}, {
"class":
"InitialWorkDirRequirement",
"listing": [{
"entryname": "cwl.inputs.json",
"entry": "$(JSON.stringify(inputs))"
}]
}]
out["arguments"] += [{
"position":
0,
"valueFrom":
"sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])"
},
"sentinel_parallel=%s" % parallel,
"sentinel_outputs=%s" %
",".join([_get_sentinel_val(v) for v in outputs]),
"sentinel_inputs=%s" % ",".join([
"%s:%s" %
(workflow.get_base_id(v["id"]),
"record" if workflow.is_cwl_record(v) else "var")
for v in inputs
]), "run_number=0"]
out = _add_inputs_to_tool(inputs, out, parallel, use_commandline_args)
out = _add_outputs_to_tool(outputs, out)
_tool_to_file(out, out_file)
return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs,
file_estimates, disk, step_cores, samples):
out_file = os.path.join(step_dir, "%s.cwl" % name)
resource_cores, mem_gb_per_core = resources.cpu_and_memory(
(programs or []) + ["default"], samples)
cores = step_cores if step_cores else resource_cores
mem_mb_total = int(mem_gb_per_core * cores * 1024)
bcbio_docker_disk = 1 * 1024 # Minimum requirements for bcbio Docker image
cwl_res = {
"class": "ResourceRequirement",
"coresMin": cores,
"ramMin": mem_mb_total,
"outdirMin": bcbio_docker_disk
}
docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image
docker = {
"class": "DockerRequirement",
"dockerPull": docker_image,
"dockerImageId": docker_image
}
if file_estimates and disk:
total_estimate = 0
for key, multiplier in disk.items():
if key in file_estimates:
total_estimate += int(multiplier * file_estimates[key])
if total_estimate:
cwl_res["tmpdirMin"] = total_estimate
cwl_res["outdirMin"] += total_estimate
out = {
"class": "CommandLineTool",
"cwlVersion": "v1.0",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"requirements": [],
"hints": [docker, cwl_res],
"arguments": [],
"inputs": [],
"outputs": []
}
if programs:
def resolve_package(p):
out = {}
parts = p.split("=")
if len(parts) == 2:
out["package"] = parts[0]
out["version"] = [parts[1]]
else:
out["package"] = p
out["specs"] = [
"https://anaconda.org/bioconda/%s" % out["package"]
]
return out
out["hints"].append({
"class": "SoftwareRequirement",
"packages": [resolve_package(p) for p in programs]
})
# Use JSON for inputs, rather than command line arguments
# Correctly handles multiple values and batching across CWL runners
use_commandline_args = False
out["requirements"] += [{
"class": "InlineJavascriptRequirement"
}, {
"class":
"InitialWorkDirRequirement",
"listing": [{
"entryname": "cwl.inputs.json",
"entry": "$(JSON.stringify(inputs))"
}]
}]
out["arguments"] += [{
"position":
0,
"valueFrom":
"sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])"
},
"sentinel_parallel=%s" % parallel,
"sentinel_outputs=%s" %
",".join([_get_sentinel_val(v) for v in outputs]),
"sentinel_inputs=%s" % ",".join([
"%s:%s" %
(workflow.get_base_id(v["id"]),
"record" if workflow.is_cwl_record(v) else "var")
for v in inputs
])]
for i, inp in enumerate(inputs):
base_id = workflow.get_base_id(inp["id"])
inp_tool = copy.deepcopy(inp)
inp_tool["id"] = base_id
if inp.get("wf_duplicate"):
inp_tool["id"] += "_toolinput"
for attr in ["source", "valueFrom", "wf_duplicate"]:
inp_tool.pop(attr, None)
if _is_scatter_parallel(parallel) and _do_scatter_var(inp, parallel):
inp_tool = workflow._flatten_nested_input(inp_tool)
if use_commandline_args:
inp_binding = {
"prefix": "%s=" % base_id,
"separate": False,
"itemSeparator": ";;",
"position": i
}
inp_tool = _place_input_binding(inp_tool, inp_binding, parallel)
else:
inp_binding = None
inp_tool = _place_secondary_files(inp_tool, inp_binding)
inp_tool = _clean_record(inp_tool)
out["inputs"].append(inp_tool)
for outp in outputs:
outp_tool = copy.deepcopy(outp)
outp_tool = _clean_record(outp_tool)
outp_tool["id"] = workflow.get_base_id(outp["id"])
out["outputs"].append(outp_tool)
with open(out_file, "w") as out_handle:
def str_presenter(dumper, data):
if len(data.splitlines()) > 1: # check for multiline string
return dumper.represent_scalar('tag:yaml.org,2002:str',
data,
style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.dump(out,
out_handle,
default_flow_style=False,
allow_unicode=False)
return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs,
file_estimates, disk, step_cores, samples):
out_file = os.path.join(step_dir, "%s.cwl" % name)
resource_cores, mem_gb_per_core = resources.cpu_and_memory((programs or []) + ["default"], samples)
cores = step_cores if step_cores else resource_cores
mem_mb_total = int(mem_gb_per_core * cores * 1024)
bcbio_docker_disk = 1 * 1024 # Minimum requirements for bcbio Docker image
cwl_res = {"class": "ResourceRequirement",
"coresMin": cores, "ramMin": mem_mb_total, "outdirMin": bcbio_docker_disk}
docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image
docker = {"class": "DockerRequirement", "dockerPull": docker_image, "dockerImageId": docker_image}
if file_estimates and disk:
total_estimate = 0
for key, multiplier in disk.items():
if key in file_estimates:
total_estimate += int(multiplier * file_estimates[key])
if total_estimate:
cwl_res["tmpdirMin"] = total_estimate
cwl_res["outdirMin"] += total_estimate
out = {"class": "CommandLineTool",
"cwlVersion": "v1.0",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"requirements": [],
"hints": [docker, cwl_res],
"arguments": [],
"inputs": [],
"outputs": []}
if programs:
def resolve_package(p):
out = {}
parts = p.split("=")
if len(parts) == 2:
out["package"] = parts[0]
out["version"] = [parts[1]]
else:
out["package"] = p
out["specs"] = ["https://anaconda.org/bioconda/%s" % out["package"]]
return out
out["hints"].append({"class": "SoftwareRequirement",
"packages": [resolve_package(p) for p in programs]})
# Use JSON for inputs, rather than command line arguments
# Correctly handles multiple values and batching across CWL runners
use_commandline_args = False
out["requirements"] += [{"class": "InlineJavascriptRequirement"},
{"class": "InitialWorkDirRequirement",
"listing": [{"entryname": "cwl.inputs.json",
"entry": "$(JSON.stringify(inputs))"}]}]
out["arguments"] += [{"position": 0, "valueFrom":
"sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])"},
"sentinel_parallel=%s" % parallel,
"sentinel_outputs=%s" % ",".join([_get_sentinel_val(v) for v in outputs]),
"sentinel_inputs=%s" % ",".join(["%s:%s" %
(workflow.get_base_id(v["id"]),
"record" if workflow.is_cwl_record(v) else "var")
for v in inputs])]
for i, inp in enumerate(inputs):
base_id = workflow.get_base_id(inp["id"])
inp_tool = copy.deepcopy(inp)
inp_tool["id"] = base_id
if inp.get("wf_duplicate"):
inp_tool["id"] += "_toolinput"
for attr in ["source", "valueFrom", "wf_duplicate"]:
inp_tool.pop(attr, None)
if _is_scatter_parallel(parallel) and _do_scatter_var(inp, parallel):
inp_tool = workflow._flatten_nested_input(inp_tool)
if use_commandline_args:
inp_binding = {"prefix": "%s=" % base_id,
"separate": False, "itemSeparator": ";;", "position": i}
inp_tool = _place_input_binding(inp_tool, inp_binding, parallel)
else:
inp_binding = None
inp_tool = _place_secondary_files(inp_tool, inp_binding)
out["inputs"].append(inp_tool)
for outp in outputs:
outp_tool = copy.deepcopy(outp)
outp_tool["id"] = workflow.get_base_id(outp["id"])
out["outputs"].append(outp_tool)
with open(out_file, "w") as out_handle:
def str_presenter(dumper, data):
if len(data.splitlines()) > 1: # check for multiline string
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False)
return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs,
file_estimates, disk, step_cores, samples, cur_remotes):
out_file = os.path.join(step_dir, "%s.cwl" % name)
resource_cores, mem_gb_per_core = resources.cpu_and_memory(
(programs or []) + ["default"], samples)
cores = min([step_cores, resource_cores]) if step_cores else resource_cores
mem_mb_total = int(mem_gb_per_core * cores * 1024)
bcbio_docker_disk = 1 * 1024 # Minimum requirements for bcbio Docker image
cwl_res = {
"class": "ResourceRequirement",
"coresMin": cores,
"ramMin": mem_mb_total,
"outdirMin": bcbio_docker_disk
}
cwl_res = _add_disk_estimates(cwl_res, inputs, file_estimates, disk)
docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image
docker = {
"class": "DockerRequirement",
"dockerPull": docker_image,
"dockerImageId": docker_image
}
out = {
"class": "CommandLineTool",
"cwlVersion": "v1.0",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"requirements": [],
"hints": [docker, cwl_res],
"arguments": [],
"inputs": [],
"outputs": []
}
if programs:
def resolve_package(p):
out = {}
parts = p.split("=")
if len(parts) == 2:
out["package"] = parts[0]
out["version"] = [parts[1]]
else:
out["package"] = p
out["specs"] = [
"https://anaconda.org/bioconda/%s" % out["package"]
]
return out
out["hints"].append({
"class": "SoftwareRequirement",
"packages": [resolve_package(p) for p in programs]
})
# GATK requires networking for setting up log4j logging, use arvados extension
if any(p.startswith(("gatk", "sentieon")) for p in programs):
out["hints"] += [{"class": "arv:APIRequirement"}]
# Multi-process methods that read heavily from BAM files need extra keep cache for Arvados
if name in ["pipeline_summary", "variantcall_batch_region"]:
out["hints"] += [{
"class": "arv:RuntimeConstraints",
"keep_cache": 4096
}]
if any(h.get("class", "").startswith("arv:") for h in out["hints"]):
out["$namespaces"] = {"arv": "http://arvados.org/cwl#"}
# Use JSON for inputs, rather than command line arguments
# Correctly handles multiple values and batching across CWL runners
use_commandline_args = False
out["requirements"] += [{
"class": "InlineJavascriptRequirement"
}, {
"class":
"InitialWorkDirRequirement",
"listing": [{
"entryname": "cwl.inputs.json",
"entry": "$(JSON.stringify(inputs))"
}]
}]
out["arguments"] += [{
"position":
0,
"valueFrom":
"sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])"
},
"sentinel_parallel=%s" % parallel,
"sentinel_outputs=%s" %
",".join([_get_sentinel_val(v) for v in outputs]),
"sentinel_inputs=%s" % ",".join([
"%s:%s" %
(workflow.get_base_id(v["id"]),
"record" if workflow.is_cwl_record(v) else "var")
for v in inputs
])]
for i, inp in enumerate(inputs):
base_id = workflow.get_base_id(inp["id"])
inp_tool = copy.deepcopy(inp)
inp_tool["id"] = base_id
if inp.get("wf_duplicate"):
inp_tool["id"] += "_toolinput"
for attr in ["source", "valueFrom", "wf_duplicate"]:
inp_tool.pop(attr, None)
# Ensure records and workflow inputs get scattered
if (_is_scatter_parallel(parallel) and _do_scatter_var(inp, parallel)
and (workflow.is_cwl_record(inp) or inp["wf_duplicate"])):
inp_tool = workflow._flatten_nested_input(inp_tool)
if use_commandline_args:
inp_binding = {
"prefix": "%s=" % base_id,
"separate": False,
"itemSeparator": ";;",
"position": i
}
inp_tool = _place_input_binding(inp_tool, inp_binding, parallel)
else:
inp_binding = None
inp_tool = _place_secondary_files(inp_tool, inp_binding)
inp_tool = _clean_record(inp_tool)
out["inputs"].append(inp_tool)
for outp in outputs:
outp_tool = copy.deepcopy(outp)
outp_tool = _clean_record(outp_tool)
outp_tool["id"] = workflow.get_base_id(outp["id"])
out["outputs"].append(outp_tool)
with open(out_file, "w") as out_handle:
def str_presenter(dumper, data):
if len(data.splitlines()) > 1: # check for multiline string
return dumper.represent_scalar('tag:yaml.org,2002:str',
data,
style='|')
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
yaml.add_representer(str, str_presenter)
yaml.dump(out,
out_handle,
default_flow_style=False,
allow_unicode=False)
return os.path.join("steps", os.path.basename(out_file))
def _write_tool(step_dir, name, inputs, outputs, parallel, image, programs,
file_estimates, disk, step_cores, samples, cur_remotes, no_files,
container_tags=None):
out_file = os.path.join(step_dir, "%s.cwl" % name)
resource_cores, mem_gb_per_core = resources.cpu_and_memory((programs or []) + ["default"], samples)
cores = min([step_cores, resource_cores]) if step_cores else resource_cores
mem_mb_total = int(mem_gb_per_core * cores * 1024)
cwl_res = {"class": "ResourceRequirement", "coresMin": cores, "ramMin": mem_mb_total}
disk_hint, input_hint = _get_disk_estimates(name, parallel, inputs, file_estimates, samples, disk,
cur_remotes, no_files)
cwl_res.update(disk_hint)
docker_image = "bcbio/bcbio" if image == "bcbio" else "quay.io/bcbio/%s" % image
if container_tags is not None:
docker_image, container_tags = _add_current_quay_tag(docker_image, container_tags)
docker = {"class": "DockerRequirement", "dockerPull": docker_image, "dockerImageId": docker_image}
out = {"class": "CommandLineTool",
"cwlVersion": "v1.0",
"baseCommand": ["bcbio_nextgen.py", "runfn", name, "cwl"],
"requirements": [],
"hints": [docker, cwl_res, input_hint],
"arguments": [],
"inputs": [],
"outputs": []}
if programs:
def resolve_package(p):
out = {}
parts = p.split("=")
if len(parts) == 2:
out["package"] = parts[0]
out["version"] = [parts[1]]
else:
out["package"] = p
out["specs"] = ["https://anaconda.org/bioconda/%s" % out["package"]]
return out
out["hints"].append({"class": "SoftwareRequirement",
"packages": [resolve_package(p) for p in programs]})
# GATK requires networking for setting up log4j logging, use arvados extension
if any(p.startswith(("gatk", "sentieon")) for p in programs):
out["hints"] += [{"class": "arv:APIRequirement"}]
# Multi-process methods that read heavily from BAM files need extra keep cache for Arvados
if name in ["pipeline_summary", "variantcall_batch_region", "detect_sv"]:
out["hints"] += [{"class": "arv:RuntimeConstraints", "keep_cache": 4096}]
def add_to_namespaces(k, v, out):
if "$namespaces" not in out:
out["$namespaces"] = {}
out["$namespaces"][k] = v
return out
if any(h.get("class", "").startswith("arv:") for h in out["hints"]):
out = add_to_namespaces("arv", "http://arvados.org/cwl#", out)
if any(h.get("class", "").startswith("dx") for h in out["hints"]):
out = add_to_namespaces("dx", "https://www.dnanexus.com/cwl#", out)
# Use JSON for inputs, rather than command line arguments
# Correctly handles multiple values and batching across CWL runners
use_commandline_args = False
out["requirements"] += [{"class": "InlineJavascriptRequirement"},
{"class": "InitialWorkDirRequirement",
"listing": [{"entryname": "cwl.inputs.json",
"entry": "$(JSON.stringify(inputs))"}]}]
out["arguments"] += [{"position": 0, "valueFrom":
"sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram'])"},
"sentinel_parallel=%s" % parallel,
"sentinel_outputs=%s" % ",".join([_get_sentinel_val(v) for v in outputs]),
"sentinel_inputs=%s" % ",".join(["%s:%s" %
(workflow.get_base_id(v["id"]),
"record" if workflow.is_cwl_record(v) else "var")
for v in inputs]),
"run_number=0"]
out = _add_inputs_to_tool(inputs, out, parallel, use_commandline_args)
out = _add_outputs_to_tool(outputs, out)
_tool_to_file(out, out_file)
return os.path.join("steps", os.path.basename(out_file))
