def generate_resources_table(
self,
hints: Dict[str, Any],
to_console=True,
to_disk=False,
output_type: str = "tsv",
):
delim = "\t" if output_type == "tsv" else ","
tools = self.get_tools()
header = ["name", "cpu", "memory (GB)"]
data = []
for t in sorted(tools.keys()):
tool = tools[t]
data.append([tool.id(), tool.cpus(hints), tool.memory(hints)])
data.sort(key=lambda a: a[0].lower())
data.insert(0, header)
if to_console:
import tabulate
print(tabulate.tabulate(data, headers="firstrow"))
if to_disk:
import csv
d = ExportPathKeywords.resolve(ExportPathKeywords.default_no_spec,
None, self.id())
path = d + f"resources.{output_type}"
if not os.path.isdir(d):
os.makedirs(d)
Logger.info(f"Writing resources {output_type} to '{path}'")
with open(path, "w+") as mf:
writer = csv.writer(mf, delimiter=delim)
for row in data:
writer.writerow(row)
return data
def translate_tool(
self,
tool,
to_console=True,
to_disk=False,
export_path=None,
with_container=True,
with_resource_overrides=False,
max_cores=None,
max_mem=None,
allow_empty_container=False,
container_override=None,
):
tool_out = self.stringify_translated_tool(
self.translate_tool_internal(
tool,
with_container=with_container,
with_resource_overrides=with_resource_overrides,
allow_empty_container=allow_empty_container,
container_override=lowercase_dictkeys(container_override),
))
if to_console:
print(tool_out)
if to_disk:
d = ExportPathKeywords.resolve(export_path,
workflow_spec=self.name,
workflow_name=tool.id())
if not os.path.exists(d):
os.makedirs(d)
fn_tool = self.tool_filename(tool)
with open(os.path.join(d, fn_tool), "w+") as wf:
Logger.log(f"Writing {fn_tool} to disk")
wf.write(tool_out)
Logger.log(f"Wrote {fn_tool} to disk")
return tool_out
def translate(
self,
tool,
to_console=True,
tool_to_console=False,
with_resource_overrides=False,
to_disk=False,
write_inputs_file=True,
export_path=ExportPathKeywords.default,
should_validate=False,
should_zip=True,
merge_resources=False,
hints=None,
allow_null_if_not_optional=True,
additional_inputs: Dict = None,
max_cores=None,
max_mem=None,
max_duration=None,
with_container=True,
allow_empty_container=False,
container_override=None,
):
str_tool, tr_tools = None, []
if tool.type() == ToolType.Workflow:
tr_tool, tr_tools = self.translate_workflow(
tool,
with_container=with_container,
with_resource_overrides=with_resource_overrides,
allow_empty_container=allow_empty_container,
container_override=lowercase_dictkeys(container_override),
)
str_tool = self.stringify_translated_workflow(tr_tool)
elif isinstance(tool, CodeTool):
tr_tool = self.translate_code_tool_internal(
tool,
allow_empty_container=allow_empty_container,
container_override=lowercase_dictkeys(container_override),
)
str_tool = self.stringify_translated_tool(tr_tool)
else:
tr_tool = self.translate_tool_internal(
tool,
with_container=with_container,
with_resource_overrides=with_resource_overrides,
allow_empty_container=allow_empty_container,
container_override=lowercase_dictkeys(container_override),
)
str_tool = self.stringify_translated_tool(tr_tool)
tr_inp = self.build_inputs_file(
tool,
recursive=False,
merge_resources=merge_resources,
hints=hints,
additional_inputs=additional_inputs,
max_cores=max_cores,
max_mem=max_mem,
max_duration=max_duration,
)
tr_res = self.build_resources_input(tool, hints)
str_inp = self.stringify_translated_inputs(tr_inp)
str_tools = [(
"tools/" + self.tool_filename(t),
self.stringify_translated_workflow(tr_tools[t]),
) for t in tr_tools]
str_resources = self.stringify_translated_inputs(tr_res)
if to_console:
print("=== WORKFLOW ===")
print(str_tool)
if tool_to_console:
print("\n=== TOOLS ===")
[print(f":: {t[0]} ::\n" + t[1]) for t in str_tools]
print("\n=== INPUTS ===")
print(str_inp)
if not merge_resources and with_resource_overrides:
print("\n=== RESOURCES ===")
print(str_resources)
d = ExportPathKeywords.resolve(export_path,
workflow_spec=self.name,
workflow_name=tool.versioned_id())
fn_workflow = self.workflow_filename(tool)
fn_inputs = self.inputs_filename(tool)
fn_resources = self.resources_filename(tool)
if to_disk and write_inputs_file:
if not os.path.isdir(d):
os.makedirs(d)
with open(os.path.join(d, fn_inputs), "w+") as f:
Logger.log(f"Writing {fn_inputs} to disk")
f.write(str_inp)
Logger.log(f"Written {fn_inputs} to disk")
else:
Logger.log("Skipping writing input (yaml) job file")
if to_disk:
toolsdir = os.path.join(d, "tools")
if not os.path.isdir(toolsdir):
os.makedirs(toolsdir)
Logger.info(f"Exporting tool files to '{d}'")
with open(os.path.join(d, fn_workflow), "w+") as wf:
Logger.log(f"Writing {fn_workflow} to disk")
wf.write(str_tool)
Logger.log(f"Wrote {fn_workflow} to disk")
for (fn_tool, disk_str_tool) in str_tools:
with open(os.path.join(d, fn_tool), "w+") as toolfp:
Logger.log(f"Writing {fn_tool} to disk")
toolfp.write(disk_str_tool)
Logger.log(f"Written {fn_tool} to disk")
if not merge_resources and with_resource_overrides:
print("\n=== RESOURCES ===")
with open(os.path.join(d, fn_resources), "w+") as wf:
Logger.log(f"Writing {fn_resources} to disk")
wf.write(str_inp)
Logger.log(f"Wrote {fn_resources} to disk")
print(str_resources)
import subprocess
if should_zip:
Logger.debug("Zipping tools")
with Path(d):
FNULL = open(os.devnull, "w")
zip_result = subprocess.run(
["zip", "-r", "tools.zip", "tools/"], stdout=FNULL)
if zip_result.returncode == 0:
Logger.debug("Zipped tools")
else:
Logger.critical(str(zip_result.stderr.decode()))
if should_validate:
with Path(d):
Logger.info(f"Validating outputted {self.name}")
enved_vcs = [
(os.getenv(x[1:]) if x.startswith("$") else x)
for x in self.validate_command_for(
fn_workflow, fn_inputs, "tools/", "tools.zip")
]
cwltool_result = subprocess.run(enved_vcs)
if cwltool_result.returncode == 0:
Logger.info("Exported tool was validated by: " +
" ".join(enved_vcs))
else:
Logger.critical(str(cwltool_result.stderr))
return str_tool, str_inp, str_tools
def test_no_preceding_slash(self):
path = "test/"
self.assertEqual(os.path.join(getcwd(), path),
ExportPathKeywords.resolve(path, None, None))
def test_random_dot(self):
path = "/mypath/./starting/with/dot"
self.assertEqual(path, ExportPathKeywords.resolve(path, None, None))
def test_replace_dontreplace(self):
path = "/.myfile/starting/with/dot"
self.assertEqual(path, ExportPathKeywords.resolve(path, None, None))
def test_replace_cwd_in_scope(self):
self.assertEqual(
os.path.join(getcwd(), "test"),
ExportPathKeywords.resolve("./test", None, None),
)
def test_replace_falsey(self):
self.assertEqual(getcwd(),
ExportPathKeywords.resolve(False, None, None))
def test_replace_empty(self):
self.assertEqual(getcwd(), ExportPathKeywords.resolve("", None, None))
def test_replace_only_cwd_dot(self):
self.assertEqual(getcwd(), ExportPathKeywords.resolve(".", None, None))
def test_combo_replace(self):
self.assertEqual(
os.path.expanduser("~") + "/Desktop/workflowname/wdl/",
ExportPathKeywords.resolve("~/Desktop/{name}/{language}/", "wdl",
"workflowname"),
)
def test_multi_replace(self):
self.assertEqual(
"/test_multi_replace/test_multi_replace/test_multi_replace",
ExportPathKeywords.resolve("/{name}/{name}/{name}", None,
"test_multi_replace"),
)
def test_workflow_name(self):
self.assertEqual(
"/my/workflow_name/path",
ExportPathKeywords.resolve("/my/{name}/path", None,
"workflow_name"),
)
def test_workflow_spec(self):
self.assertEqual(
"/my/path/to/cwl",
ExportPathKeywords.resolve("/my/path/to/{language}", "cwl", None),
)
def test_user_path(self):
self.assertEqual(os.path.expanduser("~"),
ExportPathKeywords.resolve("~", None, None))
