def from_loaded_task(self, obj: WDL.Task):
rt = obj.runtime
translated_script = self.translate_expr(obj.command)
inputs = obj.inputs
cpus = self.translate_expr(rt.get("cpu"))
if cpus is not None and not isinstance(cpus, (int, float)):
cpus = int(cpus)
c = j.CommandToolBuilder(
tool=obj.name,
base_command=["sh", "script.sh"],
container=self.container_from_runtime(rt, inputs=inputs),
version="DEV",
inputs=[
self.parse_command_tool_input(i) for i in obj.inputs
if not i.name.startswith("runtime_")
],
outputs=[self.parse_command_tool_output(o) for o in obj.outputs],
files_to_create={"script.sh": translated_script},
memory=self.parse_memory_requirement(rt.get("memory")),
cpus=cpus,
disk=self.parse_disk_requirement(rt.get("disks")),
)
return c
def test_1(self):
import janis_core as j
class FileWithSec(j.File):
def __init__(self, optional=False):
super().__init__(optional=optional, extension=".txt")
def secondary_files(self):
return [".sec"]
Tool = j.CommandToolBuilder(
tool="ls",
base_command=["ls"],
inputs=[
j.ToolInput("inp",
FileWithSec,
secondaries_present_as={".sec": "^.sec"})
],
outputs=[
j.ToolOutput("std", j.Stdout),
j.ToolOutput(
"out",
FileWithSec,
secondaries_present_as={".sec": "^.sec"},
glob=j.InputSelector("inp"),
),
],
container="ubuntu:latest",
version="v0.1.0",
)
Tool.translate("wdl")
def ingest_command_line_tool(self, clt):
docker_requirement = None # : Optional[self.cwlgen.DockerRequirement]
files_to_create = {}
memory, cpus, time = None, None, None
for req in clt.requirements or []:
if isinstance(req, self.cwlgen.DockerRequirement):
docker_requirement = req
elif isinstance(req, self.cwlgen.InitialWorkDirRequirement):
for dirent in req.listing:
if isinstance(dirent, str):
continue
files_to_create[self.parse_basic_expression(
dirent.entryname)] = dirent.entry
elif isinstance(req, self.cwlgen.ResourceRequirement):
# maybe convert mebibytes to megabytes?
memory = self.parse_basic_expression(req.ramMin or req.ramMax)
cpus = self.parse_basic_expression(req.coresMin)
elif (hasattr(self.cwlgen, "ToolTimeLimit")
and isinstance(req, self.cwlgen.ToolTimeLimit)
and req.timelimit > 0):
time = req.timelimit
container = None
if docker_requirement:
container = docker_requirement.dockerPull
tool_id = self.get_tool_tag_from_identifier(clt.id)
jclt = j.CommandToolBuilder(
tool=tool_id,
base_command=clt.baseCommand,
inputs=[self.ingest_command_tool_input(inp) for inp in clt.inputs],
outputs=[
self.ingest_command_tool_output(out) for out in clt.outputs
],
arguments=[
self.ingest_command_tool_argument(arg)
for arg in (clt.arguments or [])
],
version="v0.1.0",
container=container or "ubuntu:latest",
doc=clt.doc,
friendly_name=clt.label,
files_to_create=files_to_create or None,
memory=memory,
cpus=cpus,
time=time,
)
return jclt
def command_to_tool(self, cmd: Command) -> janis.CommandToolBuilder:
inputs: List[CliArgument] = [*cmd.named] + (
[] if self.ignore_positionals else [*cmd.positional])
names = self.choose_variable_names(inputs)
tool = janis.CommandToolBuilder(
tool=cmd.as_filename,
base_command=list(cmd.command),
inputs=self.get_inputs(names),
outputs=self.get_outputs(names),
version="v0.1.0",
container=cmd.docker_image,
)
return tool
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
How to create an empty directory, using the value of an input
"""
import janis_core as j
CLT = j.CommandToolBuilder(
tool="create_initial_stuff",
base_command=["ls", "*"],
version="dev",
container="ubuntu:latest",
inputs=[
j.ToolInput("name_of_output_folder",
j.String(optional=True),
default="some-string")
],
outputs=[j.ToolOutput("out_dir", j.Directory, selector="some-string")],
directories_to_create=[j.InputSelector("name_of_output_folder")],
files_to_create=[(
j.StringFormatter("{dir}/file.txt",
dir=j.InputSelector("name_of_output_folder")),
"contents of file",
)],
)
if __name__ == "__main__":
CLT().translate("cwl")
class MyDataTypeWithSecondaries(j.File):
def __init__(self, optional=False):
super().__init__(optional=optional)
@staticmethod
def secondary_files():
return [".csv"]
ToolWithSecondaryFiles = j.CommandToolBuilder(
tool="secondary_files",
version="v0.1.0",
container="ubuntu:latest",
base_command="echo",
inputs=[j.ToolInput("inp", MyDataTypeWithSecondaries, position=0)],
outputs=[
j.ToolOutput(
"out",
MyDataTypeWithSecondaries,
selector="file.txt",
)
],
)
WorkflowWithSecondaries = j.WorkflowBuilder("workflow_with_secondaries")
inner_dt = MyDataTypeWithSecondaries
scatter = None
# helper method to show how scatters / not scatters work
should_scatter = False
if should_scatter:
Create a step whose value is dependent on multiplying an input number,
and the length of an list input. We construct the combination of operators
using fairly natural Python syntax.
"""
import janis_core as j
from janis_core import ToolInput, ToolOutput, Int, Stdout, File
from janis_unix.tools import Echo
AddTwo = j.CommandToolBuilder(
tool="expr",
base_command=None,
arguments=[j.ToolArgument("expr 2 + ", position=0, shell_quote=False)],
version="dev", container="ubuntu:latest",
inputs=[
ToolInput("inp", Int(), position=1)
],
outputs=[
ToolOutput("out", Stdout()),
]
)
w = j.WorkflowBuilder("str_to_int")
w.input("str_input", str)
w.step("echo", Echo(inp=w.str_input))
w.step("exp", AddTwo(inp=w.echo.out.contents().as_int()))
w.output("out", source=w.exp.out)
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import janis_core as j
HelloWorldTool = j.CommandToolBuilder(
tool="HelloWorldTool",
container=None,
version="development",
base_command="echo",
inputs=[j.ToolInput("inp", j.String, position=0, default="Hello, World!")],
outputs=[
j.ToolOutput("out",
j.String,
selector=j.standard.ReadContents(j.Stdout()))
],
)
# as there is no container, you need to provide "allow_empty_containers=True" to the translate method:
HelloWorldTool.translate("wdl", allow_empty_container=True)
# OR (bash):
# $ janis translate --allow-empty-container hello wdl
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import janis_core as j
HelloContainerOverrideTool = j.CommandToolBuilder(
tool="HelloContainerOverrideTool", # <- Matching this toolID
container="ubuntu:latest",
version="development",
base_command="echo",
inputs=[j.ToolInput("inp", j.String, position=0, default="Hello, World!")],
outputs=[
j.ToolOutput("out",
j.String,
selector=j.standard.ReadContents(j.Stdout()))
],
)
# We can swap the container out during translation (and also during running with the CLI)
# by providing the container_override={"ToolID": "containerToOverride"}
# NB: the toolId must match what's in `tool="HelloContainerOverrideTool"`,
# not the variable you've assigned the tool to.
# You can also use {"*": "newcontainer"} to match all tools.
HelloContainerOverrideTool.translate(
"wdl", container_override={"HelloContainerOverrideTool": "ubuntu:bionic"})
import janis_core as j
from unittest import TestCase
from janis_assistant.management.workflowmanager import WorkflowManager
ct = j.CommandToolBuilder(
tool="TEST_EXTENSION",
base_command=["echo", "1", ">", "file.out"],
inputs=[],
outputs=[
j.ToolOutput("out", j.File(extension=".out"), selector="file.out")
],
container="ubuntu:latest",
version="TEST",
)
class TestOutputEvaluation(TestCase):
def test_basic_extension(self):
w = j.WorkflowBuilder("wf")
w.step("stp", ct)
w.output("out", source=w.stp.out)
outputs = WorkflowManager.evaluate_output_params(wf=w,
inputs={},
submission_id="SID",
run_id="RID")
self.assertEqual(".out", outputs[0].extension)
import janis_core as j
ToolWithOptionalWildcardOutput = j.CommandToolBuilder(
tool="optional_wildcard_output_tool",
version="v0.1.0",
container="ubuntu:latest",
base_command=None,
# write '1' to a file called 'out.csv'
arguments=[j.ToolArgument("echo 1 > out.csv", shell_quote=False)],
inputs=[j.ToolInput("inp", j.File, localise_file=True)],
outputs=[
j.ToolOutput("out_csv_files",
j.Array(j.File),
selector=j.WildcardSelector("*.csv")),
# the next two are functionally equivalent
j.ToolOutput("out_single_csv_file_1",
j.Array(j.File),
selector=j.WildcardSelector("*.csv", select_first=True)),
j.ToolOutput("out_single_csv_file_2",
j.Array(j.File),
selector=j.WildcardSelector("*.csv")[0]),
# OPTIONAL glob outputs
# capture all files with *.txt pattern (but select the first if possible)
j.ToolOutput("out_optional_glob",
j.File(optional=True),
selector=j.WildcardSelector("*.txt", select_first=True))
],
)
if __name__ == "__main__":
CommandLineThatBindsArrays = j.CommandToolBuilder(
tool="CommandLineThatBindsArrays",
container="ubuntu:latest",
version="development",
base_command="echo",
inputs=[
# NO prefix, joined by space (' ') [DEFAULT]
# need a position to ensure it binds onto the command line
j.ToolInput("array1", j.Array(j.String), position=0),
#
# No prefix, joined by commans (',')
j.ToolInput("array2", j.Array(j.String), separator=",", position=1),
#
# Single prefix, then joined by space (' ')
j.ToolInput("array3", j.Array(j.String), prefix="--prefix"),
#
# Single prefix, then joined by commas (',')
j.ToolInput("array4",
j.Array(j.String),
prefix="--prefix",
separator=","),
#
# Prefix applies to each element
j.ToolInput(
"array5",
j.Array(j.String),
prefix="--prefix",
prefix_applies_to_all_elements=True,
),
],
outputs=[j.ToolOutput("out", j.Stdout())],
)
import janis_core as j
from janis_unix.tools import Cat
# Declare a new subclass of j.File and write a new tool to accept it
class SubFile(j.File):
@staticmethod
def name():
return "SubFile"
ToolThatAcceptsSubFile = j.CommandToolBuilder(
tool="acceptingSubFile",
base_command="cat",
inputs=[j.ToolInput("inp", SubFile)],
outputs=[j.ToolOutput("out", j.Stdout(SubFile))],
version="test",
container="ubuntu:latest",
)
# Declare workflow we're we'll use the type alias
w = j.WorkflowBuilder("aliasing")
w.input("inp", SubFile)
w.step("stp1", Cat(file=w.inp))
# This line would cause the following critical message to be logged:
# [CRITICAL]: Mismatch of types when joining 'stp1.out' to 'stp2.inp': stdout<File> -/→ SubFile
# w.step("stp2", ToolThatAcceptsSubFile(inp=w.stp1.out))
# Instead, we'll use the .as_type method:
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
NB: Although WDL supports optional files, some backends of Cromwell do not.
"""
import janis_core as j
ToolWithOptionalOutput = j.CommandToolBuilder(
tool="optional_output_tool",
version="v0.1.0",
container="ubuntu:latest",
base_command=[],
arguments=[j.ToolArgument("echo 1 > ", shell_quote=False)],
inputs=[
j.ToolInput("outputFilename",
j.String(optional=True),
default="out.csv",
position=1)
],
outputs=[
j.ToolOutput("out",
j.File(optional=True),
selector=j.InputSelector("outputFilename"))
],
)
if __name__ == "__main__":
ToolWithOptionalOutput().translate("wdl")
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import janis_core as j
CLT = j.CommandToolBuilder(
tool="collecting_std_outputs",
base_command=["echo", "Hello, World"],
version="dev",
container="ubuntu:latest",
inputs=[],
outputs=[
# stdout
j.ToolOutput("out_stdout_1", j.Stdout()),
j.ToolOutput("out_stdout_2", j.File(), selector=j.Stdout()),
# stderr
j.ToolOutput("out_stderr_1", j.Stderr()),
j.ToolOutput("out_stderr_2", j.File(), selector=j.Stderr()),
])
if __name__ == "__main__":
CLT.translate("cwl")
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
We'll use Janis to _try_ and collect a SINGLE file with the extension '.txt',
but one won't exist. CWL will automatically coerce the empty File[0] to File? (null).
However, in WDL, we have do this slightly differently:
File? out = if length(glob("*.txt")) > 0 then glob("*.txt")[0] else None
NB: Although WDL supports optional files, some backends of Cromwell do not.
"""
import janis_core as j
ToolWithDynamicGlob = j.CommandToolBuilder(
tool="tool_with_dynamic_glob",
version="v0.1.0",
container="ubuntu:latest",
base_command=None,
# write '1' to a file called 'out.csv'
arguments=[j.ToolArgument("echo 1 > out.csv", shell_quote=False)],
inputs=[j.ToolInput("extension", str)],
outputs=[
j.ToolOutput("out_csv_files", j.Array(j.File), selector=j.WildcardSelector("*" + j.InputSelector("extension"))),
],
)
if __name__ == "__main__":
ToolWithDynamicGlob().translate("cwl")