def __init__(self, biotool, existing_tool=None):
"""
Initialize a [CommandLineTool] object from cwlgen.
:param biotool: Biotool object of an entry from https://bio.tools.
:type biotool: :class:`tooldog.biotool_model.Biotool`
"""
if existing_tool:
LOGGER.info("Loading existing CWL tool from " + existing_tool)
ctp = CWLToolParser()
self.tool = ctp.import_cwl(existing_tool)
if 'None' in self.tool.doc:
self.tool.doc = biotool.generate_cwl_doc()
else:
LOGGER.info("Creating new CwlToolGen object...")
# Initialize counters for inputs and outputs
self.input_ct = 0
self.output_ct = 0
# Initialize tool
# Get the first sentence of the description only
description = biotool.description.split('.')[0] + '.'
documentation = biotool.generate_cwl_doc()
self.tool = cwlgen.CommandLineTool(tool_id=biotool.tool_id,
label=description,
base_command="COMMAND",
doc=documentation,
cwl_version='v1.0')
self._set_meta_from_biotool(biotool)
def create_commandlinetool(
step: Dict[str, Union[str, List]]) -> cwlgen.CommandLineTool:
name = step['name']
description = step['description']
# Initialize commandlinetool
# TODO: Specify path
tool_object = cwlgen.CommandLineTool(tool_id=name,
base_command=f"./{name}.py",
doc=description,
cwl_version="v1.0")
# Specify input parameters
tool_object.inputs += [
cwlgen.CommandInputParameter(p, param_type=DEFAULT_TYPE)
for p in step['input']
]
# Specify output parameters
tool_object.outputs += [
cwlgen.CommandInputParameter(p, param_type=DEFAULT_TYPE)
for p in step['output']
]
return tool_object
def _init_tool(self, cwl_dict):
"""
Init tool from existing CWL tool.
:param cwl_dict: Full content of CWL file
:type cwl_dict: DICT
"""
return cwlgen.CommandLineTool()
def test_commandtool(self):
c = cwlgen.CommandLineTool("reqs")
docker = cwlgen.DockerRequirement(docker_pull="ubuntu:latest")
c.requirements.append(docker)
d = c.get_dict()
self.assertIn("requirements", d)
self.assertIn(docker.get_class(), d["requirements"])
def createGenericStep(id, docker_image, base_command, type, doc, input_doc,
extension, output_doc):
cwl_tool = cwlgen.CommandLineTool(tool_id=id, base_command=base_command)
cwl_tool_docker = cwlgen.DockerRequirement(docker_pull=docker_image)
implementation_file_binding = cwlgen.CommandLineBinding(position=1)
cases_file_binding = cwlgen.CommandLineBinding(position=2)
return createStep(cwl_tool, cwl_tool_docker, implementation_file_binding,
cases_file_binding, type, doc, input_doc, extension,
output_doc, base_command)
def create_command_line_tool(node):
"""
Create a command line tool description file for a single step in a CWL
workflow.
NOTE: CWL only supports workflow steps that are bash shell applications
Non-BashShellApp nodes are unable to be implemented in CWL
"""
# get inputs and outputs
inputs = node.get('inputs', [])
outputs = node.get('outputs', [])
# strip command down to just the basic command, with no input or output parameters
base_command = node.get('command', '')
# TODO: find a better way of specifying command line program + arguments
base_command = base_command[:base_command.index(" ")]
base_command = common.u2s(base_command)
# cwlgen's Serializer class doesn't support python 2.7's unicode types
cwl_tool = cwlgen.CommandLineTool(tool_id=common.u2s(node['app']),
label=common.u2s(node['nm']),
base_command=base_command,
cwl_version='v1.0')
# add inputs
for index, input in enumerate(inputs):
file_binding = cwlgen.CommandLineBinding(position=index)
input_file = cwlgen.CommandInputParameter('input_file_' + str(index),
param_type='File',
input_binding=file_binding,
doc='input file ' +
str(index))
cwl_tool.inputs.append(input_file)
if len(inputs) == 0:
cwl_tool.inputs.append(
cwlgen.CommandInputParameter('dummy',
param_type='null',
doc='dummy'))
# add outputs
for index, output in enumerate(outputs):
file_binding = cwlgen.CommandLineBinding()
output_file = cwlgen.CommandOutputParameter(
'output_file_' + str(index),
param_type='stdout',
output_binding=file_binding,
doc='output file ' + str(index))
cwl_tool.outputs.append(output_file)
return cwl_tool.export_string()
def createKNIMEStep(id, type, doc, input_doc, extension, output_doc):
cwl_tool = cwlgen.CommandLineTool(
tool_id=id, base_command='/home/kclhi/knime_4.1.1/knime')
cwl_tool_docker = cwlgen.DockerRequirement(
docker_pull="kclhi/knime:amia",
docker_output_dir="/home/kclhi/.eclipse")
cwl_tool.arguments = [
'-data', '/home/kclhi/.eclipse', '-reset', '-nosplash', '-nosave',
'-application', 'org.knime.product.KNIME_BATCH_APPLICATION'
]
implementation_file_binding = cwlgen.CommandLineBinding(
prefix="-workflowFile=", separate=False)
cases_file_binding = cwlgen.CommandLineBinding(
prefix="-workflow.variable=dm_potential_cases,file://",
separate=False,
value_from=" $(inputs.potentialCases.path),String")
return createStep(cwl_tool, cwl_tool_docker, implementation_file_binding,
cases_file_binding, type, doc, input_doc, extension,
output_doc, "knime")
def generate_cwl(cls):
"""
Produces a CWL App object which can then be exported to yaml
"""
import cwlgen
module = cls.get_module()
# Basic definition of the tool
cwl_tool = cwlgen.CommandLineTool(tool_id=cls.name,
label=cls.name,
base_command=f'python3 -m {module}')
#TODO: Add documentation in ceci elements
cwl_tool.doc = "Pipeline element from ceci"
# Add the inputs of the tool
for i,inp in enumerate(cls.input_tags()):
input_binding = cwlgen.CommandLineBinding(position=(i+1))
input_param = cwlgen.CommandInputParameter(inp,
param_type='File',
input_binding=input_binding,
doc='Some documentation about the input')
cwl_tool.inputs.append(input_param)
# Add the definition of the outputs
for i,out in enumerate(cls.output_tags()):
output_binding = cwlgen.CommandOutputBinding(glob=out)
output = cwlgen.CommandOutputParameter(out, param_type='File',
output_binding=output_binding,
param_format='http://edamontology.org/format_2330',
doc='Some results produced by the pipeline element')
cwl_tool.outputs.append(output)
# Potentially add more metadata
metadata = {'name': cls.name,
'about': 'I let you guess',
'publication': [{'id': 'one_doi'}, {'id': 'another_doi'}],
'license': ['MIT']}
cwl_tool.metadata = cwlgen.Metadata(**metadata)
return cwl_tool
def generate_cwl(cls, log_dir=None):
"""
Produces a CWL App object which can then be exported to yaml
"""
import cwlgen
module = cls.get_module()
module = module.split(".")[0]
# Basic definition of the tool
cwl_tool = cwlgen.CommandLineTool(
tool_id=cls.name,
label=cls.name,
base_command="python3",
cwl_version="v1.0",
doc=cls.__doc__,
)
if log_dir is not None:
cwl_tool.stdout = f"{cls.name}.out"
cwl_tool.stderr = f"{cls.name}.err"
# Adds the first input binding with the name of the module and pipeline stage
input_arg = cwlgen.CommandLineBinding(position=-1, value_from=f"-m{module}")
cwl_tool.arguments.append(input_arg)
input_arg = cwlgen.CommandLineBinding(position=0, value_from=f"{cls.name}")
cwl_tool.arguments.append(input_arg)
type_dict = {int: "int", float: "float", str: "string", bool: "boolean"}
# Adds the parameters of the tool
for opt, def_val in cls.config_options.items():
# Handles special case of lists:
if isinstance(def_val, list):
v = def_val[0]
param_type = {
"type": "array",
"items": type_dict[v]
if isinstance(v, type)
else type_dict[type(v)],
}
default = def_val if not isinstance(v, type) else None
input_binding = cwlgen.CommandLineBinding(
prefix=f"--{opt}=", item_separator=",", separate=False
)
else:
param_type = (
type_dict[def_val]
if isinstance(def_val, type)
else type_dict[type(def_val)]
)
default = def_val if not isinstance(def_val, type) else None
if param_type == "boolean":
input_binding = cwlgen.CommandLineBinding(prefix=f"--{opt}")
else: # pragma: no cover
input_binding = cwlgen.CommandLineBinding(
prefix=f"--{opt}=", separate=False
)
input_param = cwlgen.CommandInputParameter(
opt,
label=opt,
param_type=param_type,
input_binding=input_binding,
default=default,
doc="Some documentation about this parameter",
)
# We are bypassing the cwlgen builtin type check for the special case
# of arrays until that gets added to the standard
if isinstance(def_val, list):
input_param.type = param_type
cwl_tool.inputs.append(input_param)
# Add the inputs of the tool
for i, inp in enumerate(cls.input_tags()):
input_binding = cwlgen.CommandLineBinding(prefix=f"--{inp}")
input_param = cwlgen.CommandInputParameter(
inp,
label=inp,
param_type="File",
param_format=cls.inputs[i][1].format, # pylint: disable=no-member
input_binding=input_binding,
doc="Some documentation about the input",
)
cwl_tool.inputs.append(input_param)
# Adds the overall configuration file
input_binding = cwlgen.CommandLineBinding(prefix="--config")
input_param = cwlgen.CommandInputParameter(
"config",
label="config",
param_type="File",
param_format="http://edamontology.org/format_3750",
input_binding=input_binding,
doc="Configuration file",
)
cwl_tool.inputs.append(input_param)
# Add the definition of the outputs
for i, out in enumerate(cls.output_tags()):
output_name = cls.outputs[i][1].make_name(out) # pylint: disable=no-member
output_binding = cwlgen.CommandOutputBinding(glob=output_name)
output = cwlgen.CommandOutputParameter(
out,
label=out,
param_type="File",
output_binding=output_binding,
param_format=cls.outputs[i][1].format, # pylint: disable=no-member
doc="Some results produced by the pipeline element",
)
cwl_tool.outputs.append(output)
if log_dir is not None:
output = cwlgen.CommandOutputParameter(
f"{cls.name}@stdout",
label="stdout",
param_type="stdout",
doc="Pipeline elements standard output",
)
cwl_tool.outputs.append(output)
error = cwlgen.CommandOutputParameter(
f"{cls.name}@stderr",
label="stderr",
param_type="stderr",
doc="Pipeline elements standard output",
)
cwl_tool.outputs.append(error)
# Potentially add more metadata
# This requires a schema however...
# metadata = {'name': cls.name,
# 'about': 'Some additional info',
# 'publication': [{'id': 'one_doi'}, {'id': 'another_doi'}],
# 'license': ['MIT']}
# cwl_tool.metadata = cwlgen.Metadata(**metadata)
return cwl_tool
def generate_cwl(cls):
"""
Produces a CWL App object which can then be exported to yaml
"""
import cwlgen
module = cls.get_module()
module = module.split('.')[0]
# Basic definition of the tool
cwl_tool = cwlgen.CommandLineTool(tool_id=cls.name,
label=cls.name,
base_command='python3',
cwl_version='v1.0',
doc=cls.__doc__)
# Adds the first input binding with the name of the module and pipeline stage
input_arg = cwlgen.CommandLineBinding(position=-1,
value_from=f'-m{module}')
cwl_tool.arguments.append(input_arg)
input_arg = cwlgen.CommandLineBinding(position=0,
value_from=f'{cls.name}')
cwl_tool.arguments.append(input_arg)
type_dict = {
int: 'int',
float: 'float',
str: 'string',
bool: 'boolean'
}
# Adds the parameters of the tool
for opt in cls.config_options:
def_val = cls.config_options[opt]
# Handles special case of lists:
if type(def_val) is list:
v = def_val[0]
param_type = {
'type':
'array',
'items':
type_dict[v] if type(v) == type else type_dict[type(v)]
}
default = def_val if type(v) != type else None
input_binding = cwlgen.CommandLineBinding(
prefix='--{}='.format(opt),
item_separator=',',
separate=False)
else:
param_type = type_dict[def_val] if type(
def_val) == type else type_dict[type(def_val)]
default = def_val if type(def_val) != type else None
if param_type is 'boolean':
input_binding = cwlgen.CommandLineBinding(
prefix='--{}'.format(opt))
else:
input_binding = cwlgen.CommandLineBinding(
prefix='--{}='.format(opt), separate=False)
input_param = cwlgen.CommandInputParameter(
opt,
label=opt,
param_type=param_type,
input_binding=input_binding,
default=default,
doc='Some documentation about this parameter')
# We are bypassing the cwlgen builtin type check for the special case
# of arrays until that gets added to the standard
if type(def_val) is list:
input_param.type = param_type
cwl_tool.inputs.append(input_param)
# Add the inputs of the tool
for i, inp in enumerate(cls.input_tags()):
input_binding = cwlgen.CommandLineBinding(
prefix='--{}'.format(inp))
input_param = cwlgen.CommandInputParameter(
inp,
label=inp,
param_type='File',
param_format=cls.inputs[i][1].__name__,
input_binding=input_binding,
doc='Some documentation about the input')
cwl_tool.inputs.append(input_param)
# Adds the overall configuration file
input_binding = cwlgen.CommandLineBinding(prefix='--config')
input_param = cwlgen.CommandInputParameter('config',
label='config',
param_type='File',
param_format='YamlFile',
input_binding=input_binding,
doc='Configuration file')
cwl_tool.inputs.append(input_param)
# Add the definition of the outputs
for i, out in enumerate(cls.output_tags()):
output_name = f'{out}.{cls.outputs[i][1].suffix}'
output_binding = cwlgen.CommandOutputBinding(glob=output_name)
output = cwlgen.CommandOutputParameter(
out,
label=out,
param_type='File',
output_binding=output_binding,
param_format=cls.outputs[i][1].__name__,
doc='Some results produced by the pipeline element')
cwl_tool.outputs.append(output)
# Potentially add more metadata
# This requires a schema however...
# metadata = {'name': cls.name,
# 'about': 'Some additional info',
# 'publication': [{'id': 'one_doi'}, {'id': 'another_doi'}],
# 'license': ['MIT']}
# cwl_tool.metadata = cwlgen.Metadata(**metadata)
return cwl_tool
def setUp(self):
self.cwl = cwlgen.CommandLineTool(tool_id='an_id', label='a description '+\
'with spaces.', base_command='a_command')
def test_commandlinetool_export(self):
import cwlgen
c = cwlgen.CommandLineTool("identifier", "echo")
c.export()
import cwlgen
cwl_tool = cwlgen.CommandLineTool(tool_id='grep',
label='print lines matching a pattern',
base_command='grep',
cwl_version='v1.0')
file_binding = cwlgen.CommandLineBinding(position=2)
input_file = cwlgen.CommandInputParameter(
'input_file',
param_type='File',
input_binding=file_binding,
doc='input file from which you want to look for the pattern')
cwl_tool.inputs.append(input_file)
pattern_binding = cwlgen.CommandLineBinding(position=1)
pattern = cwlgen.CommandInputParameter('pattern',
param_type='string',
input_binding=pattern_binding,
doc='pattern to find in the input file')
cwl_tool.inputs.append(pattern)
cwl_tool.export()
Example of usage of the cwlgen library for command line tool
'''
'''
simulating this,
sh fetchWest.sh http://www.quantum-simulation.org/potentials/sg15_oncv/upf/H_ONCV_PBE-1.0.upf,http://www.quantum-simulation.org/potentials/sg15_oncv/upf/Si_ONCV_PBE-1.1.upf pw 2
'''
########### Import ###########
import cwlgen
if __name__ == "__main__":
# Initialize a command line tool
west_tool = cwlgen.CommandLineTool(
tool_id='west_tool',
label='calculates pw.x, wstat.x and generates pw.out,wstat.out file',
base_command='sh')
# Add inputs (script_file, URLs, type, no_of_cores)
script_binding = cwlgen.CommandLineBinding(position=1)
script_file = cwlgen.CommandInputParameter(
'script_file',
param_type='File',
input_binding=script_binding,
doc=
'West shell script which generates output based on inputs using docker'
)
west_tool.inputs.append(script_file)
file_binding = cwlgen.CommandLineBinding(position=2)
input_file = cwlgen.CommandInputParameter(
## Author(s): Kenzo-Hugo Hillion ## Contact(s): [email protected] ## Python version: 3.6.0 ## Creation : 12-30-2016 ''' Example of usage of the cwlgen library ''' ########### Import ########### import cwlgen if __name__ == "__main__": # Create a tool cwl_tool = cwlgen.CommandLineTool(tool_id='my_tool', label='my_tool is magic',\ base_command='run_my_tool') # Add documentation cwl_tool.doc = "Magic is no magic without secrets..." # Add 2 inputs input_1_binding = cwlgen.CommandLineBinding(position=1) input_1 = cwlgen.CommandInputParameter('config_file', param_type='File',\ input_binding=input_1_binding,\ doc='config file',\ param_format='http://edamontology.org/format_2330') cwl_tool.inputs.append(input_1) input_2_binding = cwlgen.CommandLineBinding(prefix='-t') input_2 = cwlgen.CommandInputParameter('threads', param_type='int',\ input_binding=input_2_binding,\ doc='number of threads')
import cwlgen
tool_object = cwlgen.CommandLineTool(tool_id="echo-tool", base_command="echo", label=None, doc=None,
cwl_version="v1.0", stdin=None, stderr=None, stdout=None, path=None)
tool_object.inputs.append(
cwlgen.CommandInputParameter("myParamId", label=None, secondary_files=None, param_format=None,
streamable=None, doc=None, input_binding=None, default=None, param_type="string")
)
# to get the dictionary representation:
dict_to_export = tool_object.get_dict()
# to get the string representation (YAML)
yaml_export = tool_object.export_string()
# print to console
tool_object.export()
tool_object.export("echotool.cwl")
def translate_tool_internal(cls,
tool,
with_docker=True,
with_resource_overrides=False):
metadata = tool.metadata if tool.metadata else ToolMetadata()
stdouts = [
o.output_type for o in tool.outputs()
if isinstance(o.output_type, Stdout) and o.output_type.stdoutname
]
stdout = stdouts[0].stdoutname if len(stdouts) > 0 else None
if isinstance(stdout, InputSelector):
stdout = translate_input_selector(stdout, code_environment=False)
tool_cwl = cwlgen.CommandLineTool(
tool_id=tool.id(),
base_command=tool.base_command(),
label=tool.id(),
doc=metadata.documentation,
cwl_version=CWL_VERSION,
stdin=None,
stderr=None,
stdout=stdout,
)
# if any(not i.shell_quote for i in tool.inputs()):
tool_cwl.requirements.append(cwlgen.ShellCommandRequirement())
tool_cwl.requirements.extend([cwlgen.InlineJavascriptReq()])
inputs_that_require_localisation = [
ti for ti in tool.inputs()
if ti.localise_file and (issubclass(type(ti.input_type), File) or (
issubclass(type(ti.input_type), Array)
) and issubclass(type(ti.input_type.subtype()), File))
]
if inputs_that_require_localisation:
tool_cwl.requirements.append(
cwlgen.InitialWorkDirRequirement([
"$(inputs.%s)" % ti.id()
for ti in inputs_that_require_localisation
]))
if with_docker:
tool_cwl.requirements.append(
cwlgen.DockerRequirement(
docker_pull=tool.container(),
# docker_load=None,
# docker_file=None,
# docker_import=None,
# docker_image_id=None,
# docker_output_dir=None
))
tool_cwl.inputs.extend(translate_tool_input(i) for i in tool.inputs())
tool_cwl.outputs.extend(
translate_tool_output(o, tool=tool.id()) for o in tool.outputs())
args = tool.arguments()
if args:
tool_cwl.arguments.extend(
translate_tool_argument(a) for a in tool.arguments())
if with_resource_overrides:
# work out whether (the tool of) s is a workflow or tool
tool_cwl.inputs.extend([
cwlgen.CommandInputParameter("runtime_memory",
param_type="float?"),
cwlgen.CommandInputParameter("runtime_cpu", param_type="int?"),
# cwlgen.CommandInputParameter("runtime_disks", param_type="string?"),
])
tool_cwl.requirements.append(
cwlgen.ResourceRequirement(
cores_min="$(inputs.runtime_cpu ? inputs.runtime_cpu : 1)",
ram_min=
"$(inputs.runtime_memory ? Math.floor(1024 * inputs.runtime_memory) : 4096)",
))
return tool_cwl
