def test_stdout_with_conflicting_arg(client):
"""Test stdout with conflicting argument value."""
output = Path(client.path) / "lalala"
output.touch()
client.repo.index.add([str(output)])
client.repo.index.commit("add lalala")
argv = ["echo", "lalala"]
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
stdout="lalala",
)
assert "lalala" == factory.inputs[0].default
assert "string" == factory.inputs[0].type
assert "lalala" == factory.stdout
assert "stdout" == factory.outputs[0].type
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path="dummy.yaml")
tool = tool.association.plan
assert argv == tool.to_argv()
def test_06_params(instance_path):
"""Test referencing input parameters in other fields."""
hello = Path(instance_path) / 'hello.tar'
hello.touch()
argv = ['tar', 'xf', 'hello.tar', 'goodbye.txt']
factory = CommandLineToolFactory(
argv,
directory=instance_path,
working_dir=instance_path,
)
assert factory.inputs[1].default == 'goodbye.txt'
assert factory.inputs[1].type == 'string'
assert factory.inputs[1].inputBinding.position == 2
goodbye_id = factory.inputs[1].id
# simulate run
output = Path(instance_path) / 'goodbye.txt'
output.touch()
parameters = list(factory.guess_outputs([output]))
assert parameters[0][0].type == 'File'
assert parameters[0][0].outputBinding.glob == \
'$(inputs.{0})'.format(goodbye_id)
tool = factory.generate_tool()
assert tool.to_argv() == argv
def test_stdout_with_conflicting_arg(client):
"""Test stdout with conflicting argument value."""
output = Path(client.path) / 'lalala'
output.touch()
client.repo.index.add([str(output)])
client.repo.index.commit('add lalala')
argv = ['echo', 'lalala']
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
stdout='lalala',
)
assert 'lalala' == factory.inputs[0].default
assert 'string' == factory.inputs[0].type
assert 'lalala' == factory.stdout
assert 'stdout' == factory.outputs[0].type
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert argv == tool.to_argv()
def test_09_array_inputs(instance_path):
"""Test specification of input parameters in arrays."""
argv = [
'echo',
'-A',
'one',
'two',
'three',
'-B=four',
'-B=five',
'-B=six',
'-C=seven,eight,nine',
]
tool = CommandLineToolFactory(argv,
directory=instance_path,
working_dir=instance_path).generate_tool()
# TODO add grouping for -A and -B
assert tool.inputs[-1].type == 'string[]'
assert tool.inputs[-1].default == ['seven', 'eight', 'nine']
assert tool.inputs[-1].inputBinding.prefix == '-C='
assert tool.inputs[-1].inputBinding.itemSeparator == ','
assert tool.inputs[-1].inputBinding.separate is False
assert tool.to_argv() == argv
def test_stdin_and_stdout(argv, instance_path):
"""Test stdout mapping."""
input_ = Path(instance_path) / 'input.txt'
input_.touch()
output = Path(instance_path) / 'output.txt'
output.touch()
factory = CommandLineToolFactory(
argv,
directory=instance_path,
working_dir=instance_path,
stdin='input.txt',
stdout='output.txt',
stderr='error.log',
)
assert factory.stdin
if len(argv) > 1:
assert factory.arguments
assert factory.stdout == 'output.txt'
assert factory.outputs[0].type == 'stdout'
tool = factory.generate_tool()
assert tool.to_argv() == argv
std_streams = (' < input.txt', ' > output.txt', ' 2> error.log')
tool_str = str(tool)
assert tool_str.startswith(' '.join(argv))
assert all(stream in tool_str for stream in std_streams)
def test_04_output(client):
"""Test describtion of outputs from a command."""
hello = Path(client.path) / 'hello.tar'
hello.touch()
client.repo.index.add([str(hello)])
client.repo.index.commit('add hello.tar')
argv = ['tar', 'xf', 'hello.tar']
factory = CommandLineToolFactory(argv,
directory=client.path,
working_dir=client.path)
# simulate run
output = Path(client.path) / 'hello.txt'
output.touch()
parameters = list(factory.guess_outputs([output]))
assert 'File' == parameters[0][0].type
assert 'hello.txt' == parameters[0][0].outputBinding.glob
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert argv == tool.to_argv()
def test_05_stdout(client):
"""Test stdout mapping."""
output = Path(client.path) / 'output.txt'
output.touch()
client.repo.index.add([str(output)])
client.repo.index.commit('add output')
argv = ['echo', 'Hello world!']
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
stdout='output.txt',
)
assert 'output.txt' == factory.stdout
assert 'stdout' == factory.outputs[0].type
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert argv == tool.to_argv()
def test_input_directory(instance_path):
"""Test input directory."""
cwd = Path(instance_path)
src = cwd / 'src'
src.mkdir(parents=True)
for i in range(5):
(src / str(i)).touch()
argv = ['tar', 'czvf', 'src.tar', 'src']
factory = CommandLineToolFactory(
argv,
directory=instance_path,
working_dir=instance_path,
)
src_tar = src / 'src.tar'
src_tar.touch()
tool = factory.generate_tool()
assert tool.to_argv() == argv
assert tool.inputs[0].type == 'string'
assert tool.inputs[0].default == src_tar.name
assert tool.inputs[1].type == 'Directory'
assert tool.inputs[1].default.path.samefile(src)
def test_03_input(instance_path):
"""Check the essential input parameters."""
whale = Path(instance_path) / 'whale.txt'
whale.touch()
argv = [
'echo',
'-f',
'-i42',
'--example-string',
'hello',
'--file=whale.txt',
]
tool = CommandLineToolFactory(argv,
directory=instance_path,
working_dir=instance_path).generate_tool()
assert tool.arguments[0].to_argv() == ['-f']
assert tool.inputs[0].default == 42
assert tool.inputs[0].type == 'int'
assert tool.inputs[0].inputBinding.prefix == '-i'
assert tool.inputs[0].inputBinding.separate is False
assert tool.inputs[1].default == 'hello'
assert tool.inputs[1].type == 'string'
assert tool.inputs[1].inputBinding.prefix == '--example-string'
assert tool.inputs[1].inputBinding.separate is True
assert tool.inputs[2].default.path.samefile(whale)
assert tool.inputs[2].type == 'File'
assert tool.inputs[2].inputBinding.prefix == '--file='
assert tool.inputs[2].inputBinding.separate is False
assert tool.to_argv() == argv
def test_05_stdout(client):
"""Test stdout mapping."""
output = Path(client.path) / "output.txt"
output.touch()
client.repo.index.add([str(output)])
client.repo.index.commit("add output")
argv = ["echo", "Hello world!"]
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
stdout="output.txt",
)
assert "output.txt" == factory.stdout
assert "stdout" == factory.outputs[0].type
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path="dummy.yaml")
tool = tool.association.plan
assert argv == tool.to_argv()
def test_short_base_command_detection():
"""Test base command detection without arguments."""
tool = CommandLineToolFactory(('echo', 'A')).generate_tool()
assert tool.cwlVersion == 'v1.0'
assert tool.__class__.__name__ == 'CommandLineTool'
assert tool.inputs[0].default == 'A'
assert tool.to_argv() == ['echo', 'A']
def test_short_base_command_detection(client):
"""Test base command detection without arguments."""
tool = CommandLineToolFactory(
('echo', 'A')).generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert 'A' == tool.arguments[0].value
assert ['echo', 'A'] == tool.to_argv()
def test_base_command_as_file_input(client):
"""Test base command detection when it is a script file."""
cwd = Path(client.path)
script = cwd / 'script.py'
script.touch()
input_file = cwd / 'input.csv'
input_file.touch()
client.repo.index.add([str(script), str(input_file)])
client.repo.index.commit('add file')
argv = ['script.py', 'input.csv']
tool = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
).generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert not tool.command
assert 2 == len(tool.inputs)
def test_base_command_detection(instance_path):
"""Test base command detection."""
hello = Path(instance_path) / 'hello.tar'
hello.touch()
argv = ['tar', 'xf', 'hello.tar']
tool = CommandLineToolFactory(argv,
directory=instance_path,
working_dir=instance_path).generate_tool()
assert tool.baseCommand == ['tar', 'xf']
assert tool.inputs[0].default.path.samefile(hello)
assert tool.inputs[0].type == 'File'
assert tool.inputs[0].inputBinding.prefix is None
assert tool.inputs[0].inputBinding.separate is True
assert tool.to_argv() == argv
def test_05_stdout(instance_path):
"""Test stdout mapping."""
output = Path(instance_path) / 'output.txt'
output.touch()
argv = ['echo', 'Hello world!']
factory = CommandLineToolFactory(
argv,
directory=instance_path,
working_dir=instance_path,
stdout='output.txt',
)
assert factory.stdout == 'output.txt'
assert factory.outputs[0].type == 'stdout'
tool = factory.generate_tool()
assert tool.to_argv() == argv
def test_1st_tool(client):
"""Check creation of 1st tool example from args."""
tool = CommandLineToolFactory(
('echo',
'Hello world!')).generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert 'Hello world!' == tool.arguments[0].value
def test_existing_output_directory(client, runner, project):
"""Test creation of InitialWorkDirRequirement for output."""
from renku.core.models.workflow.converters.cwl import CWLConverter
client.path = client.path
output = client.path / 'output'
argv = ['script', 'output']
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
)
with factory.watch(client, no_output=True) as tool:
# Script creates the directory.
output.mkdir(parents=True)
run = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
cwl, _ = CWLConverter.convert(run.association.plan, client)
assert 0 == len([r for r in cwl.requirements if hasattr(r, 'listing')])
output.mkdir(parents=True, exist_ok=True)
with factory.watch(client) as tool:
# The directory already exists.
(output / 'result.txt').touch()
assert 1 == len(tool.inputs)
run = tool.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
cwl, _ = CWLConverter.convert(run.association.plan, client)
reqs = [r for r in cwl.requirements if hasattr(r, 'listing')]
assert 1 == len(reqs)
assert output.name == reqs[0].listing[0].entryname
assert 1 == len(tool.outputs)
def test_stdout_with_conflicting_arg(instance_path):
"""Test stdout with conflicting argument value."""
output = Path(instance_path) / 'lalala'
output.touch()
argv = ['echo', 'lalala']
factory = CommandLineToolFactory(
argv,
directory=instance_path,
working_dir=instance_path,
stdout='lalala',
)
assert factory.inputs[0].default == 'lalala'
assert factory.inputs[0].type == 'string'
assert factory.stdout == 'lalala'
assert factory.outputs[0].type == 'stdout'
tool = factory.generate_tool()
assert tool.to_argv() == argv
def run(client, inputs, outputs, no_output, success_codes, isolation,
command_line):
"""Tracking work on a specific problem."""
working_dir = client.repo.working_dir
mapped_std = _mapped_std_streams(client.candidate_paths)
factory = CommandLineToolFactory(command_line=command_line,
explicit_inputs=inputs,
explicit_outputs=outputs,
directory=os.getcwd(),
working_dir=working_dir,
successCodes=success_codes,
**{
name:
os.path.relpath(path, working_dir)
for name, path in mapped_std.items()
})
with client.with_workflow_storage() as wf:
with factory.watch(client, no_output=no_output) as tool:
# Don't compute paths if storage is disabled.
if client.has_external_storage:
# Make sure all inputs are pulled from a storage.
paths_ = (
path
for _, path in tool.iter_input_files(client.workflow_path))
client.pull_paths_from_storage(*paths_)
return_code = call(
factory.command_line,
cwd=os.getcwd(),
**{key: getattr(sys, key)
for key in mapped_std.keys()},
)
if return_code not in (success_codes or {0}):
raise errors.InvalidSuccessCode(return_code,
success_codes=success_codes)
sys.stdout.flush()
sys.stderr.flush()
wf.add_step(run=tool)
def test_stdin_and_stdout(argv, client):
"""Test stdout mapping."""
input_ = Path(client.path) / "input.txt"
input_.touch()
output = Path(client.path) / "output.txt"
output.touch()
error = Path(client.path) / "error.log"
error.touch()
client.repo.index.add([str(input_), str(output), str(error)])
client.repo.index.commit("add files")
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
stdin="input.txt",
stdout="output.txt",
stderr="error.log",
)
assert factory.stdin
if len(argv) > 1:
assert factory.arguments
assert "output.txt" == factory.stdout
assert "stdout" == factory.outputs[0].type
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path="dummy.yaml")
tool = tool.association.plan
assert argv == tool.to_argv()
assert any(i.mapped_to and i.mapped_to.stream_type == "stdin"
for i in tool.inputs)
assert any(o.mapped_to and o.mapped_to.stream_type == "stdout"
for o in tool.outputs)
assert any(o.mapped_to and o.mapped_to.stream_type == "stderr"
for o in tool.outputs)
def test_stdin_and_stdout(argv, client):
"""Test stdout mapping."""
input_ = Path(client.path) / 'input.txt'
input_.touch()
output = Path(client.path) / 'output.txt'
output.touch()
error = Path(client.path) / 'error.log'
error.touch()
client.repo.index.add([str(input_), str(output), str(error)])
client.repo.index.commit('add files')
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
stdin='input.txt',
stdout='output.txt',
stderr='error.log',
)
assert factory.stdin
if len(argv) > 1:
assert factory.arguments
assert 'output.txt' == factory.stdout
assert 'stdout' == factory.outputs[0].type
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert argv == tool.to_argv()
assert any(i.mapped_to and i.mapped_to.stream_type == 'stdin'
for i in tool.inputs)
assert any(o.mapped_to and o.mapped_to.stream_type == 'stdout'
for o in tool.outputs)
assert any(o.mapped_to and o.mapped_to.stream_type == 'stderr'
for o in tool.outputs)
def test_04_output(instance_path):
"""Test describtion of outputs from a command."""
hello = Path(instance_path) / 'hello.tar'
hello.touch()
argv = ['tar', 'xf', 'hello.tar']
factory = CommandLineToolFactory(argv,
directory=instance_path,
working_dir=instance_path)
# simulate run
output = Path(instance_path) / 'hello.txt'
output.touch()
parameters = list(factory.guess_outputs([output]))
assert parameters[0][0].type == 'File'
assert parameters[0][0].outputBinding.glob == 'hello.txt'
tool = factory.generate_tool()
assert tool.to_argv() == argv
def test_base_command_detection(client):
"""Test base command detection."""
hello = Path(client.path) / 'hello.tar'
hello.touch()
client.repo.index.add([str(hello)])
client.repo.index.commit('add hello.tar')
argv = ['tar', 'xf', 'hello.tar']
tool = CommandLineToolFactory(
argv, directory=client.path,
working_dir=client.path).generate_process_run(
client=client, commit=client.repo.head.commit, path='dummy.yaml')
tool = tool.association.plan
assert 'tar xf' == tool.command
assert tool.inputs[0].consumes.path == 'hello.tar'
assert isinstance(tool.inputs[0].consumes, Entity)
assert not isinstance(tool.inputs[0].consumes, Collection)
assert tool.inputs[0].prefix is None
assert argv == tool.to_argv()
def test_06_params(client):
"""Test referencing input parameters in other fields."""
hello = Path(client.path) / "hello.tar"
hello.touch()
client.repo.index.add([str(hello)])
client.repo.index.commit("add hello.tar")
argv = ["tar", "xf", "hello.tar", "goodbye.txt"]
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
)
assert "goodbye.txt" == factory.inputs[1].default
assert "string" == factory.inputs[1].type
assert 2 == factory.inputs[1].inputBinding.position
goodbye_id = factory.inputs[1].id
# simulate run
output = Path(client.path) / "goodbye.txt"
output.touch()
parameters = list(factory.guess_outputs([output]))
assert "File" == parameters[0][0].type
assert "$(inputs.{0})".format(
goodbye_id) == parameters[0][0].outputBinding.glob
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path="dummy.yaml")
tool = tool.association.plan
assert argv == tool.to_argv()
def test_03_input(client):
"""Check the essential input parameters."""
whale = Path(client.path) / "whale.txt"
whale.touch()
client.repo.index.add([str(whale)])
client.repo.index.commit("add whale.txt")
argv = [
"echo",
"-f",
"-i42",
"--example-string",
"hello",
"--file=whale.txt",
]
tool = CommandLineToolFactory(
argv, directory=client.path,
working_dir=client.path).generate_process_run(
client=client, commit=client.repo.head.commit, path="dummy.yaml")
tool = tool.association.plan
assert ["-f"] == tool.arguments[0].to_argv()
assert 42 == tool.arguments[1].value
assert "-i" == tool.arguments[1].prefix
assert "hello" == tool.arguments[2].value
assert "--example-string " == tool.arguments[2].prefix
assert tool.inputs[0].consumes.path == "whale.txt"
assert isinstance(tool.inputs[0].consumes, Entity)
assert not isinstance(tool.inputs[0].consumes, Collection)
assert "--file=" == tool.inputs[0].prefix
assert argv == tool.to_argv()
def test_06_params(client):
"""Test referencing input parameters in other fields."""
hello = Path(client.path) / 'hello.tar'
hello.touch()
client.repo.index.add([str(hello)])
client.repo.index.commit('add hello.tar')
argv = ['tar', 'xf', 'hello.tar', 'goodbye.txt']
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
)
assert 'goodbye.txt' == factory.inputs[1].default
assert 'string' == factory.inputs[1].type
assert 2 == factory.inputs[1].inputBinding.position
goodbye_id = factory.inputs[1].id
# simulate run
output = Path(client.path) / 'goodbye.txt'
output.touch()
parameters = list(factory.guess_outputs([output]))
assert 'File' == parameters[0][0].type
assert '$(inputs.{0})'.format(goodbye_id) == \
parameters[0][0].outputBinding.glob
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert argv == tool.to_argv()
def test_03_input(client):
"""Check the essential input parameters."""
whale = Path(client.path) / 'whale.txt'
whale.touch()
client.repo.index.add([str(whale)])
client.repo.index.commit('add whale.txt')
argv = [
'echo',
'-f',
'-i42',
'--example-string',
'hello',
'--file=whale.txt',
]
tool = CommandLineToolFactory(
argv, directory=client.path,
working_dir=client.path).generate_process_run(
client=client, commit=client.repo.head.commit, path='dummy.yaml')
tool = tool.association.plan
assert ['-f'] == tool.arguments[0].to_argv()
assert 42 == tool.arguments[1].value
assert '-i' == tool.arguments[1].prefix
assert 'hello' == tool.arguments[2].value
assert '--example-string ' == tool.arguments[2].prefix
assert tool.inputs[0].consumes.path == 'whale.txt'
assert isinstance(tool.inputs[0].consumes, Entity)
assert not isinstance(tool.inputs[0].consumes, Collection)
assert '--file=' == tool.inputs[0].prefix
assert argv == tool.to_argv()
def test_exitings_output_directory(client):
"""Test creation of InitialWorkDirRequirement for output directory."""
instance_path = client.path
output = client.path / 'output'
argv = ['script', 'output']
factory = CommandLineToolFactory(
argv,
directory=instance_path,
working_dir=instance_path,
)
with factory.watch(client, no_output=True) as tool:
# Script creates the directory.
output.mkdir(parents=True)
initial_work_dir_requirement = [
r for r in tool.requirements
if r.__class__.__name__ == 'InitialWorkDirRequirement'
]
assert 0 == len(initial_work_dir_requirement)
output.mkdir(parents=True, exist_ok=True)
with factory.watch(client) as tool:
# The directory already exists.
(output / 'result.txt').touch()
initial_work_dir_requirement = [
r for r in tool.requirements
if r.__class__.__name__ == 'InitialWorkDirRequirement'
]
assert 1 == len(initial_work_dir_requirement)
assert initial_work_dir_requirement[0].listing[0].entryname == output.name
assert 1 == len(tool.inputs)
assert 1 == len(tool.outputs)
def test_09_array_inputs(client):
"""Test specification of input parameters in arrays."""
argv = [
'echo',
'-A',
'one',
'two',
'three',
'-B=four',
'-B=five',
'-B=six',
'-C=seven,eight,nine',
]
tool = CommandLineToolFactory(
argv, directory=client.path,
working_dir=client.path).generate_process_run(
client=client, commit=client.repo.head.commit, path='dummy.yaml')
tool = tool.association.plan
assert 'seven,eight,nine' == tool.arguments[-1].value
assert '-C=' == tool.arguments[-1].prefix
assert argv == tool.to_argv()
def test_input_directory(client):
"""Test input directory."""
cwd = Path(client.path)
src = cwd / 'src'
src.mkdir(parents=True)
for i in range(5):
(src / str(i)).touch()
src_tar = cwd / 'src.tar'
src_tar.touch()
client.repo.index.add([str(src), str(src_tar)])
client.repo.index.commit('add file and folder')
argv = ['tar', 'czvf', 'src.tar', 'src']
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
)
tool = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
tool = tool.association.plan
assert argv == tool.to_argv()
inputs = sorted(tool.inputs, key=lambda x: x.position)
assert src_tar.name == inputs[0].consumes.path
assert isinstance(inputs[0].consumes, Entity)
assert not isinstance(inputs[0].consumes, Collection)
assert inputs[1].consumes.path == src.name
assert isinstance(inputs[1].consumes, Collection)