def test_stdout_with_conflicting_arg(client): """Test stdout with conflicting argument value.""" output = Path(client.path) / "lalala" output.touch() client.repo.index.add([str(output)]) client.repo.index.commit("add lalala") argv = ["echo", "lalala"] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, stdout="lalala", ) assert "lalala" == factory.inputs[0].default assert "string" == factory.inputs[0].type assert "lalala" == factory.stdout assert "stdout" == factory.outputs[0].type tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path="dummy.yaml") tool = tool.association.plan assert argv == tool.to_argv()
def test_06_params(instance_path): """Test referencing input parameters in other fields.""" hello = Path(instance_path) / 'hello.tar' hello.touch() argv = ['tar', 'xf', 'hello.tar', 'goodbye.txt'] factory = CommandLineToolFactory( argv, directory=instance_path, working_dir=instance_path, ) assert factory.inputs[1].default == 'goodbye.txt' assert factory.inputs[1].type == 'string' assert factory.inputs[1].inputBinding.position == 2 goodbye_id = factory.inputs[1].id # simulate run output = Path(instance_path) / 'goodbye.txt' output.touch() parameters = list(factory.guess_outputs([output])) assert parameters[0][0].type == 'File' assert parameters[0][0].outputBinding.glob == \ '$(inputs.{0})'.format(goodbye_id) tool = factory.generate_tool() assert tool.to_argv() == argv
def test_stdout_with_conflicting_arg(client): """Test stdout with conflicting argument value.""" output = Path(client.path) / 'lalala' output.touch() client.repo.index.add([str(output)]) client.repo.index.commit('add lalala') argv = ['echo', 'lalala'] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, stdout='lalala', ) assert 'lalala' == factory.inputs[0].default assert 'string' == factory.inputs[0].type assert 'lalala' == factory.stdout assert 'stdout' == factory.outputs[0].type tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert argv == tool.to_argv()
def test_09_array_inputs(instance_path): """Test specification of input parameters in arrays.""" argv = [ 'echo', '-A', 'one', 'two', 'three', '-B=four', '-B=five', '-B=six', '-C=seven,eight,nine', ] tool = CommandLineToolFactory(argv, directory=instance_path, working_dir=instance_path).generate_tool() # TODO add grouping for -A and -B assert tool.inputs[-1].type == 'string[]' assert tool.inputs[-1].default == ['seven', 'eight', 'nine'] assert tool.inputs[-1].inputBinding.prefix == '-C=' assert tool.inputs[-1].inputBinding.itemSeparator == ',' assert tool.inputs[-1].inputBinding.separate is False assert tool.to_argv() == argv
def test_stdin_and_stdout(argv, instance_path): """Test stdout mapping.""" input_ = Path(instance_path) / 'input.txt' input_.touch() output = Path(instance_path) / 'output.txt' output.touch() factory = CommandLineToolFactory( argv, directory=instance_path, working_dir=instance_path, stdin='input.txt', stdout='output.txt', stderr='error.log', ) assert factory.stdin if len(argv) > 1: assert factory.arguments assert factory.stdout == 'output.txt' assert factory.outputs[0].type == 'stdout' tool = factory.generate_tool() assert tool.to_argv() == argv std_streams = (' < input.txt', ' > output.txt', ' 2> error.log') tool_str = str(tool) assert tool_str.startswith(' '.join(argv)) assert all(stream in tool_str for stream in std_streams)
def test_04_output(client): """Test describtion of outputs from a command.""" hello = Path(client.path) / 'hello.tar' hello.touch() client.repo.index.add([str(hello)]) client.repo.index.commit('add hello.tar') argv = ['tar', 'xf', 'hello.tar'] factory = CommandLineToolFactory(argv, directory=client.path, working_dir=client.path) # simulate run output = Path(client.path) / 'hello.txt' output.touch() parameters = list(factory.guess_outputs([output])) assert 'File' == parameters[0][0].type assert 'hello.txt' == parameters[0][0].outputBinding.glob tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert argv == tool.to_argv()
def test_05_stdout(client): """Test stdout mapping.""" output = Path(client.path) / 'output.txt' output.touch() client.repo.index.add([str(output)]) client.repo.index.commit('add output') argv = ['echo', 'Hello world!'] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, stdout='output.txt', ) assert 'output.txt' == factory.stdout assert 'stdout' == factory.outputs[0].type tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert argv == tool.to_argv()
def test_input_directory(instance_path): """Test input directory.""" cwd = Path(instance_path) src = cwd / 'src' src.mkdir(parents=True) for i in range(5): (src / str(i)).touch() argv = ['tar', 'czvf', 'src.tar', 'src'] factory = CommandLineToolFactory( argv, directory=instance_path, working_dir=instance_path, ) src_tar = src / 'src.tar' src_tar.touch() tool = factory.generate_tool() assert tool.to_argv() == argv assert tool.inputs[0].type == 'string' assert tool.inputs[0].default == src_tar.name assert tool.inputs[1].type == 'Directory' assert tool.inputs[1].default.path.samefile(src)
def test_03_input(instance_path): """Check the essential input parameters.""" whale = Path(instance_path) / 'whale.txt' whale.touch() argv = [ 'echo', '-f', '-i42', '--example-string', 'hello', '--file=whale.txt', ] tool = CommandLineToolFactory(argv, directory=instance_path, working_dir=instance_path).generate_tool() assert tool.arguments[0].to_argv() == ['-f'] assert tool.inputs[0].default == 42 assert tool.inputs[0].type == 'int' assert tool.inputs[0].inputBinding.prefix == '-i' assert tool.inputs[0].inputBinding.separate is False assert tool.inputs[1].default == 'hello' assert tool.inputs[1].type == 'string' assert tool.inputs[1].inputBinding.prefix == '--example-string' assert tool.inputs[1].inputBinding.separate is True assert tool.inputs[2].default.path.samefile(whale) assert tool.inputs[2].type == 'File' assert tool.inputs[2].inputBinding.prefix == '--file=' assert tool.inputs[2].inputBinding.separate is False assert tool.to_argv() == argv
def test_05_stdout(client): """Test stdout mapping.""" output = Path(client.path) / "output.txt" output.touch() client.repo.index.add([str(output)]) client.repo.index.commit("add output") argv = ["echo", "Hello world!"] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, stdout="output.txt", ) assert "output.txt" == factory.stdout assert "stdout" == factory.outputs[0].type tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path="dummy.yaml") tool = tool.association.plan assert argv == tool.to_argv()
def test_short_base_command_detection(): """Test base command detection without arguments.""" tool = CommandLineToolFactory(('echo', 'A')).generate_tool() assert tool.cwlVersion == 'v1.0' assert tool.__class__.__name__ == 'CommandLineTool' assert tool.inputs[0].default == 'A' assert tool.to_argv() == ['echo', 'A']
def test_short_base_command_detection(client): """Test base command detection without arguments.""" tool = CommandLineToolFactory( ('echo', 'A')).generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert 'A' == tool.arguments[0].value assert ['echo', 'A'] == tool.to_argv()
def test_base_command_as_file_input(client): """Test base command detection when it is a script file.""" cwd = Path(client.path) script = cwd / 'script.py' script.touch() input_file = cwd / 'input.csv' input_file.touch() client.repo.index.add([str(script), str(input_file)]) client.repo.index.commit('add file') argv = ['script.py', 'input.csv'] tool = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, ).generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert not tool.command assert 2 == len(tool.inputs)
def test_base_command_detection(instance_path): """Test base command detection.""" hello = Path(instance_path) / 'hello.tar' hello.touch() argv = ['tar', 'xf', 'hello.tar'] tool = CommandLineToolFactory(argv, directory=instance_path, working_dir=instance_path).generate_tool() assert tool.baseCommand == ['tar', 'xf'] assert tool.inputs[0].default.path.samefile(hello) assert tool.inputs[0].type == 'File' assert tool.inputs[0].inputBinding.prefix is None assert tool.inputs[0].inputBinding.separate is True assert tool.to_argv() == argv
def test_05_stdout(instance_path): """Test stdout mapping.""" output = Path(instance_path) / 'output.txt' output.touch() argv = ['echo', 'Hello world!'] factory = CommandLineToolFactory( argv, directory=instance_path, working_dir=instance_path, stdout='output.txt', ) assert factory.stdout == 'output.txt' assert factory.outputs[0].type == 'stdout' tool = factory.generate_tool() assert tool.to_argv() == argv
def test_1st_tool(client): """Check creation of 1st tool example from args.""" tool = CommandLineToolFactory( ('echo', 'Hello world!')).generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert 'Hello world!' == tool.arguments[0].value
def test_existing_output_directory(client, runner, project): """Test creation of InitialWorkDirRequirement for output.""" from renku.core.models.workflow.converters.cwl import CWLConverter client.path = client.path output = client.path / 'output' argv = ['script', 'output'] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, ) with factory.watch(client, no_output=True) as tool: # Script creates the directory. output.mkdir(parents=True) run = factory.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') cwl, _ = CWLConverter.convert(run.association.plan, client) assert 0 == len([r for r in cwl.requirements if hasattr(r, 'listing')]) output.mkdir(parents=True, exist_ok=True) with factory.watch(client) as tool: # The directory already exists. (output / 'result.txt').touch() assert 1 == len(tool.inputs) run = tool.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') cwl, _ = CWLConverter.convert(run.association.plan, client) reqs = [r for r in cwl.requirements if hasattr(r, 'listing')] assert 1 == len(reqs) assert output.name == reqs[0].listing[0].entryname assert 1 == len(tool.outputs)
def test_stdout_with_conflicting_arg(instance_path): """Test stdout with conflicting argument value.""" output = Path(instance_path) / 'lalala' output.touch() argv = ['echo', 'lalala'] factory = CommandLineToolFactory( argv, directory=instance_path, working_dir=instance_path, stdout='lalala', ) assert factory.inputs[0].default == 'lalala' assert factory.inputs[0].type == 'string' assert factory.stdout == 'lalala' assert factory.outputs[0].type == 'stdout' tool = factory.generate_tool() assert tool.to_argv() == argv
def run(client, inputs, outputs, no_output, success_codes, isolation, command_line): """Tracking work on a specific problem.""" working_dir = client.repo.working_dir mapped_std = _mapped_std_streams(client.candidate_paths) factory = CommandLineToolFactory(command_line=command_line, explicit_inputs=inputs, explicit_outputs=outputs, directory=os.getcwd(), working_dir=working_dir, successCodes=success_codes, **{ name: os.path.relpath(path, working_dir) for name, path in mapped_std.items() }) with client.with_workflow_storage() as wf: with factory.watch(client, no_output=no_output) as tool: # Don't compute paths if storage is disabled. if client.has_external_storage: # Make sure all inputs are pulled from a storage. paths_ = ( path for _, path in tool.iter_input_files(client.workflow_path)) client.pull_paths_from_storage(*paths_) return_code = call( factory.command_line, cwd=os.getcwd(), **{key: getattr(sys, key) for key in mapped_std.keys()}, ) if return_code not in (success_codes or {0}): raise errors.InvalidSuccessCode(return_code, success_codes=success_codes) sys.stdout.flush() sys.stderr.flush() wf.add_step(run=tool)
def test_stdin_and_stdout(argv, client): """Test stdout mapping.""" input_ = Path(client.path) / "input.txt" input_.touch() output = Path(client.path) / "output.txt" output.touch() error = Path(client.path) / "error.log" error.touch() client.repo.index.add([str(input_), str(output), str(error)]) client.repo.index.commit("add files") factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, stdin="input.txt", stdout="output.txt", stderr="error.log", ) assert factory.stdin if len(argv) > 1: assert factory.arguments assert "output.txt" == factory.stdout assert "stdout" == factory.outputs[0].type tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path="dummy.yaml") tool = tool.association.plan assert argv == tool.to_argv() assert any(i.mapped_to and i.mapped_to.stream_type == "stdin" for i in tool.inputs) assert any(o.mapped_to and o.mapped_to.stream_type == "stdout" for o in tool.outputs) assert any(o.mapped_to and o.mapped_to.stream_type == "stderr" for o in tool.outputs)
def test_stdin_and_stdout(argv, client): """Test stdout mapping.""" input_ = Path(client.path) / 'input.txt' input_.touch() output = Path(client.path) / 'output.txt' output.touch() error = Path(client.path) / 'error.log' error.touch() client.repo.index.add([str(input_), str(output), str(error)]) client.repo.index.commit('add files') factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, stdin='input.txt', stdout='output.txt', stderr='error.log', ) assert factory.stdin if len(argv) > 1: assert factory.arguments assert 'output.txt' == factory.stdout assert 'stdout' == factory.outputs[0].type tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert argv == tool.to_argv() assert any(i.mapped_to and i.mapped_to.stream_type == 'stdin' for i in tool.inputs) assert any(o.mapped_to and o.mapped_to.stream_type == 'stdout' for o in tool.outputs) assert any(o.mapped_to and o.mapped_to.stream_type == 'stderr' for o in tool.outputs)
def test_04_output(instance_path): """Test describtion of outputs from a command.""" hello = Path(instance_path) / 'hello.tar' hello.touch() argv = ['tar', 'xf', 'hello.tar'] factory = CommandLineToolFactory(argv, directory=instance_path, working_dir=instance_path) # simulate run output = Path(instance_path) / 'hello.txt' output.touch() parameters = list(factory.guess_outputs([output])) assert parameters[0][0].type == 'File' assert parameters[0][0].outputBinding.glob == 'hello.txt' tool = factory.generate_tool() assert tool.to_argv() == argv
def test_base_command_detection(client): """Test base command detection.""" hello = Path(client.path) / 'hello.tar' hello.touch() client.repo.index.add([str(hello)]) client.repo.index.commit('add hello.tar') argv = ['tar', 'xf', 'hello.tar'] tool = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path).generate_process_run( client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert 'tar xf' == tool.command assert tool.inputs[0].consumes.path == 'hello.tar' assert isinstance(tool.inputs[0].consumes, Entity) assert not isinstance(tool.inputs[0].consumes, Collection) assert tool.inputs[0].prefix is None assert argv == tool.to_argv()
def test_06_params(client): """Test referencing input parameters in other fields.""" hello = Path(client.path) / "hello.tar" hello.touch() client.repo.index.add([str(hello)]) client.repo.index.commit("add hello.tar") argv = ["tar", "xf", "hello.tar", "goodbye.txt"] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, ) assert "goodbye.txt" == factory.inputs[1].default assert "string" == factory.inputs[1].type assert 2 == factory.inputs[1].inputBinding.position goodbye_id = factory.inputs[1].id # simulate run output = Path(client.path) / "goodbye.txt" output.touch() parameters = list(factory.guess_outputs([output])) assert "File" == parameters[0][0].type assert "$(inputs.{0})".format( goodbye_id) == parameters[0][0].outputBinding.glob tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path="dummy.yaml") tool = tool.association.plan assert argv == tool.to_argv()
def test_03_input(client): """Check the essential input parameters.""" whale = Path(client.path) / "whale.txt" whale.touch() client.repo.index.add([str(whale)]) client.repo.index.commit("add whale.txt") argv = [ "echo", "-f", "-i42", "--example-string", "hello", "--file=whale.txt", ] tool = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path).generate_process_run( client=client, commit=client.repo.head.commit, path="dummy.yaml") tool = tool.association.plan assert ["-f"] == tool.arguments[0].to_argv() assert 42 == tool.arguments[1].value assert "-i" == tool.arguments[1].prefix assert "hello" == tool.arguments[2].value assert "--example-string " == tool.arguments[2].prefix assert tool.inputs[0].consumes.path == "whale.txt" assert isinstance(tool.inputs[0].consumes, Entity) assert not isinstance(tool.inputs[0].consumes, Collection) assert "--file=" == tool.inputs[0].prefix assert argv == tool.to_argv()
def test_06_params(client): """Test referencing input parameters in other fields.""" hello = Path(client.path) / 'hello.tar' hello.touch() client.repo.index.add([str(hello)]) client.repo.index.commit('add hello.tar') argv = ['tar', 'xf', 'hello.tar', 'goodbye.txt'] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, ) assert 'goodbye.txt' == factory.inputs[1].default assert 'string' == factory.inputs[1].type assert 2 == factory.inputs[1].inputBinding.position goodbye_id = factory.inputs[1].id # simulate run output = Path(client.path) / 'goodbye.txt' output.touch() parameters = list(factory.guess_outputs([output])) assert 'File' == parameters[0][0].type assert '$(inputs.{0})'.format(goodbye_id) == \ parameters[0][0].outputBinding.glob tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert argv == tool.to_argv()
def test_03_input(client): """Check the essential input parameters.""" whale = Path(client.path) / 'whale.txt' whale.touch() client.repo.index.add([str(whale)]) client.repo.index.commit('add whale.txt') argv = [ 'echo', '-f', '-i42', '--example-string', 'hello', '--file=whale.txt', ] tool = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path).generate_process_run( client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert ['-f'] == tool.arguments[0].to_argv() assert 42 == tool.arguments[1].value assert '-i' == tool.arguments[1].prefix assert 'hello' == tool.arguments[2].value assert '--example-string ' == tool.arguments[2].prefix assert tool.inputs[0].consumes.path == 'whale.txt' assert isinstance(tool.inputs[0].consumes, Entity) assert not isinstance(tool.inputs[0].consumes, Collection) assert '--file=' == tool.inputs[0].prefix assert argv == tool.to_argv()
def test_exitings_output_directory(client): """Test creation of InitialWorkDirRequirement for output directory.""" instance_path = client.path output = client.path / 'output' argv = ['script', 'output'] factory = CommandLineToolFactory( argv, directory=instance_path, working_dir=instance_path, ) with factory.watch(client, no_output=True) as tool: # Script creates the directory. output.mkdir(parents=True) initial_work_dir_requirement = [ r for r in tool.requirements if r.__class__.__name__ == 'InitialWorkDirRequirement' ] assert 0 == len(initial_work_dir_requirement) output.mkdir(parents=True, exist_ok=True) with factory.watch(client) as tool: # The directory already exists. (output / 'result.txt').touch() initial_work_dir_requirement = [ r for r in tool.requirements if r.__class__.__name__ == 'InitialWorkDirRequirement' ] assert 1 == len(initial_work_dir_requirement) assert initial_work_dir_requirement[0].listing[0].entryname == output.name assert 1 == len(tool.inputs) assert 1 == len(tool.outputs)
def test_09_array_inputs(client): """Test specification of input parameters in arrays.""" argv = [ 'echo', '-A', 'one', 'two', 'three', '-B=four', '-B=five', '-B=six', '-C=seven,eight,nine', ] tool = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path).generate_process_run( client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert 'seven,eight,nine' == tool.arguments[-1].value assert '-C=' == tool.arguments[-1].prefix assert argv == tool.to_argv()
def test_input_directory(client): """Test input directory.""" cwd = Path(client.path) src = cwd / 'src' src.mkdir(parents=True) for i in range(5): (src / str(i)).touch() src_tar = cwd / 'src.tar' src_tar.touch() client.repo.index.add([str(src), str(src_tar)]) client.repo.index.commit('add file and folder') argv = ['tar', 'czvf', 'src.tar', 'src'] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, ) tool = factory.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') tool = tool.association.plan assert argv == tool.to_argv() inputs = sorted(tool.inputs, key=lambda x: x.position) assert src_tar.name == inputs[0].consumes.path assert isinstance(inputs[0].consumes, Entity) assert not isinstance(inputs[0].consumes, Collection) assert inputs[1].consumes.path == src.name assert isinstance(inputs[1].consumes, Collection)