def test_existing_output_directory(client, runner, project): """Test creation of InitialWorkDirRequirement for output.""" from renku.core.models.workflow.converters.cwl import CWLConverter client.path = client.path output = client.path / 'output' argv = ['script', 'output'] factory = CommandLineToolFactory( argv, directory=client.path, working_dir=client.path, ) with factory.watch(client, no_output=True) as tool: # Script creates the directory. output.mkdir(parents=True) run = factory.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') cwl, _ = CWLConverter.convert(run.association.plan, client) assert 0 == len([r for r in cwl.requirements if hasattr(r, 'listing')]) output.mkdir(parents=True, exist_ok=True) with factory.watch(client) as tool: # The directory already exists. (output / 'result.txt').touch() assert 1 == len(tool.inputs) run = tool.generate_process_run(client=client, commit=client.repo.head.commit, path='dummy.yaml') cwl, _ = CWLConverter.convert(run.association.plan, client) reqs = [r for r in cwl.requirements if hasattr(r, 'listing')] assert 1 == len(reqs) assert output.name == reqs[0].listing[0].entryname assert 1 == len(tool.outputs)
def test_exitings_output_directory(client): """Test creation of InitialWorkDirRequirement for output directory.""" instance_path = client.path output = client.path / 'output' argv = ['script', 'output'] factory = CommandLineToolFactory( argv, directory=instance_path, working_dir=instance_path, ) with factory.watch(client, no_output=True) as tool: # Script creates the directory. output.mkdir(parents=True) initial_work_dir_requirement = [ r for r in tool.requirements if r.__class__.__name__ == 'InitialWorkDirRequirement' ] assert 0 == len(initial_work_dir_requirement) output.mkdir(parents=True, exist_ok=True) with factory.watch(client) as tool: # The directory already exists. (output / 'result.txt').touch() initial_work_dir_requirement = [ r for r in tool.requirements if r.__class__.__name__ == 'InitialWorkDirRequirement' ] assert 1 == len(initial_work_dir_requirement) assert initial_work_dir_requirement[0].listing[0].entryname == output.name assert 1 == len(tool.inputs) assert 1 == len(tool.outputs)
def run(client, inputs, outputs, no_output, success_codes, isolation, command_line): """Tracking work on a specific problem.""" working_dir = client.repo.working_dir mapped_std = _mapped_std_streams(client.candidate_paths) factory = CommandLineToolFactory(command_line=command_line, explicit_inputs=inputs, explicit_outputs=outputs, directory=os.getcwd(), working_dir=working_dir, successCodes=success_codes, **{ name: os.path.relpath(path, working_dir) for name, path in mapped_std.items() }) with client.with_workflow_storage() as wf: with factory.watch(client, no_output=no_output) as tool: # Don't compute paths if storage is disabled. if client.has_external_storage: # Make sure all inputs are pulled from a storage. paths_ = ( path for _, path in tool.iter_input_files(client.workflow_path)) client.pull_paths_from_storage(*paths_) return_code = call( factory.command_line, cwd=os.getcwd(), **{key: getattr(sys, key) for key in mapped_std.keys()}, ) if return_code not in (success_codes or {0}): raise errors.InvalidSuccessCode(return_code, success_codes=success_codes) sys.stdout.flush() sys.stderr.flush() wf.add_step(run=tool)
def run( client, explicit_inputs, explicit_outputs, no_output, no_input_detection, no_output_detection, success_codes, isolation, command_line, ): """Tracking work on a specific problem.""" paths = explicit_outputs if no_output_detection else client.candidate_paths mapped_std = get_mapped_std_streams(paths, streams=("stdout", "stderr")) paths = explicit_inputs if no_input_detection else client.candidate_paths mapped_std_in = get_mapped_std_streams(paths, streams=("stdin", )) mapped_std.update(mapped_std_in) invalid = get_mapped_std_streams(explicit_inputs, streams=("stdout", "stderr")) if invalid: raise errors.UsageError( "Explicit input file cannot be used as stdout/stderr:" "\n\t" + click.style("\n\t".join(invalid.values()), fg="yellow") + "\n") invalid = get_mapped_std_streams(explicit_outputs, streams=("stdin", )) if invalid: raise errors.UsageError( "Explicit output file cannot be used as stdin:" "\n\t" + click.style("\n\t".join(invalid.values()), fg="yellow") + "\n") system_stdout = None system_stderr = None # /dev/tty is a virtual device that points to the terminal # of the currently executed process try: with open("/dev/tty", "w"): tty_exists = True except OSError: tty_exists = False try: stdout_redirected = "stdout" in mapped_std stderr_redirected = "stderr" in mapped_std if tty_exists: # if renku was called with redirected stdout/stderr, undo the # redirection here so error messages can be printed normally if stdout_redirected: system_stdout = open("/dev/tty", "w") old_stdout = sys.stdout sys.stdout = system_stdout if stderr_redirected: system_stderr = open("/dev/tty", "w") old_stderr = sys.stderr sys.stderr = system_stderr working_dir = client.repo.working_dir factory = CommandLineToolFactory( command_line=command_line, explicit_inputs=explicit_inputs, explicit_outputs=explicit_outputs, directory=os.getcwd(), working_dir=working_dir, no_input_detection=no_input_detection, no_output_detection=no_output_detection, successCodes=success_codes, **{ name: os.path.relpath(path, working_dir) for name, path in mapped_std.items() }, ) with client.with_workflow_storage() as wf: with factory.watch(client, no_output=no_output) as tool: # Don't compute paths if storage is disabled. if client.check_external_storage(): # Make sure all inputs are pulled from a storage. paths_ = (path for _, path in tool.iter_input_files( client.workflow_path)) client.pull_paths_from_storage(*paths_) if tty_exists: # apply original output redirection if stdout_redirected: sys.stdout = old_stdout if stderr_redirected: sys.stderr = old_stderr return_code = call( factory.command_line, cwd=os.getcwd(), **{key: getattr(sys, key) for key in mapped_std.keys()}, ) sys.stdout.flush() sys.stderr.flush() if tty_exists: # change back to /dev/tty redirection if stdout_redirected: sys.stdout = system_stdout if stderr_redirected: sys.stderr = system_stderr if return_code not in (success_codes or {0}): raise errors.InvalidSuccessCode( return_code, success_codes=success_codes) wf.add_step(run=tool) if factory.messages: click.echo(factory.messages) if factory.warnings: click.echo(factory.warnings) finally: if system_stdout: sys.stdout = old_stdout system_stdout.close() if system_stderr: sys.stderr = old_stderr system_stderr.close()
def run(client, inputs, outputs, no_output, success_codes, isolation, command_line): """Tracking work on a specific problem.""" mapped_std = _mapped_std_streams(client.candidate_paths) system_stdout = None system_stderr = None # /dev/tty is a virtual device that points to the terminal # of the currently executed process try: with open('/dev/tty', 'w'): tty_exists = True except OSError: tty_exists = False try: stdout_redirected = 'stdout' in mapped_std stderr_redirected = 'stderr' in mapped_std if tty_exists: # if renku was called with redirected stdout/stderr, undo the # redirection here so error messages can be printed normally if stdout_redirected: system_stdout = open('/dev/tty', 'w') old_stdout = sys.stdout sys.stdout = system_stdout if stderr_redirected: system_stderr = open('/dev/tty', 'w') old_stderr = sys.stderr sys.stderr = system_stderr working_dir = client.repo.working_dir factory = CommandLineToolFactory( command_line=command_line, explicit_inputs=inputs, explicit_outputs=outputs, directory=os.getcwd(), working_dir=working_dir, successCodes=success_codes, **{ name: os.path.relpath(path, working_dir) for name, path in mapped_std.items() }) with client.with_workflow_storage() as wf: with factory.watch(client, no_output=no_output) as tool: # Don't compute paths if storage is disabled. if client.check_external_storage(): # Make sure all inputs are pulled from a storage. paths_ = (path for _, path in tool.iter_input_files( client.workflow_path)) client.pull_paths_from_storage(*paths_) if tty_exists: # apply original output redirection if stdout_redirected: sys.stdout = old_stdout if stderr_redirected: sys.stderr = old_stderr return_code = call( factory.command_line, cwd=os.getcwd(), **{key: getattr(sys, key) for key in mapped_std.keys()}, ) sys.stdout.flush() sys.stderr.flush() if tty_exists: # change back to /dev/tty redirection if stdout_redirected: sys.stdout = system_stdout if stderr_redirected: sys.stderr = system_stderr if return_code not in (success_codes or {0}): raise errors.InvalidSuccessCode( return_code, success_codes=success_codes) wf.add_step(run=tool) if factory.messages: click.echo(factory.messages) if factory.warnings: click.echo(factory.warnings) finally: if system_stdout: sys.stdout = old_stdout system_stdout.close() if system_stderr: sys.stderr = old_stderr system_stderr.close()