def test_existing_output_directory(client, runner, project):
"""Test creation of InitialWorkDirRequirement for output."""
from renku.core.models.workflow.converters.cwl import CWLConverter
client.path = client.path
output = client.path / 'output'
argv = ['script', 'output']
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
)
with factory.watch(client, no_output=True) as tool:
# Script creates the directory.
output.mkdir(parents=True)
run = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
cwl, _ = CWLConverter.convert(run.association.plan, client)
assert 0 == len([r for r in cwl.requirements if hasattr(r, 'listing')])
output.mkdir(parents=True, exist_ok=True)
with factory.watch(client) as tool:
# The directory already exists.
(output / 'result.txt').touch()
assert 1 == len(tool.inputs)
run = tool.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
cwl, _ = CWLConverter.convert(run.association.plan, client)
reqs = [r for r in cwl.requirements if hasattr(r, 'listing')]
assert 1 == len(reqs)
assert output.name == reqs[0].listing[0].entryname
assert 1 == len(tool.outputs)
def test_exitings_output_directory(client):
"""Test creation of InitialWorkDirRequirement for output directory."""
instance_path = client.path
output = client.path / 'output'
argv = ['script', 'output']
factory = CommandLineToolFactory(
argv,
directory=instance_path,
working_dir=instance_path,
)
with factory.watch(client, no_output=True) as tool:
# Script creates the directory.
output.mkdir(parents=True)
initial_work_dir_requirement = [
r for r in tool.requirements
if r.__class__.__name__ == 'InitialWorkDirRequirement'
]
assert 0 == len(initial_work_dir_requirement)
output.mkdir(parents=True, exist_ok=True)
with factory.watch(client) as tool:
# The directory already exists.
(output / 'result.txt').touch()
initial_work_dir_requirement = [
r for r in tool.requirements
if r.__class__.__name__ == 'InitialWorkDirRequirement'
]
assert 1 == len(initial_work_dir_requirement)
assert initial_work_dir_requirement[0].listing[0].entryname == output.name
assert 1 == len(tool.inputs)
assert 1 == len(tool.outputs)
def run(client, inputs, outputs, no_output, success_codes, isolation,
command_line):
"""Tracking work on a specific problem."""
working_dir = client.repo.working_dir
mapped_std = _mapped_std_streams(client.candidate_paths)
factory = CommandLineToolFactory(command_line=command_line,
explicit_inputs=inputs,
explicit_outputs=outputs,
directory=os.getcwd(),
working_dir=working_dir,
successCodes=success_codes,
**{
name:
os.path.relpath(path, working_dir)
for name, path in mapped_std.items()
})
with client.with_workflow_storage() as wf:
with factory.watch(client, no_output=no_output) as tool:
# Don't compute paths if storage is disabled.
if client.has_external_storage:
# Make sure all inputs are pulled from a storage.
paths_ = (
path
for _, path in tool.iter_input_files(client.workflow_path))
client.pull_paths_from_storage(*paths_)
return_code = call(
factory.command_line,
cwd=os.getcwd(),
**{key: getattr(sys, key)
for key in mapped_std.keys()},
)
if return_code not in (success_codes or {0}):
raise errors.InvalidSuccessCode(return_code,
success_codes=success_codes)
sys.stdout.flush()
sys.stderr.flush()
wf.add_step(run=tool)
def run(
client,
explicit_inputs,
explicit_outputs,
no_output,
no_input_detection,
no_output_detection,
success_codes,
isolation,
command_line,
):
"""Tracking work on a specific problem."""
paths = explicit_outputs if no_output_detection else client.candidate_paths
mapped_std = get_mapped_std_streams(paths, streams=("stdout", "stderr"))
paths = explicit_inputs if no_input_detection else client.candidate_paths
mapped_std_in = get_mapped_std_streams(paths, streams=("stdin", ))
mapped_std.update(mapped_std_in)
invalid = get_mapped_std_streams(explicit_inputs,
streams=("stdout", "stderr"))
if invalid:
raise errors.UsageError(
"Explicit input file cannot be used as stdout/stderr:"
"\n\t" + click.style("\n\t".join(invalid.values()), fg="yellow") +
"\n")
invalid = get_mapped_std_streams(explicit_outputs, streams=("stdin", ))
if invalid:
raise errors.UsageError(
"Explicit output file cannot be used as stdin:"
"\n\t" + click.style("\n\t".join(invalid.values()), fg="yellow") +
"\n")
system_stdout = None
system_stderr = None
# /dev/tty is a virtual device that points to the terminal
# of the currently executed process
try:
with open("/dev/tty", "w"):
tty_exists = True
except OSError:
tty_exists = False
try:
stdout_redirected = "stdout" in mapped_std
stderr_redirected = "stderr" in mapped_std
if tty_exists:
# if renku was called with redirected stdout/stderr, undo the
# redirection here so error messages can be printed normally
if stdout_redirected:
system_stdout = open("/dev/tty", "w")
old_stdout = sys.stdout
sys.stdout = system_stdout
if stderr_redirected:
system_stderr = open("/dev/tty", "w")
old_stderr = sys.stderr
sys.stderr = system_stderr
working_dir = client.repo.working_dir
factory = CommandLineToolFactory(
command_line=command_line,
explicit_inputs=explicit_inputs,
explicit_outputs=explicit_outputs,
directory=os.getcwd(),
working_dir=working_dir,
no_input_detection=no_input_detection,
no_output_detection=no_output_detection,
successCodes=success_codes,
**{
name: os.path.relpath(path, working_dir)
for name, path in mapped_std.items()
},
)
with client.with_workflow_storage() as wf:
with factory.watch(client, no_output=no_output) as tool:
# Don't compute paths if storage is disabled.
if client.check_external_storage():
# Make sure all inputs are pulled from a storage.
paths_ = (path for _, path in tool.iter_input_files(
client.workflow_path))
client.pull_paths_from_storage(*paths_)
if tty_exists:
# apply original output redirection
if stdout_redirected:
sys.stdout = old_stdout
if stderr_redirected:
sys.stderr = old_stderr
return_code = call(
factory.command_line,
cwd=os.getcwd(),
**{key: getattr(sys, key)
for key in mapped_std.keys()},
)
sys.stdout.flush()
sys.stderr.flush()
if tty_exists:
# change back to /dev/tty redirection
if stdout_redirected:
sys.stdout = system_stdout
if stderr_redirected:
sys.stderr = system_stderr
if return_code not in (success_codes or {0}):
raise errors.InvalidSuccessCode(
return_code, success_codes=success_codes)
wf.add_step(run=tool)
if factory.messages:
click.echo(factory.messages)
if factory.warnings:
click.echo(factory.warnings)
finally:
if system_stdout:
sys.stdout = old_stdout
system_stdout.close()
if system_stderr:
sys.stderr = old_stderr
system_stderr.close()
def run(client, inputs, outputs, no_output, success_codes, isolation,
command_line):
"""Tracking work on a specific problem."""
mapped_std = _mapped_std_streams(client.candidate_paths)
system_stdout = None
system_stderr = None
# /dev/tty is a virtual device that points to the terminal
# of the currently executed process
try:
with open('/dev/tty', 'w'):
tty_exists = True
except OSError:
tty_exists = False
try:
stdout_redirected = 'stdout' in mapped_std
stderr_redirected = 'stderr' in mapped_std
if tty_exists:
# if renku was called with redirected stdout/stderr, undo the
# redirection here so error messages can be printed normally
if stdout_redirected:
system_stdout = open('/dev/tty', 'w')
old_stdout = sys.stdout
sys.stdout = system_stdout
if stderr_redirected:
system_stderr = open('/dev/tty', 'w')
old_stderr = sys.stderr
sys.stderr = system_stderr
working_dir = client.repo.working_dir
factory = CommandLineToolFactory(
command_line=command_line,
explicit_inputs=inputs,
explicit_outputs=outputs,
directory=os.getcwd(),
working_dir=working_dir,
successCodes=success_codes,
**{
name: os.path.relpath(path, working_dir)
for name, path in mapped_std.items()
})
with client.with_workflow_storage() as wf:
with factory.watch(client, no_output=no_output) as tool:
# Don't compute paths if storage is disabled.
if client.check_external_storage():
# Make sure all inputs are pulled from a storage.
paths_ = (path for _, path in tool.iter_input_files(
client.workflow_path))
client.pull_paths_from_storage(*paths_)
if tty_exists:
# apply original output redirection
if stdout_redirected:
sys.stdout = old_stdout
if stderr_redirected:
sys.stderr = old_stderr
return_code = call(
factory.command_line,
cwd=os.getcwd(),
**{key: getattr(sys, key)
for key in mapped_std.keys()},
)
sys.stdout.flush()
sys.stderr.flush()
if tty_exists:
# change back to /dev/tty redirection
if stdout_redirected:
sys.stdout = system_stdout
if stderr_redirected:
sys.stderr = system_stderr
if return_code not in (success_codes or {0}):
raise errors.InvalidSuccessCode(
return_code, success_codes=success_codes)
wf.add_step(run=tool)
if factory.messages:
click.echo(factory.messages)
if factory.warnings:
click.echo(factory.warnings)
finally:
if system_stdout:
sys.stdout = old_stdout
system_stdout.close()
if system_stderr:
sys.stderr = old_stderr
system_stderr.close()
