def test_commandLineTool_job_tmpdir_prefix(tmp_path: Path) -> None:
"""Test that non-docker enabled CommandLineTool respects temp directory directives."""
loading_context = LoadingContext({
"metadata": {
"cwlVersion": INTERNAL_VERSION,
"http://commonwl.org/cwltool#original_cwlVersion":
INTERNAL_VERSION,
}
})
clt = CommandLineTool(
cast(
CommentedMap,
cmap({
"cwlVersion": INTERNAL_VERSION,
"class": "CommandLineTool",
"inputs": [],
"outputs": [],
"requirements": [],
}),
),
loading_context,
)
tmpdir_prefix = str(tmp_path / "1")
tmp_outdir_prefix = str(tmp_path / "2")
runtime_context = RuntimeContext({
"tmpdir_prefix": tmpdir_prefix,
"tmp_outdir_prefix": tmp_outdir_prefix,
})
job = next(clt.job({}, None, runtime_context))
assert isinstance(job, JobBase)
assert job.stagedir and job.stagedir.startswith(tmpdir_prefix)
assert job.tmpdir and job.tmpdir.startswith(tmpdir_prefix)
assert job.outdir and job.outdir.startswith(tmp_outdir_prefix)
def test_input_can_be_named_pipe(tmp_path: Path, streamable: bool,
streaming_allowed: bool,
raise_exception: bool) -> None:
"""Test that input can be a named pipe."""
clt = CommandLineTool(
toolpath_object,
loading_context,
)
runtime_context = RuntimeContext()
runtime_context.streaming_allowed = streaming_allowed
path = tmp_path / "tmp"
os.mkfifo(path)
joborder: CWLObjectType = {
"inp": {
"class": "File",
"location": str(path),
"streamable": streamable,
}
}
job = next(clt.job(joborder, None, runtime_context))
assert isinstance(job, JobBase)
if raise_exception:
with pytest.raises(WorkflowException):
job._setup(runtime_context)
else:
job._setup(runtime_context)
def test_docker_required(schema_ext11: Names) -> None:
# Docker required, MPI hinted
lc, rc, tool = mk_tool(schema_ext11, [],
reqs=[containerReq],
hints=[mpiReq])
clt = CommandLineTool(tool, lc)
jr = clt.make_job_runner(rc)
assert jr is cwltool.docker.DockerCommandLineJob
def test_regular_file() -> None:
"""Test that regular files do not raise any exception when they are checked in job._setup."""
clt = CommandLineTool(
toolpath_object,
loading_context,
)
runtime_context = RuntimeContext()
joborder: CWLObjectType = {
"inp": {
"class": "File",
"location": get_data("tests/wf/whale.txt"),
}
}
job = next(clt.job(joborder, None, runtime_context))
assert isinstance(job, JobBase)
job._setup(runtime_context)
def test_docker_commandLineTool_job_tmpdir_prefix(tmp_path: Path) -> None:
"""Test that docker enabled CommandLineTool respects temp directory directives."""
loading_context = LoadingContext({
"metadata": {
"cwlVersion": INTERNAL_VERSION,
ORIGINAL_CWLVERSION: INTERNAL_VERSION,
}
})
clt = CommandLineTool(
cast(
CommentedMap,
cmap({
"cwlVersion":
INTERNAL_VERSION,
"class":
"CommandLineTool",
"inputs": [],
"outputs": [],
"requirements": [{
"class": "DockerRequirement",
"dockerPull": "docker.io/debian:stable",
}],
}),
),
loading_context,
)
tmpdir_prefix = str(tmp_path / "1")
tmp_outdir_prefix = str(tmp_path / "2")
runtime_context = RuntimeContext({
"tmpdir_prefix": tmpdir_prefix,
"tmp_outdir_prefix": tmp_outdir_prefix,
})
job = next(clt.job({}, None, runtime_context))
assert isinstance(job, JobBase)
assert job.stagedir and job.stagedir.startswith(tmpdir_prefix)
assert job.tmpdir and job.tmpdir.startswith(tmpdir_prefix)
assert job.outdir and job.outdir.startswith(tmp_outdir_prefix)
def test_docker_mpi_both_hinted(schema_ext11: Names) -> None:
# Both hinted - error
with pytest.raises(cwltool.errors.UnsupportedRequirement):
lc, rc, tool = mk_tool(schema_ext11, [], hints=[mpiReq, containerReq])
clt = CommandLineTool(tool, lc)
jr = clt.make_job_runner(rc)
def test_udocker(schema_ext11: Names) -> None:
lc, rc, tool = mk_tool(schema_ext11, ["--udocker"], reqs=[mpiReq, containerReq])
clt = CommandLineTool(tool, lc)
jr = clt.make_job_runner(rc)
assert jr is cwltool.udocker.UDockerCommandLineJob
def test_singularity(schema_ext11: Names) -> None:
lc, rc, tool = mk_tool(schema_ext11, ["--singularity"], reqs=[mpiReq, containerReq])
clt = CommandLineTool(tool, lc)
jr = clt.make_job_runner(rc)
assert jr is cwltool.singularity.SingularityCommandLineJob
def test_anon_types(snippet: CommentedMap) -> None:
CommandLineTool(snippet, LoadingContext())
def test_clt_returns_specialchar_names(tmp_path: Path) -> None:
"""Confirm that special characters in filenames do not cause problems."""
loading_context = LoadingContext({
"metadata": {
"cwlVersion": INTERNAL_VERSION,
"http://commonwl.org/cwltool#original_cwlVersion":
INTERNAL_VERSION,
}
})
clt = CommandLineTool(
cast(
CommentedMap,
cmap({
"cwlVersion": INTERNAL_VERSION,
"class": "CommandLineTool",
"inputs": [],
"outputs": [],
"requirements": [],
}),
),
loading_context,
)
# Reserved characters will be URL encoded during the creation of a file URI
# Internal references to files are in URI form, and are therefore URL encoded
# Final output files should not retain their URL encoded filenames
rfc_3986_gen_delims = [":", "/", "?", "#", "[", "]", "@"]
rfc_3986_sub_delims = [
"!", "$", "&", "'", "(", ")", "*", "+", ",", ";", "="
]
unix_reserved = ["/", "\0"]
reserved = [
special_char
for special_char in (rfc_3986_gen_delims + rfc_3986_sub_delims)
if special_char not in unix_reserved
]
# Mock an "output" file with the above special characters in its name
special = "".join(reserved)
output_schema = cast(CWLObjectType, {
"type": "File",
"outputBinding": {
"glob": special
}
})
mock_output = tmp_path / special
mock_output.touch()
# Prepare minimal arguments for CommandLineTool.collect_output()
builder = clt._init_job({}, RuntimeContext())
builder.pathmapper = clt.make_path_mapper(builder.files, builder.stagedir,
RuntimeContext(), True)
fs_access = builder.make_fs_access(str(tmp_path))
result = cast(
CWLObjectType,
clt.collect_output(output_schema, builder, str(tmp_path), fs_access),
)
assert result["class"] == "File"
assert result["basename"] == special
assert result["nameroot"] == special
assert str(result["location"]).endswith(urllib.parse.quote(special))
# Now test when outdir is a URI, make sure it doesn't get
# incorrectly quoted as a file.
builder = clt._init_job({}, RuntimeContext())
builder.pathmapper = clt.make_path_mapper(builder.files, builder.stagedir,
RuntimeContext(), True)
builder.outdir = "/var/spool/cwl"
fs_access = TestFsAccess("")
result = cast(
CWLObjectType,
clt.collect_output(
output_schema,
builder,
"keep:ae755cd1b3cff63152ff4200f4dea7e9+52",
fs_access,
),
)
assert result["class"] == "File"
assert result["basename"] == special
assert result["nameroot"] == special
assert (
result["location"] ==
"keep:ae755cd1b3cff63152ff4200f4dea7e9+52/%3A%3F%23%5B%5D%40%21%24%26%27%28%29%2A%2B%2C%3B%3D"
)
def test_anon_types(snippet):
CommandLineTool(snippet, LoadingContext())