def run(self,
runtimeContext, # type: RuntimeContext
tmpdir_lock=None, # type: Optional[ThreadLock]
): # type: (...) -> None
make_dirs(self.tmpdir, exist_ok=True)
env = self.environment
vars_to_preserve = runtimeContext.preserve_environment
if runtimeContext.preserve_entire_environment:
vars_to_preserve = os.environ
if vars_to_preserve is not None:
for key, value in os.environ.items():
if key in vars_to_preserve and key not in env:
# On Windows, subprocess env can't handle unicode.
env[key] = str(value) if onWindows() else value
env["HOME"] = str(self.outdir) if onWindows() else self.outdir
env["TMPDIR"] = str(self.tmpdir) if onWindows() else self.tmpdir
if "PATH" not in env:
env["PATH"] = str(os.environ["PATH"]) if onWindows() else os.environ["PATH"]
if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ:
env["SYSTEMROOT"] = str(os.environ["SYSTEMROOT"]) if onWindows() else os.environ["SYSTEMROOT"]
# stageFiles(self.pathmapper, ignoreWritable=True, symLink=True, secret_store=runtimeContext.secret_store)
if self.generatemapper:
# FIXME: see if this is needed... func doesn't exist anymore in cwltool 2.x
# stageFiles(self.generatemapper, ignoreWritable=self.inplace_update,
# symLink=True, secret_store=runtimeContext.secret_store)
relink_initialworkdir(self.generatemapper, self.outdir,
self.builder.outdir, inplace_update=self.inplace_update)
self.execute([], env, runtimeContext)
def get_envvars(self):
env = self.environment
vars_to_preserve = self.runtime_context.preserve_environment
if self.runtime_context.preserve_entire_environment:
vars_to_preserve = os.environ
if vars_to_preserve is not None:
for key, value in os.environ.items():
if key in vars_to_preserve and key not in env:
# On Windows, subprocess env can't handle unicode.
env[key] = str(value) if onWindows() else value
env["HOME"] = str(self.outdir) if onWindows() else self.outdir
env["TMPDIR"] = str(self.tmpdir) if onWindows() else self.tmpdir
return env
class ToolArgparse(unittest.TestCase):
script = '''
#!/usr/bin/env cwl-runner
cwlVersion: v1.0
class: CommandLineTool
inputs:
- id: input
type: File
inputBinding:
position: 0
outputs:
- id: output
type: File
outputBinding:
glob: test.txt
stdout: test.txt
baseCommand: [cat]
'''
@pytest.mark.skipif(onWindows(),
reason="Instance of Cwltool is used, On windows that invoke a default docker Container")
def test_spaces_in_input_files(self):
with NamedTemporaryFile(mode='w', delete=False) as f:
f.write(self.script)
f.flush()
f.close()
with NamedTemporaryFile(prefix="test with spaces", delete=False) as spaces:
spaces.close()
self.assertEquals(
main(["--debug", f.name, '--input', spaces.name]), 1)
self.assertEquals(
main(["--debug", "--relax-path-checks", f.name, '--input',
spaces.name]), 0)
def get_windows_safe_factory(**execkwargs):
if onWindows():
makekwargs = {'find_default_container': functools.partial(
force_default_container, windows_default_container_id),
'use_container': True}
execkwargs['default_container'] = windows_default_container_id
else:
makekwargs = {}
return Factory(makekwargs=makekwargs, **execkwargs)
class ExecJsProcessTest(unittest.TestCase):
@pytest.mark.skipif(
onWindows(), reason="Caching processes for windows is not supported.")
def test_caches_js_processes(self):
exec_js_process("7", context="{}")
with patch("cwltool.sandboxjs.new_js_proc",
new=Mock(wraps=cwltool.sandboxjs.new_js_proc)) as mock:
exec_js_process("7", context="{}")
mock.assert_not_called()
def test_compute_checksum():
runtime_context = RuntimeContext()
runtime_context.compute_checksum = True
runtime_context.use_container = onWindows()
factory = cwltool.factory.Factory(runtime_context=runtime_context)
echo = factory.make(get_data("tests/wf/cat-tool.cwl"))
output = echo(
file1={"class": "File",
"location": get_data("tests/wf/whale.txt")},
reverse=False)
assert output['output']["checksum"] == "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376"
def test_compute_checksum(self):
f = cwltool.factory.Factory(compute_checksum=True,
use_container=onWindows())
echo = f.make(get_data("tests/wf/cat-tool.cwl"))
output = echo(file1={
"class": "File",
"location": get_data("tests/wf/whale.txt")
},
reverse=False
)
self.assertEquals(output['output']["checksum"], "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376")
def get_windows_safe_factory(runtime_context=None, # type: RuntimeContext
loading_context=None, # type: LoadingContext
executor=None # type: Any
): # type: (...) -> Factory
if onWindows():
if not runtime_context:
runtime_context = RuntimeContext()
runtime_context.find_default_container = functools.partial(
force_default_container, windows_default_container_id)
runtime_context.use_container = True
runtime_context.default_container = windows_default_container_id
return Factory(executor, loading_context, runtime_context)
def test_bioconda(self):
if onWindows() or not deps:
pytest.skip("bioconda currently not working on MS Windows.")
elif not deps:
pytest.skip("galaxy-lib is not installed.")
wflow = get_data("tests/seqtk_seq.cwl")
job = get_data("tests/seqtk_seq_job.json")
error_code, stdout, stderr = self.get_main_output(
["--beta-conda-dependencies", "--debug", wflow, job])
print(stderr)
print(stdout)
assert error_code is 0
def get_windows_safe_factory(runtime_context=None, # type: RuntimeContext
loading_context=None, # type: LoadingContext
executor=None # type: Any
): # type: (...) -> Factory
if onWindows():
if not runtime_context:
runtime_context = RuntimeContext()
runtime_context.find_default_container = functools.partial(
force_default_container, windows_default_container_id)
runtime_context.use_container = True
runtime_context.default_container = windows_default_container_id
return Factory(executor, loading_context, runtime_context)
def get_windows_safe_factory(
runtime_context: Optional[RuntimeContext] = None,
loading_context: Optional[LoadingContext] = None,
executor: Optional[JobExecutor] = None,
) -> Factory:
if onWindows():
if not runtime_context:
runtime_context = RuntimeContext()
runtime_context.find_default_container = functools.partial(
force_default_container, windows_default_container_id)
runtime_context.use_container = True
runtime_context.default_container = windows_default_container_id
return Factory(executor, loading_context, runtime_context)
def test_compute_checksum():
runtime_context = RuntimeContext()
runtime_context.compute_checksum = True
runtime_context.use_container = onWindows()
factory = cwltool.factory.Factory(runtime_context=runtime_context)
echo = factory.make(get_data("tests/wf/cat-tool.cwl"))
output = echo(file1={
"class": "File",
"location": get_data("tests/wf/whale.txt")
},
reverse=False)
assert output['output'][
"checksum"] == "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376"
def test_env_filtering(factor):
test_file = "tests/env.cwl"
commands = factor.split()
commands.extend([get_data(test_file)])
error_code, stdout, stderr = get_main_output(commands)
assert "completed success" in stderr, (error_code, stdout, stderr)
assert error_code == 0, (error_code, stdout, stderr)
if onWindows():
target = 5
else:
target = 4
assert json.loads(stdout)['env_count'] == target, (error_code, stdout,
stderr)
class TestOverride(unittest.TestCase):
@pytest.mark.skipif(onWindows(),
reason="Instance of Cwltool is used, On windows that invoke a default docker Container")
def test_overrides(self):
sio = StringIO()
self.assertEquals(main([get_data('tests/override/echo.cwl'),
get_data('tests/override/echo-job.yml')],
stdout=sio), 0)
self.assertEquals({"out": "zing hello1\n"}, json.loads(sio.getvalue()))
sio = StringIO()
self.assertEquals(main(["--overrides", get_data('tests/override/ov.yml'),
get_data('tests/override/echo.cwl'),
get_data('tests/override/echo-job.yml')],
stdout=sio), 0)
self.assertEquals({"out": "zing hello2\n"}, json.loads(sio.getvalue()))
sio = StringIO()
self.assertEquals(main([get_data('tests/override/echo.cwl'),
get_data('tests/override/echo-job-ov.yml')],
stdout=sio), 0)
self.assertEquals({"out": "zing hello3\n"}, json.loads(sio.getvalue()))
sio = StringIO()
self.assertEquals(main([get_data('tests/override/echo-job-ov2.yml')],
stdout=sio), 0)
self.assertEquals({"out": "zing hello4\n"}, json.loads(sio.getvalue()))
sio = StringIO()
self.assertEquals(main(["--overrides", get_data('tests/override/ov.yml'),
get_data('tests/override/echo-wf.cwl'),
get_data('tests/override/echo-job.yml')],
stdout=sio), 0)
self.assertEquals({"out": "zing hello2\n"}, json.loads(sio.getvalue()))
sio = StringIO()
self.assertEquals(main(["--overrides", get_data('tests/override/ov2.yml'),
get_data('tests/override/echo-wf.cwl'),
get_data('tests/override/echo-job.yml')],
stdout=sio), 0)
self.assertEquals({"out": "zing hello5\n"}, json.loads(sio.getvalue()))
sio = StringIO()
self.assertEquals(main(["--overrides", get_data('tests/override/ov3.yml'),
get_data('tests/override/echo-wf.cwl'),
get_data('tests/override/echo-job.yml')],
stdout=sio), 0)
self.assertEquals({"out": "zing hello6\n"}, json.loads(sio.getvalue()))
def test_env_filtering(factor):
test_file = "tests/env.cwl"
commands = factor.split()
commands.extend([get_data(test_file)])
error_code, stdout, stderr = get_main_output(commands)
process = subprocess.Popen(
[
"sh",
"-c",
r"""getTrueShellExeName() {
local trueExe nextTarget 2>/dev/null
trueExe=$(ps -o comm= $$) || return 1
[ "${trueExe#-}" = "$trueExe" ] || trueExe=${trueExe#-}
[ "${trueExe#/}" != "$trueExe" ] || trueExe=$([ -n "$ZSH_VERSION" ] && which -p "$trueExe" || which "$trueExe")
while nextTarget=$(readlink "$trueExe"); do trueExe=$nextTarget; done
printf '%s\n' "$(basename "$trueExe")"
} ; getTrueShellExeName""",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=None,
)
sh_name, sh_name_err = process.communicate()
sh_name = sh_name.decode("utf-8").strip()
assert "completed success" in stderr, (error_code, stdout, stderr)
assert error_code == 0, (error_code, stdout, stderr)
if onWindows():
target = 5
elif sh_name == "dash":
target = 4
else: # bash adds "SHLVL" and "_" environment variables
target = 6
result = json.loads(stdout)["env_count"]
details = ""
if result != target:
_, details, _ = get_main_output(
["--quiet", get_data("tests/env2.cwl")])
print(sh_name)
print(sh_name_err)
print(details)
assert result == target, (error_code, sh_name, sh_name_err, details,
stdout, stderr)
class TestParallel(unittest.TestCase):
@pytest.mark.skipif(
onWindows(),
reason=
"Unexplainable behavior: cwltool on AppVeyor does not recognize cwlVersion"
"in count-lines1-wf.cwl")
def test_sequential_workflow(self):
test_file = "tests/wf/count-lines1-wf.cwl"
f = cwltool.factory.Factory(executor=MultithreadedJobExecutor())
echo = f.make(get_data(test_file))
self.assertEqual(
echo(file1={
"class": "File",
"location": get_data("tests/wf/whale.txt")
}), {"count_output": 16})
def test_scattered_workflow(self):
test_file = "tests/wf/scatter-wf4.cwl"
job_file = "tests/wf/scatter-job2.json"
f = cwltool.factory.Factory(executor=MultithreadedJobExecutor())
echo = f.make(get_data(test_file))
with open(get_data(job_file)) as job:
self.assertEqual(echo(**json.load(job)),
{'out': ['foo one three', 'foo two four']})
self.assertIn("[err] Error message", output)
self.assertEquals(error_code, 0, output)
def test_no_js_console(self):
for test_file in ("js_output.cwl", "js_output_workflow.cwl"):
error_code, output = self.get_main_stderr(
["--no-container",
get_data("tests/wf/" + test_file)])
self.assertNotIn("[log] Log message", output)
self.assertNotIn("[err] Error message", output)
@pytest.mark.skipif(
onWindows(),
reason=
"Instance of cwltool is used, on Windows it invokes a default docker container"
"which is not supported on AppVeyor")
class TestCache(TestCmdLine):
def test_wf_without_container(self):
test_file = "hello-workflow.cwl"
error_code, output = self.get_main_stderr([
"--cachedir", "cache",
get_data("tests/wf/" + test_file), "--usermessage", "hello"
])
self.assertIn("completed success", output)
self.assertEquals(error_code, 0)
if __name__ == '__main__':
def norm(uri):
if onWindows():
return uri.lower()
return uri
except ImportError:
deps = None
@needs_docker
@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed")
def test_biocontainers() -> None:
wflow = get_data("tests/seqtk_seq.cwl")
job = get_data("tests/seqtk_seq_job.json")
error_code, _, _ = get_main_output(
["--beta-use-biocontainers", wflow, job])
assert error_code == 0
@pytest.mark.skipif(onWindows(),
reason="bioconda currently not working on MS Windows")
@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed")
def test_bioconda() -> None:
wflow = get_data("tests/seqtk_seq.cwl")
job = get_data("tests/seqtk_seq_job.json")
error_code, _, stderr = get_main_output(
["--beta-conda-dependencies", "--debug", wflow, job])
assert error_code == 0, stderr
@pytest.mark.skipif(not spawn.find_executable("modulecmd"),
reason="modulecmd not installed")
def test_modules() -> None:
wflow = get_data("tests/random_lines.cwl")
try:
from galaxy.tools import deps
except ImportError:
deps = None
@needs_docker
@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed")
def test_biocontainers():
wflow = get_data("tests/seqtk_seq.cwl")
job = get_data("tests/seqtk_seq_job.json")
error_code, _, _ = get_main_output(
["--beta-use-biocontainers", wflow, job])
assert error_code == 0
@pytest.mark.skipif(onWindows(), reason="bioconda currently not working on MS Windows")
@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed")
def test_bioconda():
wflow = get_data("tests/seqtk_seq.cwl")
job = get_data("tests/seqtk_seq_job.json")
error_code, _, stderr = get_main_output(
["--beta-conda-dependencies", "--debug", wflow, job])
assert error_code == 0, stderr
import os
from distutils import spawn
@pytest.mark.skipif(not spawn.find_executable("modulecmd"), reason="modulecmd not installed")
def test_modules():
wflow = get_data("tests/random_lines.cwl")
job = get_data("tests/random_lines_job.json")
needs_singularity_2_6 = pytest.mark.skipif(
not bool(
distutils.spawn.find_executable("singularity") and is_version_2_6()),
reason=
"Requires that version 2.6.x of singularity executable version is on the system path.",
)
needs_singularity_3_or_newer = pytest.mark.skipif(
(not bool(distutils.spawn.find_executable("singularity")))
or (not is_version_3_or_newer()),
reason=
"Requires that version 3.x of singularity executable version is on the system path.",
)
windows_needs_docker = pytest.mark.skipif(
onWindows() and not bool(distutils.spawn.find_executable("docker")),
reason="Running this test on MS Windows requires the docker executable "
"on the system path.",
)
def get_main_output(
args: List[str],
env: Union[Mapping[bytes, Union[bytes, str]],
Mapping[str, Union[bytes, str]], None] = None,
) -> Tuple[Optional[int], str, str]:
process = subprocess.Popen(
[sys.executable, "-m", "cwltool"] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
def norm(uri: str) -> str:
if onWindows():
return uri.lower()
return uri
pass
if not filepath or not os.path.isfile(filepath):
filepath = os.path.join(os.path.dirname(__file__), os.pardir, filename)
return Text(Path(filepath).resolve())
needs_docker = pytest.mark.skipif(not bool(distutils.spawn.find_executable('docker')),
reason="Requires the docker executable on the "
"system path.")
needs_singularity = pytest.mark.skipif(not bool(distutils.spawn.find_executable('singularity')),
reason="Requires the singularity executable on the "
"system path.")
windows_needs_docker = pytest.mark.skipif(
onWindows() and not bool(distutils.spawn.find_executable('docker')),
reason="Running this test on MS Windows requires the docker executable "
"on the system path.")
def get_main_output(args):
process = subprocess.Popen(
[sys.executable, "-m", "cwltool"] + args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
return process.returncode, stdout.decode(), stderr.decode()
@contextlib.contextmanager
def temp_dir(suffix=""):
c_dir = tempfile.mkdtemp(suffix, dir=os.curdir)
try:
def make_job_runner(
self,
runtimeContext # type: RuntimeContext
): # type: (...) -> Type[JobBase]
dockerReq, _ = self.get_requirement("DockerRequirement")
if not dockerReq and runtimeContext.use_container:
if runtimeContext.find_default_container:
default_container = runtimeContext.find_default_container(self)
if default_container:
self.requirements.insert(
0, {
"class": "DockerRequirement",
"dockerPull": default_container
})
dockerReq = self.requirements[0]
if default_container == windows_default_container_id and runtimeContext.use_container and onWindows(
):
_logger.warning(DEFAULT_CONTAINER_MSG %
(windows_default_container_id,
windows_default_container_id))
if dockerReq and runtimeContext.use_container:
if runtimeContext.singularity:
SingularityCommandLineJob._execute = _parsl_execute
return SingularityCommandLineJob
elif runtimeContext.shifter:
ShifterCommandLineJob._execute = _parsl_execute
return ShifterCommandLineJob
else:
DockerCommandLineJob._execute = _parsl_execute
return DockerCommandLineJob
else:
for t in reversed(self.requirements):
if t["class"] == "DockerRequirement":
raise UnsupportedRequirement(
"--no-container, but this CommandLineTool has "
"DockerRequirement under 'requirements'.")
CommandLineJob._execute = _parsl_execute
return CommandLineJob
commands.extend(
[
"--no-compute-checksum",
"--outdir",
str(tmp_path),
get_data(test_file),
get_data(job_file),
]
)
error_code, stdout, stderr = get_main_output(commands)
assert "completed success" in stderr
assert error_code == 0
assert "checksum" not in stdout
@pytest.mark.skipif(onWindows(), reason="udocker is Linux/macOS only")
@pytest.mark.parametrize("factor", test_factors)
def test_bad_userspace_runtime(factor: str) -> None:
test_file = "tests/wf/wc-tool.cwl"
job_file = "tests/wf/wc-job.json"
commands = factor.split()
commands.extend(
[
"--user-space-docker-cmd=quaquioN",
"--default-container=debian",
get_data(test_file),
get_data(job_file),
]
)
error_code, stdout, stderr = get_main_output(commands)
assert "or quaquioN is missing or broken" in stderr, stderr
def norm(uri):
if onWindows():
return uri.lower()
else:
return uri
(b'v0.10.26\n', True),
(b'v4.4.2\n', True),
(b'v7.7.3\n', True)
]
@pytest.mark.parametrize('version,supported', node_versions)
def test_node_version(version, supported, mocker):
mocked_subprocess = mocker.patch("cwltool.sandboxjs.subprocess")
mocked_subprocess.check_output = mocker.Mock(return_value=version)
assert sandboxjs.check_js_threshold_version('node') == supported
@windows_needs_docker
def test_value_from_two_concatenated_expressions():
factory = get_windows_safe_factory()
echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
file = {"class": "File",
"location": get_data("tests/wf/whale.txt")}
assert echo(file1=file) == {u"out": u"a string\n"}
@pytest.mark.skipif(onWindows(), reason="Caching processes for windows is not supported.")
def test_caches_js_processes(mocker):
sandboxjs.exec_js_process("7", context="{}")
mocked_new_js_proc = mocker.patch("cwltool.sandboxjs.new_js_proc")
sandboxjs.exec_js_process("7", context="{}")
mocked_new_js_proc.assert_not_called()
class ToolArgparse(unittest.TestCase):
script = '''
#!/usr/bin/env cwl-runner
cwlVersion: v1.0
class: CommandLineTool
doc: "This tool is developed for SMC-RNA Challenge for detecting gene fusions (STAR fusion)"
inputs:
#Give it a list of input files
- id: input
type: File
inputBinding:
position: 0
outputs:
- id: output
type: File
outputBinding:
glob: test.txt
stdout: test.txt
baseCommand: [cat]
'''
script2 = '''
#!/usr/bin/env cwl-runner
cwlVersion: v1.0
class: CommandLineTool
inputs:
- id: bdg
type: "boolean"
outputs:
- id: output
type: File
outputBinding:
glob: foo
baseCommand:
- echo
- "ff"
stdout: foo
'''
script3 = '''
#!/usr/bin/env cwl-runner
cwlVersion: v1.0
class: ExpressionTool
inputs:
foo:
type:
type: record
fields:
one: File
two: string
expression: $(inputs.foo.two)
outputs: []
'''
@pytest.mark.skipif(
onWindows(),
reason=
"Instance of Cwltool is used, On windows that invoke a default docker Container"
)
def test_help(self):
with NamedTemporaryFile(mode='w', delete=False) as f:
f.write(self.script)
f.flush()
f.close()
self.assertEquals(
main(
["--debug", f.name, '--input',
get_data('tests/echo.cwl')]), 0)
self.assertEquals(
main(
["--debug", f.name, '--input',
get_data('tests/echo.cwl')]), 0)
@pytest.mark.skipif(
onWindows(),
reason=
"Instance of Cwltool is used, On windows that invoke a default docker Container"
)
def test_bool(self):
with NamedTemporaryFile(mode='w', delete=False) as f:
f.write(self.script2)
f.flush()
f.close()
try:
self.assertEquals(main([f.name, '--help']), 0)
except SystemExit as e:
self.assertEquals(e.code, 0)
def test_record_help(self):
with NamedTemporaryFile(mode='w', delete=False) as f:
f.write(self.script3)
f.flush()
f.close()
try:
self.assertEquals(main([f.name, '--help']), 0)
except SystemExit as e:
self.assertEquals(e.code, 0)
def test_record(self):
with NamedTemporaryFile(mode='w', delete=False) as f:
f.write(self.script3)
f.flush()
f.close()
try:
self.assertEquals(
main([
f.name, '--foo.one',
get_data('tests/echo.cwl'), '--foo.two', 'test'
]), 0)
except SystemExit as e:
self.assertEquals(e.code, 0)
# but it shouldn't be expanded.
self.assertEquals(main(["--enable-ext", get_data('tests/wf/listing_shallow.cwl'), get_data('tests/listing-job.yml')]), 1)
def test_listing_none(self):
# This fails on purpose, because it tries to access listing but it shouldn't be there.
self.assertEquals(main(["--enable-ext", get_data('tests/wf/listing_none.cwl'), get_data('tests/listing-job.yml')]), 1)
def test_listing_v1_0(self):
# Default behavior in 1.0 is deep expansion.
self.assertEquals(main([get_data('tests/wf/listing_v1_0.cwl'), get_data('tests/listing-job.yml')]), 0)
# def test_listing_v1_1(self):
# # Default behavior in 1.1 will be no expansion
# self.assertEquals(main([get_data('tests/wf/listing_v1_1.cwl'), get_data('tests/listing-job.yml')]), 1)
@pytest.mark.skipif(onWindows(),
reason="InplaceUpdate uses symlinks,does not run on windows without admin privileges")
class TestInplaceUpdate(unittest.TestCase):
def test_updateval(self):
try:
tmp = tempfile.mkdtemp()
with open(os.path.join(tmp, "value"), "w") as f:
f.write("1")
out = tempfile.mkdtemp()
self.assertEquals(main(["--outdir", out, get_data('tests/wf/updateval.cwl'), "-r", os.path.join(tmp, "value")]), 0)
with open(os.path.join(tmp, "value"), "r") as f:
self.assertEquals("1", f.read())
with open(os.path.join(out, "value"), "r") as f:
self.assertEquals("2", f.read())
class TestPack(unittest.TestCase):
maxDiff = None
def test_pack(self):
document_loader, workflowobj, uri = fetch_document(
get_data("tests/wf/revsort.cwl"))
document_loader, avsc_names, processobj, metadata, uri = validate_document(
document_loader, workflowobj, uri)
packed = cwltool.pack.pack(document_loader, processobj, uri, metadata)
with open(get_data("tests/wf/expect_packed.cwl")) as f:
expect_packed = json.load(f)
adjustFileObjs(packed, partial(makeRelative,
os.path.abspath(get_data("tests/wf"))))
adjustDirObjs(packed, partial(makeRelative,
os.path.abspath(get_data("tests/wf"))))
self.assertIn("$schemas", packed)
del packed["$schemas"]
del expect_packed["$schemas"]
self.assertEqual(expect_packed, packed)
def test_pack_missing_cwlVersion(self):
"""Test to ensure the generated pack output is not missing
the `cwlVersion` in case of single tool workflow and single step workflow"""
# Testing single tool workflow
document_loader, workflowobj, uri = fetch_document(
get_data("tests/wf/hello_single_tool.cwl"))
document_loader, avsc_names, processobj, metadata, uri = validate_document(
document_loader, workflowobj, uri)
# generate pack output dict
packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
self.assertEqual('v1.0', packed["cwlVersion"])
# Testing single step workflow
document_loader, workflowobj, uri = fetch_document(
get_data("tests/wf/hello-workflow.cwl"))
document_loader, avsc_names, processobj, metadata, uri = validate_document(
document_loader, workflowobj, uri)
# generate pack output dict
packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
self.assertEqual('v1.0', packed["cwlVersion"])
def test_pack_idempotence_tool(self):
"""Test to ensure that pack produces exactly the same document for
an already packed document"""
# Testing single tool
self._pack_idempotently("tests/wf/hello_single_tool.cwl")
def test_pack_idempotence_workflow(self):
"""Test to ensure that pack produces exactly the same document for
an already packed document"""
# Testing workflow
self._pack_idempotently("tests/wf/count-lines1-wf.cwl")
def _pack_idempotently(self, document):
document_loader, workflowobj, uri = fetch_document(
get_data(document))
document_loader, avsc_names, processobj, metadata, uri = validate_document(
document_loader, workflowobj, uri)
# generate pack output dict
packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
document_loader, workflowobj, uri2 = fetch_document(packed)
document_loader, avsc_names, processobj, metadata, uri2 = validate_document(
document_loader, workflowobj, uri)
double_packed = json.loads(print_pack(document_loader, processobj, uri2, metadata))
self.assertEqual(packed, double_packed)
@pytest.mark.skipif(onWindows(),
reason="Instance of cwltool is used, on Windows it invokes a default docker container"
"which is not supported on AppVeyor")
def test_packed_workflow_execution(self):
test_wf = "tests/wf/count-lines1-wf.cwl"
test_wf_job = "tests/wf/wc-job.json"
document_loader, workflowobj, uri = fetch_document(
get_data(test_wf))
document_loader, avsc_names, processobj, metadata, uri = validate_document(
document_loader, workflowobj, uri)
packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
temp_packed_path = "/tmp/packedwf"
with open(temp_packed_path, 'w') as f:
json.dump(packed, f)
normal_output = StringIO()
packed_output = StringIO()
self.assertEquals(main(['--debug', get_data(temp_packed_path),
get_data(test_wf_job)],
stdout=packed_output), 0)
self.assertEquals(main([get_data(test_wf),
get_data(test_wf_job)],
stdout=normal_output), 0)
self.assertEquals(json.loads(packed_output.getvalue()), json.loads(normal_output.getvalue()))
from typing import Any, cast
import pytest
from ruamel.yaml.comments import CommentedMap
from schema_salad.sourceline import cmap
from cwltool import command_line_tool
from cwltool.context import LoadingContext, RuntimeContext
from cwltool.utils import onWindows, windows_default_container_id
@pytest.mark.skip(not onWindows(), reason="MS Windows only") # type: ignore
def test_default_docker_warning(mocker: Any) -> None:
"""Check warning when default docker Container is used on Windows."""
mocker.patch("cwltool.command_line_tool._logger")
tool = command_line_tool.CommandLineTool(
cast(CommentedMap, cmap({
"inputs": [],
"outputs": []
})), LoadingContext())
tool.make_job_runner(
RuntimeContext(
{"find_default_container": lambda x: "frolvlad/alpine-bash"}))
command_line_tool._logger.warning.assert_called_with( # type: ignore
command_line_tool.DEFAULT_CONTAINER_MSG,
windows_default_container_id,
windows_default_container_id,
)