def test_clone_repos(self):
wf_data = {"steps": [{"uses": "popperized/bin/sh@master"}]}
wf = WorkflowParser.parse(wf_data=wf_data)
cache_dir = os.path.join(os.environ["HOME"], ".cache/popper/")
# clone repos in the default cache directory.
conf = ConfigLoader.load()
runner = WorkflowRunner(conf)
runner._clone_repos(wf)
step_dir = os.path.join(cache_dir, conf.wid,
"github.com/popperized/bin")
self.assertTrue(os.path.exists(step_dir))
# clone repos in custom cache directory
os.environ["POPPER_CACHE_DIR"] = "/tmp/smdir"
conf = ConfigLoader.load()
runner = WorkflowRunner(conf)
runner._clone_repos(wf)
step_dir = os.path.join("/tmp/smdir", conf.wid,
"github.com/popperized/bin")
self.assertTrue(os.path.exists(step_dir))
os.environ.pop("POPPER_CACHE_DIR")
# check failure when container is not available and we skip cloning
shutil.rmtree("/tmp/smdir")
shutil.rmtree(cache_dir)
conf = ConfigLoader.load(skip_clone=True)
runner = WorkflowRunner(conf)
self.assertRaises(SystemExit, runner._clone_repos, wf)
def test_check_secrets(self):
wf_data = {
"steps": [{
"uses": "docker://alpine:3.9",
"args": ["ls", "-ltr"],
"secrets": ["SECRET_ONE", "SECRET_TWO"],
}]
}
wf = WorkflowParser.parse(wf_data=wf_data)
# in dry-run, secrets are ignored
runner = WorkflowRunner(ConfigLoader.load(dry_run=True))
runner._process_secrets(wf)
# now go back to not dry-running
runner = WorkflowRunner(ConfigLoader.load())
# when CI=true it should fail
os.environ["CI"] = "true"
self.assertRaises(SystemExit, runner._process_secrets, wf)
# add one secret
os.environ["SECRET_ONE"] = "1234"
# it should fail again, as we're missing one
self.assertRaises(SystemExit, runner._process_secrets, wf)
os.environ.pop("CI")
# now is fine
with patch("getpass.getpass", return_value="5678"):
runner._process_secrets(wf)
# pop the other
os.environ.pop("SECRET_ONE")
def test_docker_basic_run(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
test_string = "STEP_INFO:popper:Successfully tagged popperized/bin:master"
with WorkflowRunner(conf) as r:
wf_data = {
"steps": [{
"uses": "popperized/bin/sh@master",
"args": ["ls"],
}]
}
with self.assertLogs(log, level="STEP_INFO") as cm:
r.run(WorkflowParser.parse(wf_data=wf_data))
self.assertTrue(test_string in cm.output,
f"Got cmd output: {cm.output}")
wf_data = {
"steps": [{
"uses": "docker://*****:*****@master",
"args": ["ls"],
}]
}
with self.assertLogs(log, level="STEP_INFO") as cm:
r.run(WorkflowParser.parse(wf_data=wf_data))
self.assertTrue(test_string not in cm.output)
repo.close()
shutil.rmtree(repo.working_dir, ignore_errors=True)
def test_create_cmd(self):
self.maxDiff = None
config = {"workspace_dir": "/w"}
with DockerRunner(config=ConfigLoader.load(**config)) as drunner:
step = Box({"args": ["-two", "-flags"]}, default_box=True)
cmd = drunner._create_cmd(step, "foo:1.9", "container_name")
expected = ("docker create"
" --name container_name"
" --workdir /workspace"
" -v /w:/workspace:Z"
" -v /var/run/docker.sock:/var/run/docker.sock"
" foo:1.9 -two -flags")
self.assertEqual(expected, cmd)
config_dict = {
"engine": {
"name": "docker",
"options": {
"privileged": True,
"hostname": "popper.local",
"domainname": "www.example.org",
"volumes": ["/path/in/host:/path/in/container"],
"environment": {
"FOO": "bar"
},
},
},
"resource_manager": {
"name": "slurm"
},
}
config = {"workspace_dir": "/w", "config_file": config_dict}
with DockerRunner(config=ConfigLoader.load(**config)) as drunner:
step = Box({"args": ["-two", "-flags"]}, default_box=True)
cmd = drunner._create_cmd(step, "foo:1.9", "container_name")
expected = ("docker create --name container_name "
"--workdir /workspace "
"-v /w:/workspace:Z "
"-v /var/run/docker.sock:/var/run/docker.sock "
"-v /path/in/host:/path/in/container "
"-e FOO=bar --privileged --hostname popper.local "
"--domainname www.example.org"
" foo:1.9 -two -flags")
self.assertEqual(expected, cmd)
def test_get_container_options(self):
config_dict = {
"engine": {
"name": "singularity",
"options": {
"hostname": "popper.local",
"ipc": True,
"bind": ["/path/in/host:/path/in/container"],
},
}
}
config = ConfigLoader.load(config_file=config_dict)
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
options = sr._get_container_options()
self.assertEqual(
options,
[
"--userns",
"--pwd",
"/workspace",
"--bind",
f"{os.getcwd()}:/workspace",
"--bind",
"/path/in/host:/path/in/container",
"--hostname",
"popper.local",
"--ipc",
],
)
def test_exec_mpi_failure(self, mock_kill):
config_dict = {
"engine": {
"name": "singularity",
"options": {},
},
"resource_manager": {
"name": "slurm",
"options": {
"1": {
"nodes": 2,
"nodelist": "worker1,worker2",
"overcommit": True
}
},
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
self.Popen.set_command(
"sbatch "
"--wait --overcommit "
f"popper_1_{config.wid}.sh",
returncode=12,
)
self.Popen.set_command(f"tail -f popper_1_{config.wid}.out",
returncode=0)
with WorkflowRunner(config) as r:
wf_data = {"steps": [{"uses": "docker://alpine", "args": ["ls"]}]}
self.assertRaises(SystemExit, r.run,
WorkflowParser.parse(wf_data=wf_data))
def cli(ctx, file, step, entrypoint, skip_pull, skip_clone, substitution,
conf):
"""Opens an interactive shell using all the attributes defined in the workflow file
for the given STEP, but ignoring ``runs`` and ``args``. By default, it invokes
/bin/bash. If you need to invoke another one, you can specify it in the --entrypoint
flag.
NOTE: this command only works for (local) host runner in Docker.
"""
wf = WorkflowParser.parse(file=file,
step=step,
immutable=False,
substitutions=substitution)
# override entrypoint
step = wf.steps[0]
step.args = []
step.runs = entrypoint
# configure runner so containers execute in attached mode and create a tty
config = ConfigLoader.load(
engine_name="docker",
pty=True,
skip_pull=skip_pull,
skip_clone=skip_clone,
config_file=conf,
)
with WorkflowRunner(config) as runner:
try:
runner.run(wf)
except Exception as e:
log.debug(traceback.format_exc())
log.fail(e)
def test_prepare_environment_with_git(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with StepRunner(conf) as r:
step = Box({
"name": "a",
"env": {
"FOO": "BAR"
},
"secrets": ["A"]
},
default_box=True)
os.environ["A"] = "BC"
env = r._prepare_environment(step, {"other": "b"})
expected = {
"FOO": "BAR",
"A": "BC",
"other": "b",
"GIT_COMMIT": conf.git_commit,
"GIT_BRANCH": conf.git_branch,
"GIT_SHA_SHORT": conf.git_sha_short,
"GIT_REMOTE_ORIGIN_URL": conf.git_remote_origin_url,
"GIT_TAG": conf.git_tag,
}
self.assertDictEqual(expected, env)
os.environ.pop("A")
def test_config_with_git_repo(self):
r = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=r.working_dir)
sha = r.head.object.hexsha
self.assertEqual(r.git.rev_parse(sha), conf.git_commit)
self.assertEqual(r.git.rev_parse(sha, short=7), conf.git_sha_short)
self.assertEqual(r.active_branch.name, conf.git_branch)
def test_create_container(self):
config = ConfigLoader.load()
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["echo hello"],
"id": "kontainer_one",
},
default_box=True,
)
cid = pu.sanitized_name(step.id, config.wid)
with DockerRunner(init_docker_client=True, config=config) as dr:
c = dr._create_container(cid, step)
self.assertEqual(c.status, "created")
c.remove()
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["echo", "hello_world"],
"id": "KoNtAiNeR tWo",
},
default_box=True,
)
cid = pu.sanitized_name(step.id, config.wid)
with DockerRunner(init_docker_client=True, config=config) as dr:
c = dr._create_container(cid, step)
self.assertEqual(c.status, "created")
c.remove()
def test_docker_basic_run(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with WorkflowRunner(conf) as r:
wf_data = {"steps": [{"uses": "popperized/bin/sh@master", "args": ["ls"],}]}
r.run(WorkflowParser.parse(wf_data=wf_data))
wf_data = {
"steps": [
{
"uses": "docker://alpine:3.9",
"args": ["sh", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {"FOO": "bar"},
}
]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(os.path.join(repo.working_dir, "hello.txt"), "r") as f:
self.assertEqual(f.read(), "bar\n")
wf_data = {
"steps": [
{
"uses": "docker://alpine:3.9",
"args": ["nocommandisnamedlikethis"],
}
]
}
self.assertRaises(SystemExit, r.run, WorkflowParser.parse(wf_data=wf_data))
repo.close()
shutil.rmtree(repo.working_dir, ignore_errors=True)
def test_setup_singularity_cache(self):
config = ConfigLoader.load()
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
self.assertEqual(
f'{os.environ["HOME"]}/.cache/popper/singularity/{config.wid}',
sr._singularity_cache,
)
def test_create_cmd(self):
config = ConfigLoader.load(workspace_dir="/w")
with SingularityRunner(config=config) as sr:
step = Box({"args": ["-two", "-flags"]}, default_box=True)
sr._setup_singularity_cache()
sr._container = os.path.join(sr._singularity_cache, "c1.sif")
cmd = sr._create_cmd(step, "c1.sif")
expected = (
"singularity run"
" --userns --pwd /workspace"
" --bind /w:/workspace"
f' {os.environ["HOME"]}/.cache/popper/singularity/{config.wid}/c1.sif'
" -two -flags")
self.assertEqual(expected, cmd)
config_dict = {
"engine": {
"name": "singularity",
"options": {
"hostname": "popper.local",
"ipc": True,
"bind": ["/path/in/host:/path/in/container"],
},
},
"resource_manager": {
"name": "slurm"
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
with SingularityRunner(config=config) as sr:
step = Box({"args": ["-two", "-flags"]}, default_box=True)
sr._setup_singularity_cache()
sr._container = os.path.join(sr._singularity_cache, "c2.sif")
cmd = sr._create_cmd(step, "c2.sif")
# fmt: off
expected = f"singularity run --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local --ipc {os.environ['HOME']}/.cache/popper/singularity/{config.wid}/c2.sif -two -flags"
# fmt: on
self.assertEqual(expected, cmd)
def test_submit_batch_job(self, mock_kill):
config = ConfigLoader.load(workspace_dir="/w")
self.Popen.set_command(
"sbatch --wait "
f"--job-name popper_sample_{config.wid} "
f"--output {slurm_cache_dir}/popper_sample_{config.wid}.out "
f"{slurm_cache_dir}/popper_sample_{config.wid}.sh",
returncode=0,
)
self.Popen.set_command(
f"tail -f {slurm_cache_dir}/popper_sample_{config.wid}.out",
returncode=0)
step = Box({"id": "sample"}, default_box=True)
with SlurmRunner(config=config) as sr:
sr._submit_batch_job(["ls -la"], step)
with open(f"{slurm_cache_dir}/popper_sample_{config.wid}.sh",
"r") as f:
content = f.read()
self.assertEqual(content, "#!/bin/bash\nls -la")
self.assertEqual(len(sr._spawned_jobs), 0)
self.assertEqual(sr._out_stream_thread.is_alive(), False)
call_tail = call.Popen(
[
"tail", "-f",
f"{slurm_cache_dir}/popper_sample_{config.wid}.out"
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
)
call_sbatch = call.Popen(
[
"sbatch",
"--wait",
"--job-name",
f"popper_sample_{config.wid}",
"--output",
f"{slurm_cache_dir}/popper_sample_{config.wid}.out",
f"{slurm_cache_dir}/popper_sample_{config.wid}.sh",
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_pod_create_delete_exitcode(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with KubernetesRunner(config=conf) as kr:
kr._vol_claim_create()
step = Box(
{
"id": "test",
"uses": "docker://alpine:3.9",
"runs": ("echo", "hello"),
},
default_box=True,
)
kr._pod_name = kr._base_pod_name + f"-{step.id}"
kr._pod_create(step, "alpine:3.9")
self.assertEqual(kr._pod_exit_code(), 0)
response = self._kclient.read_namespaced_pod(kr._pod_name,
namespace="default")
self.assertEqual(response.status.phase, "Succeeded")
kr._pod_delete()
self.assertRaises(
Exception,
self._kclient.read_namespaced_pod,
**{
"name": kr._pod_name,
"namespace": "default"
},
)
time.sleep(5)
step = Box(
{
"id": "test",
"uses": "docker://alpine:3.9",
"runs": ("ecdho", "hello"),
},
default_box=True,
)
kr._pod_name = kr._base_pod_name + f"-{step.id}"
kr._pod_create(step, "alpine:3.9")
self.assertEqual(kr._pod_exit_code(), 1)
response = self._kclient.read_namespaced_pod(kr._pod_name,
namespace="default")
self.assertEqual(response.status.phase, "Failed")
kr._pod_delete()
kr._vol_claim_delete()
self.assertRaises(
Exception,
self._kclient.read_namespaced_pod,
**{
"name": kr._pod_name,
"namespace": "default"
},
)
def test_config_non_defaults(self):
expected = {
"skip_clone": True,
"skip_pull": True,
"dry_run": True,
"workspace_dir": os.path.realpath("/tmp/foo"),
"quiet": True,
"reuse": True,
}
conf = ConfigLoader.load(**expected)
self.assertEqual(expected,
TestPopperConfig.extract_dict(expected, conf))
def test_run(self, mock_kill):
config_dict = {
"engine": {
"name": "podman",
"options": {
"privileged": True,
"hostname": "popper.local",
"domainname": "www.example.org",
"volumes": ["/path/in/host:/path/in/container"],
"environment": {
"FOO": "bar"
},
},
},
"resource_manager": {
"name": "slurm"
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
self.Popen.set_command(
f"sbatch --wait --job-name popper_1_{config.wid} "
f"--output {slurm_cache_dir}/popper_1_{config.wid}.out "
f"{slurm_cache_dir}/popper_1_{config.wid}.sh",
returncode=0,
)
self.Popen.set_command(
f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out",
returncode=0)
with WorkflowRunner(config) as r:
wf_data = {
"steps": [{
"uses": "popperized/bin/sh@master",
"runs": ["cat"],
"args": ["README.md"],
}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(f"{slurm_cache_dir}/popper_1_{config.wid}.sh", "r") as f:
# fmt: off
expected = f"""#!/bin/bash
podman rm -f popper_1_{config.wid} || true
podman build -t popperized/bin:master {os.environ['HOME']}/.cache/popper/{config.wid}/github.com/popperized/bin/sh
podman create --name popper_1_{config.wid} --workdir /workspace --entrypoint cat -v /w:/workspace:Z -v /path/in/host:/path/in/container -e FOO=bar --privileged --hostname popper.local --domainname www.example.org popperized/bin:master README.md
podman start --attach popper_1_{config.wid}"""
# fmt: on
actual = f.read()
self.maxDiff = None
self.assertEqual(expected, actual)
def test_create_cmd(self):
config = ConfigLoader.load(workspace_dir="/w")
with SingularityRunner(config=config) as sr:
step = Box({"args": ["-two", "-flags"]}, default_box=True)
sr._container = "c1.sif"
cmd = sr._create_cmd(step, "c1.sif")
expected = ("singularity run"
" --userns --pwd /workspace"
" --bind /w:/workspace"
" c1.sif"
" -two -flags")
self.assertEqual(expected.split(" "), cmd)
config_dict = {
"engine": {
"name": "singularity",
"options": {
"hostname": "popper.local",
"ipc": True,
"bind": ["/path/in/host:/path/in/container"],
},
},
"resource_manager": {
"name": "slurm"
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
with SingularityRunner(config=config) as sr:
step = Box({"args": ["-two", "-flags"]}, default_box=True)
sr._container = "c2.sif"
cmd = sr._create_cmd(step, "c2.sif")
# fmt: off
expected = f"singularity run --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local --ipc c2.sif -two -flags"
# fmt: on
self.assertEqual(expected.split(" "), cmd)
def test_config_from_file(self):
config = {
"engine": {
"options": {
"privileged": True
}
},
"resource_manager": {
"options": {
"foo": "bar"
}
},
}
kwargs = {"config_file": config}
# engine name missing
with self.assertLogs("popper", level="INFO") as cm:
self.assertRaises(SystemExit, ConfigLoader.load, **kwargs)
self.assertEqual(len(cm.output), 1)
self.assertTrue("No engine name given" in cm.output[0])
# resman name missing
config.update({"engine": {"name": "foo"}})
with self.assertLogs("popper", level="INFO") as cm:
self.assertRaises(SystemExit, ConfigLoader.load, **kwargs)
self.assertEqual(len(cm.output), 1)
self.assertTrue("No resource manager name given" in cm.output[0])
# now all OK
config.update({"resource_manager": {"name": "bar"}})
conf = ConfigLoader.load(**kwargs)
self.assertEqual(conf.engine_name, "foo")
self.assertEqual(conf.resman_name, "bar")
self.assertEqual(conf.engine_opts, {})
self.assertEqual(conf.resman_opts, {})
config.update({"engine": {"name": "bar", "options": {"foo": "baz"}}})
conf = ConfigLoader.load(**kwargs)
self.assertEqual(conf.engine_opts, {"foo": "baz"})
def test_exec_srun(self, mock_kill):
config_dict = {
"engine": {
"name": "singularity",
"options": {},
},
"resource_manager": {
"name": "slurm",
"options": {
"sample": {
"gpus-per-task": 2,
"overcommit": True
}
},
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
self.Popen.set_command(
"srun --nodes 1 --ntasks 1 --ntasks-per-node 1 --gpus-per-task 2 --overcommit ls -la",
returncode=0,
)
step = Box({"id": "sample"}, default_box=True)
with SlurmRunner(config=config) as sr:
e = sr._exec_srun(["ls", "-la"], step, logging=True)
self.assertEqual(e, 0)
call_srun = call.Popen(
[
"srun",
"--nodes",
"1",
"--ntasks",
"1",
"--ntasks-per-node",
"1",
"--gpus-per-task",
"2",
"--overcommit",
"ls",
"-la",
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
)
self.assertEqual(call_srun in self.Popen.all_calls, True)
def test_prepare_environment_without_git(self):
with StepRunner(ConfigLoader.load(workspace_dir="/tmp/foo")) as r:
step = Box({
"name": "a",
"env": {
"FOO": "BAR"
},
"secrets": ["A"]
},
default_box=True)
os.environ["A"] = "BC"
env = r._prepare_environment(step, {"other": "b"})
self.assertDictEqual({"FOO": "BAR", "A": "BC", "other": "b"}, env)
os.environ.pop("A")
def test_create_container(self):
config = ConfigLoader.load()
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["echo hello"],
"id": "kontainer_one",
},
default_box=True,
)
cid = pu.sanitized_name(step.id, config.wid)
with PodmanRunner(init_podman_client=True, config=config) as pr:
c = pr._create_container(cid, step)
c_status_cmd = [
"podman",
"container",
"inspect",
"-f",
str("{{.State.Status}}"),
c,
]
__, _, c_status = HostRunner._exec_cmd(c_status_cmd, logging=False)
self.assertEqual(c_status, "configured")
cmd = ["podman", "container", "rm", c]
HostRunner._exec_cmd(cmd, logging=False)
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["echo", "hello_world"],
"id": "KoNtAiNeR tWo",
},
default_box=True,
)
cid = pu.sanitized_name(step.id, config.wid)
with PodmanRunner(init_podman_client=True, config=config) as pr:
c = pr._create_container(cid, step)
c_status_cmd = [
"podman",
"container",
"inspect",
"-f",
str("{{.State.Status}}"),
c,
]
__, _, c_status = HostRunner._exec_cmd(c_status_cmd, logging=False)
self.assertEqual(c_status, "configured")
cmd = ["podman", "container", "rm", c]
HostRunner._exec_cmd(cmd, logging=False)
def test_copy_ctx(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with KubernetesRunner(config=conf) as kr:
kr._vol_claim_create()
kr._init_pod_create()
time.sleep(5)
response = self._kclient.read_namespaced_pod(kr._init_pod_name,
namespace="default")
self.assertEqual(response.status.phase, "Running")
kr._copy_ctx()
kr._init_pod_delete()
kr._vol_claim_delete()
repo.close()
shutil.rmtree(repo.working_dir, ignore_errors=True)
def test_dry_run(self):
config = ConfigLoader.load(engine_name="singularity",
resman_name="slurm",
dry_run=True)
with WorkflowRunner(config) as r:
wf_data = {
"steps": [{
"uses": "docker://alpine",
"runs": ["cat"],
"args": ["README.md"],
}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
self.assertEqual(self.Popen.all_calls, [])
def test_stop_running_tasks(self):
self.Popen.set_command("scancel --name job_a", returncode=0)
with SlurmRunner(config=ConfigLoader.load()) as sr:
sr._spawned_jobs.add("job_a")
sr.stop_running_tasks()
compare(
call.Popen(
["scancel", "--name", "job_a"],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
),
self.Popen.all_calls[0],
)
def test_slurm_singularity_run(self, mock_kill):
config_dict = {
"engine": {
"name": "singularity",
"options": {
"hostname": "popper.local",
"bind": ["/path/in/host:/path/in/container"],
},
},
"resource_manager": {
"name": "slurm"
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
# fmt: off
self.Popen.set_command(
f"sbatch --wait --job-name popper_1_{config.wid} --output {slurm_cache_dir}/popper_1_{config.wid}.out {slurm_cache_dir}/popper_1_{config.wid}.sh",
returncode=0,
)
# fmt: on
self.Popen.set_command(
f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out",
returncode=0)
with WorkflowRunner(config) as r:
wf_data = {
"steps": [{
"uses": "popperized/bin/sh@master",
"args": ["ls"],
}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(f"{slurm_cache_dir}/popper_1_{config.wid}.sh", "r") as f:
# fmt: off
expected = f"""#!/bin/bash
singularity run --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local {os.environ['HOME']}/.cache/popper/singularity/{config.wid}/popper_1_{config.wid}.sif ls"""
# fmt: on
actual = f.read()
self.assertEqual(expected, actual)
def test_find_container(self):
config = ConfigLoader.load()
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["echo hello"],
"id": "kontainer_one",
},
default_box=True,
)
cid = pu.sanitized_name(step.id, config.wid)
with PodmanRunner(init_podman_client=True, config=config) as pr:
c = pr._find_container(cid)
self.assertEqual(c, None)
with PodmanRunner(init_podman_client=True, config=config) as pr:
container = pr._create_container(cid, step)
c = pr._find_container(cid)
self.assertEqual(c, container)
cmd = ["podman", "container", "rm", "-f", cid]
HostRunner._exec_cmd(cmd, logging=False)
def test_vol_claim_create_delete(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with KubernetesRunner(config=conf) as kr:
kr._vol_claim_create()
response = self._kclient.read_namespaced_persistent_volume_claim(
kr._vol_claim_name, namespace="default")
self.assertEqual(response.status.phase, "Bound")
kr._vol_claim_delete()
self.assertRaises(
Exception,
self._kclient.read_namespaced_persistent_volume_claim,
{
"name": kr._vol_claim_name,
"namespace": "default"
},
)
repo.close()
shutil.rmtree(repo.working_dir, ignore_errors=True)
def test_host_run(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with WorkflowRunner(conf) as r:
wf_data = {
"steps": [{"uses": "sh", "runs": ["cat"], "args": ["README.md"],}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
wf_data = {
"steps": [
{
"uses": "sh",
"runs": ["bash", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {"FOO": "bar"},
}
]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(os.path.join(repo.working_dir, "hello.txt"), "r") as f:
self.assertEqual(f.read(), "bar\n")
wf_data = {"steps": [{"uses": "sh", "runs": ["nocommandisnamedlikethis"]}]}
self.assertRaises(SystemExit, r.run, WorkflowParser.parse(wf_data=wf_data))
# check exit code 78
wf_data = {
"steps": [
{"uses": "sh", "runs": ["touch", "one.txt"]},
{"uses": "sh", "runs": ["bash", "-c", "exit 78"]},
{"uses": "sh", "runs": ["touch", "two.txt"]},
]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
self.assertTrue(os.path.isfile(os.path.join(repo.working_dir, "one.txt")))
self.assertFalse(os.path.isfile(os.path.join(repo.working_dir, "two.txt")))
repo.close()
shutil.rmtree(repo.working_dir, ignore_errors=True)
def test_config_defaults(self):
conf = ConfigLoader.load()
expected = Box(
{
"skip_clone": False,
"engine_name": "docker",
"engine_opts": {},
"resman_name": "host",
"resman_opts": {},
"skip_pull": False,
"dry_run": False,
"workspace_dir": os.getcwd(),
"quiet": False,
"reuse": False,
"pty": False,
"allow_undefined_secrets_in_ci": False,
},
default_box=True,
)
self.assertEqual(expected,
TestPopperConfig.extract_dict(expected, conf))
