def test_docker_basic_run(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with WorkflowRunner(conf) as r:
wf_data = {"steps": [{"uses": "popperized/bin/sh@master", "args": ["ls"],}]}
r.run(WorkflowParser.parse(wf_data=wf_data))
wf_data = {
"steps": [
{
"uses": "docker://alpine:3.9",
"args": ["sh", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {"FOO": "bar"},
}
]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(os.path.join(repo.working_dir, "hello.txt"), "r") as f:
self.assertEqual(f.read(), "bar\n")
wf_data = {
"steps": [
{
"uses": "docker://alpine:3.9",
"args": ["nocommandisnamedlikethis"],
}
]
}
self.assertRaises(SystemExit, r.run, WorkflowParser.parse(wf_data=wf_data))
repo.close()
shutil.rmtree(repo.working_dir, ignore_errors=True)
def test_skip_steps(self):
wf_data = {
"steps": [
{
"uses": "foo",
"id": "one"
},
{
"uses": "bar",
"id": "two"
},
{
"uses": "baz",
"id": "three"
},
]
}
# skip one step
wf = WorkflowParser.parse(wf_data=wf_data, skipped_steps=["two"])
self.assertEqual(2, len(wf.steps))
self.assertEqual("one", wf.steps[0].id)
self.assertEqual("three", wf.steps[1].id)
# more than one
wf = WorkflowParser.parse(wf_data=wf_data,
skipped_steps=["one", "three"])
self.assertEqual(1, len(wf.steps))
self.assertEqual("two", wf.steps[0].id)
# non-existing name
self.assertRaises(SystemExit, WorkflowParser.parse, **{
"wf_data": wf_data,
"skipped_steps": ["four"]
})
# skip one step
wf = WorkflowParser.parse(wf_data=wf_data, skipped_steps=["two"])
self.assertEqual(2, len(wf.steps))
self.assertEqual("one", wf.steps[0].id)
self.assertEqual("three", wf.steps[1].id)
# without id
wf_data = {
"steps": [
{
"uses": "foo"
},
{
"uses": "bar"
},
{
"uses": "baz"
},
]
}
wf = WorkflowParser.parse(wf_data=wf_data, skipped_steps=["1", "3"])
self.assertEqual(1, len(wf.steps))
self.assertEqual("2", wf.steps[0].id)
def test_empty_file(self):
try:
f = open("test.yml", "a")
f.close()
WorkflowParser.parse(file="test.yml")
self.assertTrue(
False, "Empty Workflow file does not raise an exception.")
except SystemExit:
self.assertTrue(True)
else:
self.assertTrue(
False, "Empty Workflow file does not raise a SystemExit.")
def test_docker_basic_run(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
test_string = "STEP_INFO:popper:Successfully tagged popperized/bin:master"
with WorkflowRunner(conf) as r:
wf_data = {
"steps": [{
"uses": "popperized/bin/sh@master",
"args": ["ls"],
}]
}
with self.assertLogs(log, level="STEP_INFO") as cm:
r.run(WorkflowParser.parse(wf_data=wf_data))
self.assertTrue(test_string in cm.output,
f"Got cmd output: {cm.output}")
wf_data = {
"steps": [{
"uses": "docker://*****:*****@master",
"args": ["ls"],
}]
}
with self.assertLogs(log, level="STEP_INFO") as cm:
r.run(WorkflowParser.parse(wf_data=wf_data))
self.assertTrue(test_string not in cm.output)
repo.close()
shutil.rmtree(repo.working_dir, ignore_errors=True)
def test_exec_mpi_failure(self, mock_kill):
config_dict = {
"engine": {
"name": "singularity",
"options": {},
},
"resource_manager": {
"name": "slurm",
"options": {
"1": {
"nodes": 2,
"nodelist": "worker1,worker2",
"overcommit": True
}
},
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
self.Popen.set_command(
"sbatch "
"--wait --overcommit "
f"popper_1_{config.wid}.sh",
returncode=12,
)
self.Popen.set_command(f"tail -f popper_1_{config.wid}.out",
returncode=0)
with WorkflowRunner(config) as r:
wf_data = {"steps": [{"uses": "docker://alpine", "args": ["ls"]}]}
self.assertRaises(SystemExit, r.run,
WorkflowParser.parse(wf_data=wf_data))
def test_check_secrets(self):
wf_data = {
"steps": [{
"uses": "docker://alpine:3.9",
"args": ["ls", "-ltr"],
"secrets": ["SECRET_ONE", "SECRET_TWO"],
}]
}
wf = WorkflowParser.parse(wf_data=wf_data)
# in dry-run, secrets are ignored
runner = WorkflowRunner(ConfigLoader.load(dry_run=True))
runner._process_secrets(wf)
# now go back to not dry-running
runner = WorkflowRunner(ConfigLoader.load())
# when CI=true it should fail
os.environ["CI"] = "true"
self.assertRaises(SystemExit, runner._process_secrets, wf)
# add one secret
os.environ["SECRET_ONE"] = "1234"
# it should fail again, as we're missing one
self.assertRaises(SystemExit, runner._process_secrets, wf)
os.environ.pop("CI")
# now is fine
with patch("getpass.getpass", return_value="5678"):
runner._process_secrets(wf)
# pop the other
os.environ.pop("SECRET_ONE")
def test_clone_repos(self):
wf_data = {"steps": [{"uses": "popperized/bin/sh@master"}]}
wf = WorkflowParser.parse(wf_data=wf_data)
cache_dir = os.path.join(os.environ["HOME"], ".cache/popper/")
# clone repos in the default cache directory.
conf = ConfigLoader.load()
runner = WorkflowRunner(conf)
runner._clone_repos(wf)
step_dir = os.path.join(cache_dir, conf.wid,
"github.com/popperized/bin")
self.assertTrue(os.path.exists(step_dir))
# clone repos in custom cache directory
os.environ["POPPER_CACHE_DIR"] = "/tmp/smdir"
conf = ConfigLoader.load()
runner = WorkflowRunner(conf)
runner._clone_repos(wf)
step_dir = os.path.join("/tmp/smdir", conf.wid,
"github.com/popperized/bin")
self.assertTrue(os.path.exists(step_dir))
os.environ.pop("POPPER_CACHE_DIR")
# check failure when container is not available and we skip cloning
shutil.rmtree("/tmp/smdir")
shutil.rmtree(cache_dir)
conf = ConfigLoader.load(skip_clone=True)
runner = WorkflowRunner(conf)
self.assertRaises(SystemExit, runner._clone_repos, wf)
def cli(ctx, file, step, entrypoint, skip_pull, skip_clone, substitution,
conf):
"""Opens an interactive shell using all the attributes defined in the workflow file
for the given STEP, but ignoring ``runs`` and ``args``. By default, it invokes
/bin/bash. If you need to invoke another one, you can specify it in the --entrypoint
flag.
NOTE: this command only works for (local) host runner in Docker.
"""
wf = WorkflowParser.parse(file=file,
step=step,
immutable=False,
substitutions=substitution)
# override entrypoint
step = wf.steps[0]
step.args = []
step.runs = entrypoint
# configure runner so containers execute in attached mode and create a tty
config = ConfigLoader.load(
engine_name="docker",
pty=True,
skip_pull=skip_pull,
skip_clone=skip_clone,
config_file=conf,
)
with WorkflowRunner(config) as runner:
try:
runner.run(wf)
except Exception as e:
log.debug(traceback.format_exc())
log.fail(e)
def test_new_workflow(self):
wf_data = {}
self.assertRaises(SystemExit, WorkflowParser.parse,
**{"wf_data": wf_data})
wf_data = {"unexpected": []}
self.assertRaises(SystemExit, WorkflowParser.parse,
**{"wf_data": wf_data})
wf_data = {
"steps": [
{
"uses": "foo",
"id": "step",
"env": {
"EN": "EE"
},
"secrets": ["S"],
"dir": "/path/to/",
"options": {
"name": "spam"
},
},
{
"uses": "bar",
"runs": ["a", "b"],
"args": ["c"],
"skip_pull": True
},
],
"options": {
"env": {
"FOO": "bar"
},
"secrets": ["Z"],
},
}
wf = WorkflowParser.parse(wf_data=wf_data)
step = wf.steps[0]
self.assertEqual("step", step.id)
self.assertEqual("foo", step.uses)
self.assertEqual(("Z", "S"), step.secrets)
self.assertEqual({"EN": "EE", "FOO": "bar"}, step.env)
self.assertEqual("/path/to/", step.dir)
self.assertEqual("spam", step.options.name)
self.assertTrue(not step.runs)
self.assertTrue(not step.args)
self.assertFalse(step.skip_pull)
step = wf.steps[1]
self.assertEqual("bar", step.uses)
self.assertEqual(("a", "b"), step.runs)
self.assertEqual(("c", ), step.args)
self.assertTrue(step.skip_pull)
self.assertTrue(not step.dir)
self.assertEqual({"FOO": "bar"}, step.env)
self.assertEqual(("Z", ), step.secrets)
self.assertEqual({"FOO": "bar"}, wf.options.env)
self.assertEqual(("Z", ), wf.options.secrets)
def test_filter_all_but_given_step(self):
wf_data = {
"steps": [
{
"uses": "foo",
"id": "one"
},
{
"uses": "bar",
"id": "two"
},
{
"uses": "baz",
"id": "three"
},
]
}
wf = WorkflowParser.parse(wf_data=wf_data, step="two")
self.assertEqual(1, len(wf.steps))
self.assertEqual("two", wf.steps[0].id)
self.assertEqual("bar", wf.steps[0].uses)
# non-existing name
self.assertRaises(SystemExit, WorkflowParser.parse, **{
"wf_data": wf_data,
"step": "four"
})
# without id
wf_data = {
"steps": [
{
"uses": "foo"
},
{
"uses": "bar"
},
{
"uses": "baz"
},
]
}
wf = WorkflowParser.parse(wf_data=wf_data, step="2")
self.assertEqual(1, len(wf.steps))
self.assertEqual("2", wf.steps[0].id)
def test_host_run(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with WorkflowRunner(conf) as r:
wf_data = {
"steps": [{"uses": "sh", "runs": ["cat"], "args": ["README.md"],}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
wf_data = {
"steps": [
{
"uses": "sh",
"runs": ["bash", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {"FOO": "bar"},
}
]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(os.path.join(repo.working_dir, "hello.txt"), "r") as f:
self.assertEqual(f.read(), "bar\n")
wf_data = {"steps": [{"uses": "sh", "runs": ["nocommandisnamedlikethis"]}]}
self.assertRaises(SystemExit, r.run, WorkflowParser.parse(wf_data=wf_data))
# check exit code 78
wf_data = {
"steps": [
{"uses": "sh", "runs": ["touch", "one.txt"]},
{"uses": "sh", "runs": ["bash", "-c", "exit 78"]},
{"uses": "sh", "runs": ["touch", "two.txt"]},
]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
self.assertTrue(os.path.isfile(os.path.join(repo.working_dir, "one.txt")))
self.assertFalse(os.path.isfile(os.path.join(repo.working_dir, "two.txt")))
repo.close()
shutil.rmtree(repo.working_dir, ignore_errors=True)
def test_run(self, mock_kill):
config_dict = {
"engine": {
"name": "podman",
"options": {
"privileged": True,
"hostname": "popper.local",
"domainname": "www.example.org",
"volumes": ["/path/in/host:/path/in/container"],
"environment": {
"FOO": "bar"
},
},
},
"resource_manager": {
"name": "slurm"
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
self.Popen.set_command(
f"sbatch --wait --job-name popper_1_{config.wid} "
f"--output {slurm_cache_dir}/popper_1_{config.wid}.out "
f"{slurm_cache_dir}/popper_1_{config.wid}.sh",
returncode=0,
)
self.Popen.set_command(
f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out",
returncode=0)
with WorkflowRunner(config) as r:
wf_data = {
"steps": [{
"uses": "popperized/bin/sh@master",
"runs": ["cat"],
"args": ["README.md"],
}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(f"{slurm_cache_dir}/popper_1_{config.wid}.sh", "r") as f:
# fmt: off
expected = f"""#!/bin/bash
podman rm -f popper_1_{config.wid} || true
podman build -t popperized/bin:master {os.environ['HOME']}/.cache/popper/{config.wid}/github.com/popperized/bin/sh
podman create --name popper_1_{config.wid} --workdir /workspace --entrypoint cat -v /w:/workspace:Z -v /path/in/host:/path/in/container -e FOO=bar --privileged --hostname popper.local --domainname www.example.org popperized/bin:master README.md
podman start --attach popper_1_{config.wid}"""
# fmt: on
actual = f.read()
self.maxDiff = None
self.assertEqual(expected, actual)
def test_dry_run(self):
config = ConfigLoader.load(engine_name="singularity",
resman_name="slurm",
dry_run=True)
with WorkflowRunner(config) as r:
wf_data = {
"steps": [{
"uses": "docker://alpine",
"runs": ["cat"],
"args": ["README.md"],
}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
self.assertEqual(self.Popen.all_calls, [])
def cli(ctx, file, skip, colors):
"""Creates a graph in the .dot format representing the workflow."""
wf = WorkflowParser.parse(file=file)
node_attrs = 'shape=box, style="filled{}", fillcolor=transparent{}'
wf_attr = node_attrs.format(",rounded", ",color=red" if colors else "")
act_attr = node_attrs.format("", ",color=cyan" if colors else "")
dot_str = ""
dot_str += f' "Workflow" [{wf_attr}];\n'
for i, step in enumerate(wf.steps):
n = wf.steps[i]["id"]
dot_str += f' "{n}" [{act_attr}];\n'
if i == 0:
parent = "Workflow"
else:
parent = wf.steps[i - 1]["id"]
dot_str += f' "{parent}" -> "{n}";\n'
log.info("digraph G { graph [bgcolor=transparent];\n" + dot_str + "}\n")
def test_slurm_singularity_run(self, mock_kill):
config_dict = {
"engine": {
"name": "singularity",
"options": {
"hostname": "popper.local",
"bind": ["/path/in/host:/path/in/container"],
},
},
"resource_manager": {
"name": "slurm"
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
# fmt: off
self.Popen.set_command(
f"sbatch --wait --job-name popper_1_{config.wid} --output {slurm_cache_dir}/popper_1_{config.wid}.out {slurm_cache_dir}/popper_1_{config.wid}.sh",
returncode=0,
)
# fmt: on
self.Popen.set_command(
f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out",
returncode=0)
with WorkflowRunner(config) as r:
wf_data = {
"steps": [{
"uses": "popperized/bin/sh@master",
"args": ["ls"],
}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(f"{slurm_cache_dir}/popper_1_{config.wid}.sh", "r") as f:
# fmt: off
expected = f"""#!/bin/bash
singularity run --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local {os.environ['HOME']}/.cache/popper/singularity/{config.wid}/popper_1_{config.wid}.sif ls"""
# fmt: on
actual = f.read()
self.assertEqual(expected, actual)
def test_substitutions(self):
# test wrong format for substitution key
wf_data = {"steps": [{"uses": "whatever"}]}
self.assertRaises(
SystemExit, WorkflowParser.parse, **{
"wf_data": wf_data,
"substitutions": ["SUB1=WRONG"]
})
# expect error when not all given subs are used
wf_data = {
"steps": [{
"uses": "some_$_SUB1",
"id": "some other $_SUB2",
"env": {
"FOO": "env_$_SUB3"
},
"secrets": ["secret_$_SUB4"],
}]
}
substitutions = [
"_SUB1=ONE",
"_SUB2=TWO",
"_SUB3=THREE",
"_SUB4=4",
"_SUB5=UNUSED",
]
self.assertRaises(
SystemExit, WorkflowParser.parse, **{
"wf_data": wf_data,
"substitutions": substitutions
})
# allow loose substitutions
wf = WorkflowParser.parse(wf_data=wf_data,
substitutions=substitutions,
allow_loose=True)
step = wf.steps[0]
self.assertEqual("some_ONE", step.uses)
self.assertEqual("some other TWO", step.id)
self.assertEqual("env_THREE", step.env["FOO"])
self.assertEqual(("secret_4", ), step.secrets)
def test_run(self, mock_kill):
self.maxDiff = None
config_dict = {
"engine": {
"name": "singularity",
"options": {
"hostname": "popper.local",
"bind": ["/path/in/host:/path/in/container"],
},
},
"resource_manager": {
"name": "slurm",
"options": {
"1": {
"nodes": 2,
"ntasks": 2,
"nodelist": "worker1,worker2"
}
},
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
# fmt: off
self.Popen.set_command(
"sbatch "
"--wait "
f"popper_1_{config.wid}.sh",
returncode=0,
)
# fmt: on
self.Popen.set_command(f"tail -f popper_1_{config.wid}.out",
returncode=0)
with WorkflowRunner(config) as r:
wf_data = {"steps": [{"uses": "docker://alpine", "args": ["ls"]}]}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(f"popper_1_{config.wid}.sh", "r") as f:
# fmt: off
expected = f"""#!/bin/bash
#SBATCH --job-name=popper_1_{config.wid}
#SBATCH --output=popper_1_{config.wid}.out
#SBATCH --nodes=2
#SBATCH --ntasks=2
#SBATCH --ntasks-per-node=1
#SBATCH --nodelist=worker1,worker2
mpirun singularity run --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local {os.environ['HOME']}/.cache/popper/singularity/{config.wid}/popper_1_{config.wid}.sif ls"""
# fmt: on
actual = f.read()
self.assertEqual(expected, actual)
config_dict = {
"engine": {
"name": "singularity",
"options": {
"hostname": "popper.local",
"bind": ["/path/in/host:/path/in/container"],
},
},
"resource_manager": {
"name": "slurm",
"options": {
"1": {
"mpi": False,
"nodes": 2,
"ntasks": 2,
"nodelist": "worker1,worker2",
}
},
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
self.Popen.set_command(
f"srun --nodes 2 --ntasks 2 --ntasks-per-node 1 --nodelist worker1,worker2 rm -rf popper_1_{config.wid}.sif",
returncode=0,
)
self.Popen.set_command(
f"srun --nodes 2 --ntasks 2 --ntasks-per-node 1 --nodelist worker1,worker2 singularity pull docker://alpine:latest",
returncode=0,
)
self.Popen.set_command(
f"srun --nodes 2 --ntasks 2 --ntasks-per-node 1 --nodelist worker1,worker2 singularity run --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local {os.environ['HOME']}/.cache/popper/singularity/{config.wid}/popper_1_{config.wid}.sif ls",
returncode=0,
)
with WorkflowRunner(config) as r:
wf_data = {"steps": [{"uses": "docker://alpine", "args": ["ls"]}]}
r.run(WorkflowParser.parse(wf_data=wf_data))
def test_submit_job_failure(self, mock_kill):
config_dict = {
"engine": {
"name": "docker",
"options": {}
},
"resource_manager": {
"name": "slurm",
"options": {}
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
self.Popen.set_command(
f"sbatch --wait --job-name popper_1_{config.wid} "
f"--output {slurm_cache_dir}/popper_1_{config.wid}.out "
f"{slurm_cache_dir}/popper_1_{config.wid}.sh",
returncode=12,
)
self.Popen.set_command(
f"tail -f {slurm_cache_dir}/popper_1_{config.wid}.out",
returncode=0)
with WorkflowRunner(config) as r:
wf_data = {
"steps": [{
"uses": "popperized/bin/sh@master",
"runs": ["cat"],
"args": ["README.md"],
}]
}
self.assertRaises(SystemExit, r.run,
WorkflowParser.parse(wf_data=wf_data))
call_tail = call.Popen(
["tail", "-f", f"{slurm_cache_dir}/popper_1_{config.wid}.out"],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
)
call_sbatch = call.Popen(
[
"sbatch",
"--wait",
"--job-name",
f"popper_1_{config.wid}",
"--output",
f"{slurm_cache_dir}/popper_1_{config.wid}.out",
f"{slurm_cache_dir}/popper_1_{config.wid}.sh",
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_singularity_start(self):
repo = self.mk_repo()
conf = ConfigLoader.load(engine_name="singularity",
workspace_dir=repo.working_dir)
step = Box(
{
"uses": "docker://*****:*****@master",
"args": ["ls"],
}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
wf_data = {
"steps": [{
"uses": "docker://alpine:3.9",
"args": ["sh", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {
"FOO": "bar"
},
}]
}
r.run(WorkflowParser.parse(wf_data=wf_data))
with open(os.path.join(repo.working_dir, "hello.txt"), "r") as f:
self.assertEqual(f.read(), "bar\n")
wf_data = {
"steps": [{
"uses": "docker://alpine:3.9",
"args": ["nocommandisnamedlikethis"],
}]
}
self.assertRaises(SystemExit, r.run,
WorkflowParser.parse(wf_data=wf_data))
repo.close()
def test_substitutions(self):
# test wrong format for substitution key
wf_data = {"steps": [{"uses": "whatever"}]}
self.assertRaises(
SystemExit,
WorkflowParser.parse,
**{"wf_data": wf_data, "substitutions": ["SUB1=WRONG"]}
)
# allow non-str values
wf_data = {
"steps": [
{
"uses": "some_image",
"id": "some_$_DUMMY",
"env": {"FOO": "BAR"},
"secrets": ["a-secret"],
"options": {"auto_remove": True},
}
]
}
substitutions = [
"_DUMMY=dummy",
]
wf = WorkflowParser.parse(
wf_data=wf_data, substitutions=substitutions, allow_loose=True
)
step = wf.steps[0]
self.assertEqual("some_image", step.uses)
self.assertEqual("some_dummy", step.id)
self.assertEqual("BAR", step.env["FOO"])
self.assertEqual(("a-secret",), step.secrets)
self.assertEqual({"auto_remove": True}, step.options)
# expect error when not all given subs are used
wf_data = {
"steps": [
{
"uses": "some_$_SUB1",
"id": "some other $_SUB2",
"env": {"FOO": "env_$_SUB3"},
"secrets": ["secret_$_SUB4"],
}
]
}
substitutions = [
"_SUB1=ONE",
"_SUB2=TWO",
"_SUB3=THREE",
"_SUB4=4",
"_SUB5=UNUSED",
]
self.assertRaises(
SystemExit,
WorkflowParser.parse,
**{"wf_data": wf_data, "substitutions": substitutions}
)
# substitute nested
wf_data = {
"steps": [
{
"uses": "some_$_SUB1",
"id": "some other $_SUB2",
"env": {"FOO": "env_$_SUB3"},
"secrets": ["secret_$_SUB4"],
"options": {"labels": {"timestamp": "$_TIMESTAMP"}},
}
]
}
substitutions = [
"_TIMESTAMP=1613916937",
"_SUB1=ONE",
"_SUB2=TWO",
"_SUB3=THREE",
"_SUB4=4",
]
wf = WorkflowParser.parse(wf_data=wf_data, substitutions=substitutions)
step = wf.steps[0]
self.assertEqual("some_ONE", step.uses)
self.assertEqual("some other TWO", step.id)
self.assertEqual("env_THREE", step.env["FOO"])
self.assertEqual(("secret_4",), step.secrets)
self.assertEqual({"timestamp": "1613916937"}, step.options.labels)
# allow loose substitutions
wf = WorkflowParser.parse(
wf_data=wf_data, substitutions=substitutions, allow_loose=True
)
step = wf.steps[0]
self.assertEqual("some_ONE", step.uses)
self.assertEqual("some other TWO", step.id)
self.assertEqual("env_THREE", step.env["FOO"])
self.assertEqual(("secret_4",), step.secrets)
def test_add_missing_ids(self):
wf_data = {"steps": [{"uses": "foo"}, {"uses": "bar"}]}
# skip one step
wf = WorkflowParser.parse(wf_data=wf_data)
self.assertEqual("1", wf.steps[0].id)
self.assertEqual("2", wf.steps[1].id)
def cli(
ctx,
step,
wfile,
debug,
dry_run,
log_file,
quiet,
reuse,
engine,
resource_manager,
skip,
skip_pull,
skip_clone,
substitution,
allow_loose,
workspace,
conf,
):
"""Runs a Popper workflow. Only executes STEP if given.
To specify a container engine to use other than docker, use the --engine/-e
flag. For executing on a resource manager such as SLURM or Kubernetes, use
the --resource-manager/-r flag. Alternatively, a configuration file can be
given (--conf flag) that can specify container options, resource manager
options, or both (see "Workflow Syntax and Execution Runtime" section of
the Popper documentation for more).
If the container engine (-e) or resource manager (-r) are specified with a
flag and a configuration file is given as well, the values passed via the
flags are given preference over those contained in the configuration file.
"""
# set the logging levels.
level = "STEP_INFO"
if quiet:
level = "INFO"
if debug:
level = "DEBUG"
log.setLevel(level)
if dry_run:
logging.msg_prefix = "DRYRUN: "
if log_file:
# also log to a file
logging.add_log(log, log_file)
# check conflicting flags and fail if needed
if skip and step:
log.fail("`--skip` can not be used when STEP argument is passed.")
# invoke wf factory; handles formats, validations, filtering
wf = WorkflowParser.parse(
wfile,
step=step,
skipped_steps=skip,
substitutions=substitution,
allow_loose=allow_loose,
)
config = ConfigLoader.load(
engine_name=engine,
resman_name=resource_manager,
config_file=conf,
reuse=reuse,
dry_run=dry_run,
skip_pull=skip_pull,
skip_clone=skip_clone,
workspace_dir=workspace,
)
with WorkflowRunner(config) as runner:
try:
runner.run(wf)
except Exception as e:
log.debug(traceback.format_exc())
log.fail(e)
