def test_clone_repos(self):
wf = YMLWorkflow("""
version: '1'
steps:
- uses: popperized/bin/sh@master
""")
wf.parse()
conf = PopperConfig()
cache_dir = os.path.join(os.environ['HOME'], '.cache/popper/')
# clone repos in the default cache directory.
runner = WorkflowRunner(conf)
runner._clone_repos(wf)
step_dir = os.path.join(cache_dir, conf.wid,
'github.com/popperized/bin')
self.assertTrue(os.path.exists(step_dir))
# clone repos in custom cache directory
os.environ['POPPER_CACHE_DIR'] = '/tmp/smdir'
runner._clone_repos(wf)
step_dir = os.path.join('/tmp/smdir', conf.wid,
'github.com/popperized/bin')
self.assertTrue(os.path.exists(step_dir))
os.environ.pop('POPPER_CACHE_DIR')
# check failure when container is not available and we skip cloning
shutil.rmtree('/tmp/smdir')
shutil.rmtree(cache_dir)
conf = PopperConfig(skip_clone=True)
runner = WorkflowRunner(conf)
self.assertRaises(SystemExit, runner._clone_repos, wf)
def test_check_secrets(self):
wf = YMLWorkflow("""
version: '1'
steps:
- uses: docker://alpine:3.9
args: ["ls -ltr"]
secrets: ["SECRET_ONE", "SECRET_TWO"]
""")
wf.parse()
# in dry-run, secrets are ignored
runner = WorkflowRunner(PopperConfig(dry_run=True))
runner._process_secrets(wf)
# now go back to not dry-running
runner = WorkflowRunner(PopperConfig())
# when CI=true it should fail
os.environ['CI'] = 'true'
self.assertRaises(SystemExit, runner._process_secrets, wf)
# add one secret
os.environ['SECRET_ONE'] = '1234'
# it should fail again, as we're missing one
self.assertRaises(SystemExit, runner._process_secrets, wf)
os.environ.pop('CI')
# now is fine
with patch('getpass.getpass', return_value='5678'):
runner._process_secrets(wf)
# pop the other
os.environ.pop('SECRET_ONE')
def test_setup_singularity_cache(self):
config = PopperConfig()
config.wid = "abcd"
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
self.assertEqual(
f'{os.environ["HOME"]}/.cache/popper/singularity/abcd',
sr._singularity_cache)
def test_submit_job_failure(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=12)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'docker',
'options': {}
},
'resource_manager': {
'name': 'slurm',
'options': {}
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
self.assertRaises(SystemExit, r.run, wf)
call_tail = call.Popen(
['tail', '-f', '/tmp/popper/slurm/popper_1_123abc.out'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
call_sbatch = call.Popen([
'sbatch', '--wait', '--job-name', 'popper_1_123abc',
'--output', '/tmp/popper/slurm/popper_1_123abc.out',
'/tmp/popper/slurm/popper_1_123abc.sh'
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_run(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=0)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'docker',
'options': {
'privileged': True,
'hostname': 'popper.local',
'domainname': 'www.example.org',
'volumes': ['/path/in/host:/path/in/container'],
'environment': {
'FOO': 'bar'
}
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
r.run(wf)
with open('/tmp/popper/slurm/popper_1_123abc.sh', 'r') as f:
content = f.read()
self.assertEqual(
content, f"""#!/bin/bash
docker rm -f popper_1_123abc || true
docker build -t popperized/bin:master {os.environ['HOME']}/.cache/popper/123abc/github.com/popperized/bin/sh
docker create --name popper_1_123abc --workdir /workspace --entrypoint cat -v /w:/workspace -v /var/run/docker.sock:/var/run/docker.sock -v /path/in/host:/path/in/container -e FOO=bar --privileged --hostname popper.local --domainname www.example.org popperized/bin:master README.md
docker start --attach popper_1_123abc""")
def test_create_cmd(self):
config = {'workspace_dir': '/w'}
with DockerRunner(config=PopperConfig(**config)) as drunner:
step = {'args': ['-two', '-flags']}
cmd = drunner._create_cmd(step, 'foo:1.9', 'container_name')
expected = ('docker create'
' --name container_name'
' --workdir /workspace'
' -v /w:/workspace'
' -v /var/run/docker.sock:/var/run/docker.sock'
' foo:1.9 -two -flags')
self.assertEqual(expected, cmd)
config_dict = {
'engine': {
'name': 'docker',
'options': {
'privileged': True,
'hostname': 'popper.local',
'domainname': 'www.example.org',
'volumes': ['/path/in/host:/path/in/container'],
'environment': {
'FOO': 'bar'
}
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = {'workspace_dir': '/w', 'config_file': config_dict}
with DockerRunner(config=PopperConfig(**config)) as drunner:
step = {'args': ['-two', '-flags']}
cmd = drunner._create_cmd(step, 'foo:1.9', 'container_name')
expected = ('docker create --name container_name '
'--workdir /workspace '
'-v /w:/workspace '
'-v /var/run/docker.sock:/var/run/docker.sock '
'-v /path/in/host:/path/in/container '
'-e FOO=bar --privileged --hostname popper.local '
'--domainname www.example.org '
'foo:1.9 -two -flags')
self.assertEqual(expected, cmd)
def test_steprunner_factory(self):
with WorkflowRunner(PopperConfig()) as r:
self.assertEqual(
r._step_runner('host', None).__class__.__name__, 'HostRunner')
self.assertEqual(
r._step_runner('docker', None).__class__.__name__,
'DockerRunner')
def test_config_defaults(self):
conf = PopperConfig()
actual = conf.__dict__
expected = TestPopperConfig.default_args
self.assertEqual(expected,
TestPopperConfig.extract_dict(expected, actual))
def test_submit_batch_job(self, mock_kill):
self.Popen.set_command(
'sbatch --wait '
'--job-name popper_sample_123abc '
'--output /tmp/popper/slurm/popper_sample_123abc.out '
'/tmp/popper/slurm/popper_sample_123abc.sh',
returncode=0)
self.Popen.set_command(
'tail -f /tmp/popper/slurm/popper_sample_123abc.out', returncode=0)
config = PopperConfig(workspace_dir='/w')
config.wid = "123abc"
step = {"name": "sample"}
with SlurmRunner(config=config) as sr:
sr._submit_batch_job(["ls -la"], step)
with open("/tmp/popper/slurm/popper_sample_123abc.sh", 'r') as f:
content = f.read()
self.assertEqual(content, "#!/bin/bash\nls -la")
self.assertEqual(len(sr._spawned_jobs), 0)
self.assertEqual(sr._out_stream_thread.is_alive(), False)
call_tail = call.Popen(
['tail', '-f', '/tmp/popper/slurm/popper_sample_123abc.out'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
call_sbatch = call.Popen([
'sbatch', '--wait', '--job-name', 'popper_sample_123abc',
'--output', '/tmp/popper/slurm/popper_sample_123abc.out',
'/tmp/popper/slurm/popper_sample_123abc.sh'
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_run(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=0)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'singularity',
'options': {
'hostname': 'popper.local',
'bind': ['/path/in/host:/path/in/container']
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: ls
""")
wf.parse()
r.run(wf)
with open('/tmp/popper/slurm/popper_1_123abc.sh', 'r') as f:
content = f.read()
self.assertEqual(
content, f"""#!/bin/bash
singularity exec --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local {os.environ['HOME']}/.cache/popper/singularity/123abc/popper_1_123abc.sif ls"""
)
def test_create_container(self):
config = PopperConfig()
step = {
'uses': 'docker://alpine:3.9',
'runs': ['echo hello'],
'name': 'kontainer_one'
}
cid = pu.sanitized_name(step['name'], config.wid)
with DockerRunner(init_docker_client=True, config=config) as dr:
c = dr._create_container(cid, step)
self.assertEqual(c.status, 'created')
c.remove()
def test_config_from_file(self):
config = {
'engine': {
'options': {
'privileged': True
}
},
'resource_manager': {
'options': {
'foo': 'bar'
}
}
}
kwargs = {'config_file': config}
# engine name missing
with self.assertLogs('popper', level='INFO') as cm:
self.assertRaises(SystemExit, PopperConfig, **kwargs)
self.assertEqual(len(cm.output), 1)
self.assertTrue('No engine name given' in cm.output[0])
# resman name missing
config.update({'engine': {'name': 'foo'}})
with self.assertLogs('popper', level='INFO') as cm:
self.assertRaises(SystemExit, PopperConfig, **kwargs)
self.assertEqual(len(cm.output), 1)
self.assertTrue('No resource manager name given' in cm.output[0])
# now all OK
config.update({'resource_manager': {'name': 'bar'}})
conf = PopperConfig(**kwargs)
self.assertEqual(conf.engine_name, 'foo')
self.assertEqual(conf.resman_name, 'bar')
self.assertEqual(conf.engine_opts, {})
self.assertEqual(conf.resman_opts, {})
config.update({'engine': {'name': 'bar', 'options': {'foo': 'baz'}}})
conf = PopperConfig(**kwargs)
self.assertEqual(conf.engine_opts, {'foo': 'baz'})
def test_get_container_options(self):
config_dict = {
'engine': {
'name': 'singularity',
'options': {
'hostname': 'popper.local',
'ipc': True,
'bind': ['/path/in/host:/path/in/container']
}
}
}
config = PopperConfig(config_file=config_dict)
config.wid = "abcd"
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
options = sr._get_container_options()
self.assertEqual(options, [
'--userns', '--pwd', '/workspace', '--bind',
f'{os.getcwd()}:/workspace', '--bind',
'/path/in/host:/path/in/container', '--hostname',
'popper.local', '--ipc'
])
def test_config_non_defaults(self):
expected = {
'skip_clone': True,
'skip_pull': True,
'dry_run': True,
'workspace_dir': os.path.realpath('/tmp/foo'),
'quiet': True,
'reuse': True
}
conf = PopperConfig(**expected)
actual = conf.__dict__
self.assertEqual(expected,
TestPopperConfig.extract_dict(expected, actual))
def test_stop_running_tasks(self):
self.Popen.set_command('scancel --name job_a', returncode=0)
with SlurmRunner(config=PopperConfig()) as sr:
sr._spawned_jobs.add('job_a')
sr.stop_running_tasks()
self.assertEqual(
call.Popen(['scancel', '--name', 'job_a'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True) in self.Popen.all_calls,
True)
def test_create_container(self):
config = PopperConfig()
config.wid = "abcd"
step_one = {
'uses': 'docker://*****:*****@master',
'args': ['ls'],
'name': 'kontainer_two',
'repo_dir':
f'{os.environ["HOME"]}/.cache/popper/abcd/github.com/popperized/bin',
'step_dir': 'sh'
}
cid_one = pu.sanitized_name(step_one['name'], config.wid)
cid_two = pu.sanitized_name(step_two['name'], config.wid)
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
c_one = sr._create_container(step_one, cid_one)
self.assertEqual(
os.path.exists(os.path.join(sr._singularity_cache, cid_one)),
True)
os.remove(os.path.join(sr._singularity_cache, cid_one))
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
c_two = sr._create_container(step_one, cid_two)
self.assertEqual(
os.path.exists(os.path.join(sr._singularity_cache, cid_two)),
True)
os.remove(os.path.join(sr._singularity_cache, cid_two))
def test_create_cmd(self):
config = PopperConfig(workspace_dir='/w')
config.wid = "abcd"
with SingularityRunner(config=config) as sr:
step = {'args': ['-two', '-flags']}
sr._setup_singularity_cache()
sr._container = os.path.join(sr._singularity_cache, 'c1.sif')
cmd = sr._create_cmd(step, 'c1.sif')
expected = (
'singularity run'
' --userns --pwd /workspace'
' --bind /w:/workspace'
f' {os.environ["HOME"]}/.cache/popper/singularity/abcd/c1.sif'
' -two -flags')
self.assertEqual(expected, cmd)
config_dict = {
'engine': {
'name': 'singularity',
'options': {
'hostname': 'popper.local',
'ipc': True,
'bind': ['/path/in/host:/path/in/container']
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "abcd"
with SingularityRunner(config=config) as sr:
step = {'args': ['-two', '-flags']}
sr._setup_singularity_cache()
sr._container = os.path.join(sr._singularity_cache, 'c2.sif')
cmd = sr._create_cmd(step, 'c2.sif')
expected = (
'singularity run --userns --pwd /workspace'
' --bind /w:/workspace'
' --bind /path/in/host:/path/in/container'
' --hostname popper.local'
' --ipc'
f' {os.environ["HOME"]}/.cache/popper/singularity/abcd/c2.sif'
' -two -flags')
self.assertEqual(expected, cmd)
def test_dry_run(self):
repo = self.mk_repo()
config = PopperConfig(engine_name='docker',
resman_name='slurm',
dry_run=True,
workspace_dir=repo.working_dir)
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
r.run(wf)
self.assertEqual(self.Popen.all_calls, [])
def test_run(self):
repo = self.mk_repo()
conf = PopperConfig(workspace_dir=repo.working_dir)
with WorkflowRunner(conf) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: sh
runs: [cat]
args: README.md
""")
wf.parse()
r.run(wf)
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'sh'
runs: ['bash', '-c', 'echo $FOO > hello.txt ; pwd']
env: {
FOO: bar
}
""")
wf.parse()
r.run(wf)
with open(os.path.join(repo.working_dir, 'hello.txt'), 'r') as f:
self.assertEqual(f.read(), 'bar\n')
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'sh'
runs: 'nocommandisnamedlikethis'
""")
wf.parse()
self.assertRaises(SystemExit, r.run, wf)
repo.close()
def test_get_container_kwargs(self):
step = {
'uses': 'popperized/bin/sh@master',
'args': ['ls'],
'name': 'one',
'repo_dir': '/path/to/repo/dir',
'step_dir': 'sh'
}
config_dict = {
'engine': {
'name': 'docker',
'options': {
'privileged': True,
'hostname': 'popper.local',
'domainname': 'www.example.org',
'volumes': ['/path/in/host:/path/in/container'],
'environment': {
'FOO': 'bar'
}
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(config_file=config_dict,
workspace_dir='/path/to/workdir')
with DockerRunner(init_docker_client=False, config=config) as dr:
args = dr._get_container_kwargs(step, 'alpine:3.9', 'container_a')
self.assertEqual(
args, {
'image':
'alpine:3.9',
'command': ['ls'],
'name':
'container_a',
'volumes': [
'/path/to/workdir:/workspace',
'/var/run/docker.sock:/var/run/docker.sock',
'/path/in/host:/path/in/container'
],
'working_dir':
'/workspace',
'environment': {
'FOO': 'bar'
},
'entrypoint':
None,
'detach':
True,
'privileged':
True,
'hostname':
'popper.local',
'domainname':
'www.example.org'
})
def test_tail_output(self):
self.Popen.set_command('tail -f slurm-x.out', returncode=0)
with SlurmRunner(config=PopperConfig()) as sr:
self.assertEqual(sr._tail_output('slurm-x.out'), 0)
self.assertEqual(len(sr._out_stream_pid), 1)
def test_singularity_start(self):
repo = self.mk_repo()
conf = PopperConfig(engine_name='singularity',
workspace_dir=repo.working_dir)
step = {
'uses': 'docker://*****:*****@master'
args: 'ls'
""")
wf.parse()
r.run(wf)
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'docker://alpine:3.9'
runs: ['sh', '-c', 'echo $FOO > hello.txt ; pwd']
env: {
FOO: bar
}
""")
wf.parse()
r.run(wf)
with open(os.path.join(repo.working_dir, 'hello.txt'), 'r') as f:
self.assertEqual(f.read(), 'bar\n')
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'docker://alpine:3.9'
runs: 'nocommandisnamedlikethis'
""")
wf.parse()
self.assertRaises(SystemExit, r.run, wf)
repo.close()
def __init__(self, config=PopperConfig()):
self._config = config
def cli(ctx, step, wfile, debug, dry_run, log_file, quiet, reuse, engine,
resource_manager, skip, skip_pull, skip_clone, substitution,
allow_loose, with_dependencies, workspace, conf):
"""Runs a Popper workflow. Only executes STEP if given.
To specify a container engine to use other than docker, use the --engine/-e
flag. For executing on a resource manager such as SLURM or Kubernetes, use
the --resource-manager/-r flag. Alternatively, a configuration file can be
given (--conf flag) that can specify container options, resource manager
options, or both (see "Workflow Syntax and Execution Runtime" section of
the Popper documentation for more).
If the container engine (-e) or resource manager (-r) are specified with a
flag and a configuration file is given as well, the values passed via the
flags are given preference over those contained in the configuration file.
"""
# set the logging levels.
level = 'STEP_INFO'
if quiet:
level = 'INFO'
if debug:
level = 'DEBUG'
log.setLevel(level)
if dry_run:
logging.msg_prefix = "DRYRUN: "
if log_file:
# also log to a file
logging.add_log(log, log_file)
# check conflicting flags and fail if needed
if with_dependencies and not step:
log.fail('`--with-dependencies` can only be used when '
'STEP argument is given.')
if skip and step:
log.fail('`--skip` can not be used when STEP argument is passed.')
# invoke wf factory; handles formats, validations, filtering
wf = Workflow.new(wfile,
step=step,
skipped_steps=skip,
substitutions=substitution,
allow_loose=allow_loose,
include_step_dependencies=with_dependencies)
config = PopperConfig(engine_name=engine,
resman_name=resource_manager,
config_file=conf,
reuse=reuse,
dry_run=dry_run,
skip_pull=skip_pull,
skip_clone=skip_clone,
workspace_dir=workspace)
runner = WorkflowRunner(config)
try:
runner.run(wf)
except Exception as e:
log.debug(traceback.format_exc())
log.fail(e)