def test_create_cmd(self):
config = PopperConfig(workspace_dir='/w')
config.wid = "abcd"
with SingularityRunner(config=config) as sr:
step = {'args': ['-two', '-flags']}
sr._setup_singularity_cache()
sr._container = os.path.join(sr._singularity_cache, 'c1.sif')
cmd = sr._create_cmd(step, 'c1.sif')
expected = (
'singularity run'
' --userns --pwd /workspace'
' --bind /w:/workspace'
f' {os.environ["HOME"]}/.cache/popper/singularity/abcd/c1.sif'
' -two -flags')
self.assertEqual(expected, cmd)
config_dict = {
'engine': {
'name': 'singularity',
'options': {
'hostname': 'popper.local',
'ipc': True,
'bind': ['/path/in/host:/path/in/container']
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "abcd"
with SingularityRunner(config=config) as sr:
step = {'args': ['-two', '-flags']}
sr._setup_singularity_cache()
sr._container = os.path.join(sr._singularity_cache, 'c2.sif')
cmd = sr._create_cmd(step, 'c2.sif')
expected = (
'singularity run --userns --pwd /workspace'
' --bind /w:/workspace'
' --bind /path/in/host:/path/in/container'
' --hostname popper.local'
' --ipc'
f' {os.environ["HOME"]}/.cache/popper/singularity/abcd/c2.sif'
' -two -flags')
self.assertEqual(expected, cmd)
def test_setup_singularity_cache(self):
config = PopperConfig()
config.wid = "abcd"
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
self.assertEqual(
f'{os.environ["HOME"]}/.cache/popper/singularity/abcd',
sr._singularity_cache)
def test_submit_job_failure(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=12)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'docker',
'options': {}
},
'resource_manager': {
'name': 'slurm',
'options': {}
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
self.assertRaises(SystemExit, r.run, wf)
call_tail = call.Popen(
['tail', '-f', '/tmp/popper/slurm/popper_1_123abc.out'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
call_sbatch = call.Popen([
'sbatch', '--wait', '--job-name', 'popper_1_123abc',
'--output', '/tmp/popper/slurm/popper_1_123abc.out',
'/tmp/popper/slurm/popper_1_123abc.sh'
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_run(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=0)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'docker',
'options': {
'privileged': True,
'hostname': 'popper.local',
'domainname': 'www.example.org',
'volumes': ['/path/in/host:/path/in/container'],
'environment': {
'FOO': 'bar'
}
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
r.run(wf)
with open('/tmp/popper/slurm/popper_1_123abc.sh', 'r') as f:
content = f.read()
self.assertEqual(
content, f"""#!/bin/bash
docker rm -f popper_1_123abc || true
docker build -t popperized/bin:master {os.environ['HOME']}/.cache/popper/123abc/github.com/popperized/bin/sh
docker create --name popper_1_123abc --workdir /workspace --entrypoint cat -v /w:/workspace -v /var/run/docker.sock:/var/run/docker.sock -v /path/in/host:/path/in/container -e FOO=bar --privileged --hostname popper.local --domainname www.example.org popperized/bin:master README.md
docker start --attach popper_1_123abc""")
def test_submit_batch_job(self, mock_kill):
self.Popen.set_command(
'sbatch --wait '
'--job-name popper_sample_123abc '
'--output /tmp/popper/slurm/popper_sample_123abc.out '
'/tmp/popper/slurm/popper_sample_123abc.sh',
returncode=0)
self.Popen.set_command(
'tail -f /tmp/popper/slurm/popper_sample_123abc.out', returncode=0)
config = PopperConfig(workspace_dir='/w')
config.wid = "123abc"
step = {"name": "sample"}
with SlurmRunner(config=config) as sr:
sr._submit_batch_job(["ls -la"], step)
with open("/tmp/popper/slurm/popper_sample_123abc.sh", 'r') as f:
content = f.read()
self.assertEqual(content, "#!/bin/bash\nls -la")
self.assertEqual(len(sr._spawned_jobs), 0)
self.assertEqual(sr._out_stream_thread.is_alive(), False)
call_tail = call.Popen(
['tail', '-f', '/tmp/popper/slurm/popper_sample_123abc.out'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
call_sbatch = call.Popen([
'sbatch', '--wait', '--job-name', 'popper_sample_123abc',
'--output', '/tmp/popper/slurm/popper_sample_123abc.out',
'/tmp/popper/slurm/popper_sample_123abc.sh'
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_run(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=0)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'singularity',
'options': {
'hostname': 'popper.local',
'bind': ['/path/in/host:/path/in/container']
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: ls
""")
wf.parse()
r.run(wf)
with open('/tmp/popper/slurm/popper_1_123abc.sh', 'r') as f:
content = f.read()
self.assertEqual(
content, f"""#!/bin/bash
singularity exec --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local {os.environ['HOME']}/.cache/popper/singularity/123abc/popper_1_123abc.sif ls"""
)
def test_get_container_options(self):
config_dict = {
'engine': {
'name': 'singularity',
'options': {
'hostname': 'popper.local',
'ipc': True,
'bind': ['/path/in/host:/path/in/container']
}
}
}
config = PopperConfig(config_file=config_dict)
config.wid = "abcd"
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
options = sr._get_container_options()
self.assertEqual(options, [
'--userns', '--pwd', '/workspace', '--bind',
f'{os.getcwd()}:/workspace', '--bind',
'/path/in/host:/path/in/container', '--hostname',
'popper.local', '--ipc'
])
def test_create_container(self):
config = PopperConfig()
config.wid = "abcd"
step_one = {
'uses': 'docker://*****:*****@master',
'args': ['ls'],
'name': 'kontainer_two',
'repo_dir':
f'{os.environ["HOME"]}/.cache/popper/abcd/github.com/popperized/bin',
'step_dir': 'sh'
}
cid_one = pu.sanitized_name(step_one['name'], config.wid)
cid_two = pu.sanitized_name(step_two['name'], config.wid)
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
c_one = sr._create_container(step_one, cid_one)
self.assertEqual(
os.path.exists(os.path.join(sr._singularity_cache, cid_one)),
True)
os.remove(os.path.join(sr._singularity_cache, cid_one))
with SingularityRunner(config=config) as sr:
sr._setup_singularity_cache()
c_two = sr._create_container(step_one, cid_two)
self.assertEqual(
os.path.exists(os.path.join(sr._singularity_cache, cid_two)),
True)
os.remove(os.path.join(sr._singularity_cache, cid_two))