def test_clone_repos(self):
wf = YMLWorkflow("""
version: '1'
steps:
- uses: popperized/bin/sh@master
""")
wf.parse()
conf = PopperConfig()
cache_dir = os.path.join(os.environ['HOME'], '.cache/popper/')
# clone repos in the default cache directory.
runner = WorkflowRunner(conf)
runner._clone_repos(wf)
step_dir = os.path.join(cache_dir, conf.wid,
'github.com/popperized/bin')
self.assertTrue(os.path.exists(step_dir))
# clone repos in custom cache directory
os.environ['POPPER_CACHE_DIR'] = '/tmp/smdir'
runner._clone_repos(wf)
step_dir = os.path.join('/tmp/smdir', conf.wid,
'github.com/popperized/bin')
self.assertTrue(os.path.exists(step_dir))
os.environ.pop('POPPER_CACHE_DIR')
# check failure when container is not available and we skip cloning
shutil.rmtree('/tmp/smdir')
shutil.rmtree(cache_dir)
conf = PopperConfig(skip_clone=True)
runner = WorkflowRunner(conf)
self.assertRaises(SystemExit, runner._clone_repos, wf)
def test_check_secrets(self):
wf = YMLWorkflow("""
version: '1'
steps:
- uses: docker://alpine:3.9
args: ["ls -ltr"]
secrets: ["SECRET_ONE", "SECRET_TWO"]
""")
wf.parse()
# in dry-run, secrets are ignored
runner = WorkflowRunner(PopperConfig(dry_run=True))
runner._process_secrets(wf)
# now go back to not dry-running
runner = WorkflowRunner(PopperConfig())
# when CI=true it should fail
os.environ['CI'] = 'true'
self.assertRaises(SystemExit, runner._process_secrets, wf)
# add one secret
os.environ['SECRET_ONE'] = '1234'
# it should fail again, as we're missing one
self.assertRaises(SystemExit, runner._process_secrets, wf)
os.environ.pop('CI')
# now is fine
with patch('getpass.getpass', return_value='5678'):
runner._process_secrets(wf)
# pop the other
os.environ.pop('SECRET_ONE')
def test_submit_job_failure(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=12)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'docker',
'options': {}
},
'resource_manager': {
'name': 'slurm',
'options': {}
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
self.assertRaises(SystemExit, r.run, wf)
call_tail = call.Popen(
['tail', '-f', '/tmp/popper/slurm/popper_1_123abc.out'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
call_sbatch = call.Popen([
'sbatch', '--wait', '--job-name', 'popper_1_123abc',
'--output', '/tmp/popper/slurm/popper_1_123abc.out',
'/tmp/popper/slurm/popper_1_123abc.sh'
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_run(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=0)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'docker',
'options': {
'privileged': True,
'hostname': 'popper.local',
'domainname': 'www.example.org',
'volumes': ['/path/in/host:/path/in/container'],
'environment': {
'FOO': 'bar'
}
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
r.run(wf)
with open('/tmp/popper/slurm/popper_1_123abc.sh', 'r') as f:
content = f.read()
self.assertEqual(
content, f"""#!/bin/bash
docker rm -f popper_1_123abc || true
docker build -t popperized/bin:master {os.environ['HOME']}/.cache/popper/123abc/github.com/popperized/bin/sh
docker create --name popper_1_123abc --workdir /workspace --entrypoint cat -v /w:/workspace -v /var/run/docker.sock:/var/run/docker.sock -v /path/in/host:/path/in/container -e FOO=bar --privileged --hostname popper.local --domainname www.example.org popperized/bin:master README.md
docker start --attach popper_1_123abc""")
def test_dry_run(self):
repo = self.mk_repo()
config = PopperConfig(engine_name='docker',
resman_name='slurm',
dry_run=True,
workspace_dir=repo.working_dir)
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
r.run(wf)
self.assertEqual(self.Popen.all_calls, [])
def test_run(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=0)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'singularity',
'options': {
'hostname': 'popper.local',
'bind': ['/path/in/host:/path/in/container']
}
},
'resource_manager': {
'name': 'slurm'
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: ls
""")
wf.parse()
r.run(wf)
with open('/tmp/popper/slurm/popper_1_123abc.sh', 'r') as f:
content = f.read()
self.assertEqual(
content, f"""#!/bin/bash
singularity exec --userns --pwd /workspace --bind /w:/workspace --bind /path/in/host:/path/in/container --hostname popper.local {os.environ['HOME']}/.cache/popper/singularity/123abc/popper_1_123abc.sif ls"""
)
def test_singularity_start(self):
repo = self.mk_repo()
conf = PopperConfig(engine_name='singularity',
workspace_dir=repo.working_dir)
step = {
'uses': 'docker://*****:*****@master'
args: 'ls'
""")
wf.parse()
r.run(wf)
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'docker://alpine:3.9'
runs: ['sh', '-c', 'echo $FOO > hello.txt ; pwd']
env: {
FOO: bar
}
""")
wf.parse()
r.run(wf)
with open(os.path.join(repo.working_dir, 'hello.txt'), 'r') as f:
self.assertEqual(f.read(), 'bar\n')
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'docker://alpine:3.9'
runs: 'nocommandisnamedlikethis'
""")
wf.parse()
self.assertRaises(SystemExit, r.run, wf)
repo.close()
def test_run(self):
repo = self.mk_repo()
conf = PopperConfig(workspace_dir=repo.working_dir)
with WorkflowRunner(conf) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: sh
runs: [cat]
args: README.md
""")
wf.parse()
r.run(wf)
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'sh'
runs: ['bash', '-c', 'echo $FOO > hello.txt ; pwd']
env: {
FOO: bar
}
""")
wf.parse()
r.run(wf)
with open(os.path.join(repo.working_dir, 'hello.txt'), 'r') as f:
self.assertEqual(f.read(), 'bar\n')
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'sh'
runs: 'nocommandisnamedlikethis'
""")
wf.parse()
self.assertRaises(SystemExit, r.run, wf)
repo.close()