def _get_container_kwargs(self, step, img, name):
args = {
"image":
img,
"command":
step.get('args', None),
"name":
name,
"volumes": [
f'{self._config.workspace_dir}:/workspace',
'/var/run/docker.sock:/var/run/docker.sock'
],
"working_dir":
'/workspace',
"environment":
StepRunner.prepare_environment(step),
"entrypoint":
step.get('runs', None),
"detach":
True
}
self._update_with_engine_config(args)
log.debug(f'container args: {pu.prettystr(args)}\n')
return args
def test_prepare_environment_with_git(self):
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with StepRunner(conf) as r:
step = Box({
"name": "a",
"env": {
"FOO": "BAR"
},
"secrets": ["A"]
},
default_box=True)
os.environ["A"] = "BC"
env = r._prepare_environment(step, {"other": "b"})
expected = {
"FOO": "BAR",
"A": "BC",
"other": "b",
"GIT_COMMIT": conf.git_commit,
"GIT_BRANCH": conf.git_branch,
"GIT_SHA_SHORT": conf.git_sha_short,
"GIT_REMOTE_ORIGIN_URL": conf.git_remote_origin_url,
"GIT_TAG": conf.git_tag,
}
self.assertDictEqual(expected, env)
os.environ.pop("A")
def test_prepare_environment_without_git(self):
with StepRunner(ConfigLoader.load(workspace_dir="/tmp/foo")) as r:
step = Box({
"name": "a",
"env": {
"FOO": "BAR"
},
"secrets": ["A"]
},
default_box=True)
os.environ["A"] = "BC"
env = r._prepare_environment(step, {"other": "b"})
self.assertDictEqual({"FOO": "BAR", "A": "BC", "other": "b"}, env)
os.environ.pop("A")
def run(self, step):
step_env = StepRunner.prepare_environment(step, os.environ)
cmd = step.get('runs', [])
if not cmd:
raise AttributeError(f"Expecting 'runs' attribute in step.")
cmd.extend(step.get('args', []))
log.info(f'[{step["name"]}] {cmd}')
if self._config.dry_run:
return 0
log.debug(f'Environment:\n{pu.prettystr(os.environ)}')
pid, ecode, _ = HostRunner._exec_cmd(cmd, step_env,
self._config.workspace_dir,
self._spawned_pids)
if pid != 0:
self._spawned_pids.remove(pid)
return ecode
def _singularity_start(self, step, cid):
env = StepRunner.prepare_environment(step)
# set the environment variables
for k, v in env.items():
os.environ[k] = v
args = step.get('args', None)
runs = step.get('runs', None)
ecode = None
if runs:
info = f'[{step["name"]}] singularity exec {cid} {runs}'
commands = runs
start_fn = self._s.execute
else:
info = f'[{step["name"]}] singularity run {cid} {args}'
commands = args
start_fn = self._s.run
log.info(info)
if self._config.dry_run:
return 0
options = self._get_container_options()
output = start_fn(self._container,
commands,
stream=True,
options=options)
try:
for line in output:
log.step_info(line.strip('\n'))
ecode = 0
except CalledProcessError as ex:
ecode = ex.returncode
return ecode
def test_prepare_environment(self):
step = {'name': 'a', 'env': {'FOO': 'BAR'}, 'secrets': ['A']}
os.environ['A'] = 'BC'
env = StepRunner.prepare_environment(step, {'another': 'b'})
self.assertDictEqual(env, {'FOO': 'BAR', 'A': 'BC', 'another': 'b'})
os.environ.pop('A')
def test_get_container_kwargs(self):
step = Box(
{
"uses": "popperized/bin/sh@master",
"args": ["ls"],
"id": "one",
"dir": "/tmp/",
"options": {"ports": {"8888/tcp": 8888}},
},
default_box=True,
)
config_dict = {
"engine": {
"name": "docker",
"options": {
"privileged": True,
"hostname": "popper.local",
"domainname": "www.example.org",
"volumes": ["/path/in/host:/path/in/container"],
"environment": {"FOO": "bar"},
},
},
}
config = ConfigLoader.load(
config_file=config_dict, workspace_dir="/path/to/workdir"
)
with StepRunner(config=config) as r:
args = r._get_container_kwargs(step, "alpine:3.9", "container_a")
self.assertEqual(
args,
{
"image": "alpine:3.9",
"command": ["ls"],
"name": "container_a",
"volumes": [
"/path/to/workdir:/workspace:Z",
"/path/in/host:/path/in/container",
],
"working_dir": "/tmp/",
"environment": {"FOO": "bar"},
"entrypoint": None,
"detach": True,
"stdin_open": False,
"tty": False,
"privileged": True,
"hostname": "popper.local",
"domainname": "www.example.org",
"ports": {"8888/tcp": 8888},
},
)
# check container kwargs when pty is enabled
config = ConfigLoader.load(
config_file=config_dict, workspace_dir="/path/to/workdir", pty=True
)
with StepRunner(config=config) as r:
args = r._get_container_kwargs(step, "alpine:3.9", "container_a")
self.assertEqual(
args,
{
"image": "alpine:3.9",
"command": ["ls"],
"name": "container_a",
"volumes": [
"/path/to/workdir:/workspace:Z",
"/path/in/host:/path/in/container",
],
"working_dir": "/tmp/",
"environment": {"FOO": "bar"},
"entrypoint": None,
"detach": False,
"stdin_open": True,
"tty": True,
"privileged": True,
"hostname": "popper.local",
"domainname": "www.example.org",
"ports": {"8888/tcp": 8888},
},
)
def test_get_build_info(self):
step = Box(
{"uses": "popperized/bin/sh@master", "args": ["ls"], "id": "one",},
default_box=True,
)
with StepRunner() as r:
build, _, img, tag, build_ctx_path = r._get_build_info(step)
self.assertEqual(build, True)
self.assertEqual(img, "popperized/bin")
self.assertEqual(tag, "master")
self.assertTrue(f"{os.environ['HOME']}/.cache/popper" in build_ctx_path)
self.assertTrue("github.com/popperized/bin/sh" in build_ctx_path)
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["sh", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {"FOO": "bar"},
"id": "1",
},
default_box=True,
)
with StepRunner() as r:
build, _, img, tag, build_sources = r._get_build_info(step)
self.assertEqual(build, False)
self.assertEqual(img, "alpine")
self.assertEqual(tag, "3.9")
self.assertEqual(build_sources, None)
step = Box({"uses": "./", "args": ["ls"], "id": "one",}, default_box=True,)
conf = ConfigLoader.load(workspace_dir="/tmp")
with StepRunner(config=conf) as r:
build, _, img, tag, build_ctx_path = r._get_build_info(step)
self.assertEqual(build, True)
self.assertEqual(img, "popper_one_step")
self.assertEqual(tag, "na")
self.assertEqual(build_ctx_path, f"{os.path.realpath('/tmp')}/./")
# test within a git repo
repo = self.mk_repo()
conf = ConfigLoader.load(workspace_dir=repo.working_dir)
with StepRunner(config=conf) as r:
build, _, img, tag, build_ctx_path = r._get_build_info(step)
self.assertEqual(build, True)
self.assertEqual(img, "popper_one_step")
self.assertEqual(tag, scm.get_sha(repo, short=7))
self.assertEqual(build_ctx_path, f"{os.path.realpath(repo.working_dir)}/./")
step = Box(
{
"uses": "docker://alpine:3.9",
"runs": ["sh", "-c", "echo $FOO > hello.txt ; pwd"],
"env": {"FOO": "bar"},
"name": "1",
},
default_box=True,
)
with StepRunner() as r:
build, img_full, _, _, build_ctx_path = r._get_build_info(step)
self.assertEqual(build, False)
self.assertEqual(img_full, "docker://alpine:3.9")
self.assertEqual(build_ctx_path, None)