def buildah_run_cmd(container_image: str,
host_name: str,
cmd: List[str],
log_stderr: bool = True,
log_output: bool = False):
"""
run provided command in selected container image using buildah; raise exc when command fails
:param container_image: str
:param host_name: str
:param cmd: list of str
:param log_stderr: bool, log errors to stdout as ERROR level
:param log_output: bool, print output of the command to logs
"""
container_name = "{}-{}".format(
host_name,
datetime.datetime.now().strftime(TIMESTAMP_FORMAT_TOGETHER))
# was the temporary container created? if so, remove it
created = False
try:
create_buildah_container(container_image, container_name, None, None,
False)
created = True
run_cmd(["buildah", "run", container_name] + cmd,
log_stderr=log_stderr,
log_output=log_output)
except subprocess.CalledProcessError:
logger.error(
f"Unable to create or run a container using {container_image} with buildah"
)
raise
finally:
if created:
run_cmd(["buildah", "rm", container_name], log_stderr=log_stderr)
def sanity_check(self):
"""
invoke container tooling and thus verify they work well
"""
# doing podman info would be super-handy, but it will immensely clutter the output
logger.debug("checking that podman command works")
run_cmd(["podman", "version"], log_stderr=True, log_output=True)
logger.debug("checking that buildah command works")
run_cmd(["buildah", "version"], log_stderr=True, log_output=True)
def test_clean(target_image, tmpdir):
cmd = ["build", basic_playbook_path, base_image, target_image]
ab(cmd, str(tmpdir))
run_cmd(["podman", "rmi", target_image], print_output=True)
cmd = ["clean"]
ab(cmd, str(tmpdir))
out = ab(["clean"], str(tmpdir), return_output=True)
assert out.startswith("Cleaning images from database which are no longer present on the disk...")
assert out.endswith("Done!\n")
def test_multiplay(build, application):
im = "multiplay"
build.playbook_path = multiplay_path
build.target_image = im
application.build(build)
try:
build = application.db.get_build(build.build_id)
podman_run_cmd(im, ["ls", "/queen"]) # the file has to be in there
assert len(build.layers) == 3
finally:
run_cmd(["buildah", "rmi", im], ignore_status=True, print_output=True)
def sanity_check(self):
"""
invoke container tooling and thus verify they work well
"""
logger.debug("checking that docker command works")
run_cmd(["docker", "version"], log_stderr=True, log_output=True)
logger.debug("Checking container creation using docker")
docker_run_cmd_in_container(
self.build.base_image,
self.ansible_host, ["true"],
log_stderr=True,
extra_from_args=self.build.buildah_from_extra_args)
def sanity_check(self):
"""
invoke container tooling and thus verify they work well
"""
# doing podman info would be super-handy, but it will immensely clutter the output
logger.debug("checking that podman command works")
run_cmd(["podman", "version"], log_stderr=True, log_output=True)
logger.debug("checking that buildah command works")
run_cmd(["buildah", "version"], log_stderr=True, log_output=True)
logger.debug("Checking container creation using buildah")
buildah_run_cmd(self.build.base_image,
self.ansible_host, ["true"],
log_stderr=True)
def push(self, build, target, force=False):
"""
push built image into a remote location using `podman push`
:param target: str, transport:details
:param build: instance of Build
:param force: bool, bypass checks if True
:return: None
"""
built_image = build.get_target_image_id()
cmd = ["buildah", "push", built_image, target]
# podman prints progress to stderr
run_cmd(cmd, print_output=False, log_stderr=False)
def test_pb_with_role(build, application):
im = "image-built-with-role"
build.playbook_path = role_pb_path
build.target_image = im
os.environ["ANSIBLE_ROLES_PATH"] = roles_dir
application.build(build)
try:
build = application.db.get_build(build.build_id)
podman_run_cmd(im, ["ls", "/officer"]) # the file has to be in there
# base image + 2 from roles: [] + 2 from import_role
# + 3 from include_role (include_role is a task)
assert len(build.layers) == 8
finally:
run_cmd(["buildah", "rmi", im], ignore_status=True, print_output=True)
def buildah(command, args_and_opts, print_output=False, debug=False):
cmd = ["buildah"]
# if debug:
# cmd += ["--debug"]
cmd += [command] + args_and_opts
logger.debug("running command: %s", command)
return run_cmd(cmd, print_output=print_output, log_stderr=False)
def _clean(self):
builds = self.app.list_builds()
print(
"Cleaning images from database which are no longer present on the disk..."
)
for b in builds:
try:
run_cmd(["podman", "inspect", b.target_image],
return_output=False,
log_output=False)
except subprocess.CalledProcessError:
self.app.remove_build(b.build_id)
print(
f"Build entry with ID {b.build_id} has been removed from DB as it no longer has it's corresponding image"
)
continue
print("Done!")
def buildah_with_output(command, args_and_opts, debug=False):
cmd = ["buildah"]
# if debug:
# cmd += ["--debug"]
cmd += [command] + args_and_opts
output = run_cmd(cmd, return_output=True)
logger.debug("output: %s", output)
return output
def docker(command,
args_and_opts,
print_output=False,
debug=False,
log_stderr=False):
cmd = ["docker"]
cmd += [command] + args_and_opts
logger.debug("running command: %s", command)
return run_cmd(cmd, print_output=print_output, log_stderr=log_stderr)
def run_playbook(playbook_path,
inventory_path,
a_cfg_path,
connection,
extra_variables=None,
ansible_args=None,
debug=False,
environment=None):
"""
run selected ansible playbook and return output from ansible-playbook run
:param playbook_path:
:param inventory_path:
:param a_cfg_path:
:param connection:
:param extra_variables:
:param ansible_args:
:param debug:
:param environment:
:return: output
"""
ap = ap_command_exists()
cmd_args = [
ap,
"-i",
inventory_path,
"-c",
connection,
]
if debug:
cmd_args += ["-vvv"]
if extra_variables:
cmd_args += ["--extra-vars"] + \
[" ".join(
["{}={}".format(k, v)
for k, v in extra_variables.items()]
)]
cmd_args += [playbook_path]
logger.debug("%s", " ".join(cmd_args))
env = os.environ.copy()
if environment:
env.update(environment)
env["ANSIBLE_CONFIG"] = a_cfg_path
# TODO: does ansible have an API?
try:
return run_cmd(
cmd_args,
print_output=True,
env=env,
return_all_output=True,
)
except subprocess.CalledProcessError as ex:
raise AbBuildUnsuccesful("ansible-playbook execution failed: %s" % ex,
ex.output)
def inspect_buildah_resource(resource_type, resource_id):
try:
i = run_cmd(["buildah", "inspect", "-t", resource_type, resource_id],
return_output=True,
log_output=False)
except subprocess.CalledProcessError:
logger.info("no such %s %s", resource_type, resource_id)
return None
metadata = json.loads(i)
return metadata
def run(self, image_name, command):
"""
run provided command in the selected image and return output
:param image_name: str
:param command: list of str
:return: str (output)
"""
cmd = ["podman", "run", "--rm", image_name] + command
return run_cmd(cmd, return_output=True)
def test_caching_non_ex_image(tmpdir, application, build):
"""
scenario: we perform a build, we remove an image from cache, we perform the build again, ab should recover
"""
t = str(tmpdir)
non_ex_pb_basename = os.path.basename(non_ex_pb)
p = os.path.join(t, non_ex_pb_basename)
shutil.copy(non_ex_pb, p)
with open(p) as fd:
d = yaml.safe_load(fd)
d[0]["tasks"][0]["debug"]["msg"] = f"Hello {random_str()}"
with open(p, "w") as fd:
yaml.safe_dump(d, fd)
image_name = random_str(5)
build.playbook_path = p
build.target_image = image_name
application.build(build)
build = application.db.get_build(build.build_id)
# for debugging
layers = build.layers
final_layer_id = build.final_layer_id
import subprocess
subprocess.call(["podman", "images", "--all"])
subprocess.call(["podman", "inspect", build.target_image])
# FIXME: this command fails in CI, which is super weird
run_cmd(["buildah", "rmi", build.target_image],
ignore_status=True,
print_output=True)
run_cmd(["buildah", "rmi", build.final_layer_id],
ignore_status=True,
print_output=True)
# now remove all images from the cache
layers = build.layers[1:]
layers.reverse()
for l in layers:
if l.base_image_id:
run_cmd(["buildah", "rmi", l.layer_id],
ignore_status=True,
print_output=True)
second_build = Build.from_json(build.to_dict())
second_build.build_id = "33"
application.build(second_build)
run_cmd(["buildah", "rmi", build.target_image],
ignore_status=True,
print_output=True)
def find_python_interpreter(self):
"""
find python executable in the base image
:return: str, path to python interpreter
"""
for i in self.python_interpr_prio:
cmd = ["ls", i]
try:
run_cmd(["podman", "run", "--rm", self.build.base_image] + cmd,
log_stderr=False, log_output=True)
except subprocess.CalledProcessError:
logger.info("python interpreter %s does not exist", i)
continue
else:
logger.info("using python interpreter %s", i)
return i
logger.error("couldn't locate python interpreter, tried these paths: %s", self.python_interpr_prio)
raise RuntimeError(f"no python interpreter was found in the base image \"{self.build.base_image}\""
", you can specify the path via CLI option --python-interpreter")
def podman_run_cmd(container_image, cmd, log_stderr=True, return_output=False):
"""
run provided command in selected container image using podman; raise exc when command fails
:param container_image: str
:param cmd: list of str
:param log_stderr: bool, log errors to stdout as ERROR level
:param return_output: bool, if True, return output of the command
:return: stdout output
"""
return run_cmd(["podman", "run", "--rm", container_image] + cmd,
return_output=return_output, log_stderr=log_stderr)
def get_buildah_version(self):
out = run_cmd(["buildah", "version"],
log_stderr=True,
return_output=True,
log_output=False)
version = re.findall(r"Version:\s*([\d\.]+)", out)[0].split(".")
logger.debug("buildah version = %s", version)
# buildah version = ['1', '11', '3']
try:
return tuple(map(int, version))
except (IndexError, ValueError) as ex:
logger.error("Unable to parse buildah's version: %s", ex)
return 0, 0, 0
def docker_run_cmd_in_container(
container_image: str,
host_name: str,
cmd: List[str],
log_stderr: bool = True,
log_output: bool = False,
extra_from_args: Optional[List[str]] = None,
):
"""
run provided command in selected container image using docker; raise exc when command fails
:param container_image: str
:param host_name: str
:param cmd: list of str
:param log_stderr: bool, log errors to stdout as ERROR level
:param log_output: bool, print output of the command to logs
:param extra_from_args: a list of extra arguments for `buildah from`
"""
container_name = "{}-{}".format(
host_name,
datetime.datetime.now().strftime(TIMESTAMP_FORMAT_TOGETHER))
# was the temporary container created? if so, remove it
created = False
try:
create_docker_container(container_image,
container_name,
build_volumes=None,
extra_from_args=extra_from_args,
command=cmd,
debug=False)
created = True
except subprocess.CalledProcessError:
logger.error(
f"Unable to create or run a container using {container_image} with docker"
)
raise
finally:
if created:
run_cmd(["docker", "rm", container_name], log_stderr=log_stderr)
def inspect_resource(resource_type, resource_id):
try:
i = run_cmd(
["docker", "inspect", "--type", resource_type, resource_id],
return_output=True,
log_output=False)
except subprocess.CalledProcessError:
logger.info("no such %s %s", resource_type, resource_id)
return None
try:
metadata = json.loads(i)
except IndexError:
logger.info("no such %s %s", resource_type, resource_id)
return None
return metadata
def run(self, image_name, command):
"""
run provided command in the selected image and return output
:param image_name: str
:param command: list of str
:return: str (output)
"""
print("HUY")
# let's apply configuration before execing the playbook, except for user
# configure_docker_container(
# self.ansible_host, working_dir=self.build.metadata.working_dir,
# user=self.build.build_user,
# env_vars=self.build.metadata.env_vars,
# ports=self.build.metadata.ports,
# labels=self.build.metadata.labels, # labels are not applied when they are configured
# # before doing commit
# annotations=self.build.metadata.annotations,
# debug=self.debug
# )
cmd = ["podman", "run", "--rm", image_name] + command
return run_cmd(cmd, return_output=True)
def test_run_cmd():
assert "etc" in run_cmd(["ls", "-1", "/"], return_all_output=True)
def does_image_exist(container_image):
# cmd = ["podman", "image", "exists", container_image]
# https://github.com/containers/libpod/issues/2924
# https://github.com/ansible-community/ansible-bender/issues/114
cmd = ["buildah", "inspect", "-t", "image", container_image]
run_cmd(cmd, print_output=False)
def pull_buildah_image(container_image):
run_cmd(["buildah", "pull", container_image],
save_output_in_exc=False,
log_stderr=False,
print_output=True,
log_output=False)
def pull_docker_image(container_image):
run_cmd(["docker", "pull", "--quiet", container_image],
save_output_in_exc=False,
log_stderr=False,
print_output=True,
log_output=False)
def does_image_exist(container_image):
cmd = ["docker", "inspect", "--type", "image", container_image]
run_cmd(cmd, print_output=False)
def does_image_exist(container_image):
run_cmd(["podman", "image", "exists", container_image])
def run_playbook(playbook_path,
inventory_path,
a_cfg_path,
connection,
extra_variables=None,
ansible_args=None,
debug=False,
environment=None,
try_unshare=True,
provide_output=True,
log_stderr=False):
"""
run selected ansible playbook and return output from ansible-playbook run
:param playbook_path:
:param inventory_path:
:param a_cfg_path:
:param connection:
:param extra_variables:
:param ansible_args: list of str, extra arguments for a-p
:param debug:
:param environment:
:param try_unshare: bool, do `buildah unshare` if uid > 0
:param provide_output: bool, present output to user
:param log_stderr: bool, log errors coming from stderr to our logger
:return: output
"""
ap = ap_command_exists()
if is_ansibles_python_2(ap):
# I just realized it could work, we would just had to disable the
# callback plugin: no caching and layering
raise RuntimeError(
"ansible-bender is written in python 3 and does not work in python 2,\n"
f"it seems that {ap} is using python 2 - ansible-bender will not"
"work in such environment\n")
cmd_args = [
ap,
"-c",
connection,
]
if inventory_path:
cmd_args += ["-i", inventory_path]
if debug:
cmd_args += ["-vvv"]
if extra_variables:
cmd_args += ["--extra-vars"] + \
[" ".join(
["{}={}".format(k, v)
for k, v in extra_variables.items()]
)]
if ansible_args:
cmd_args += ansible_args
cmd_args += [playbook_path]
logger.debug("%s", " ".join(cmd_args))
env = os.environ.copy()
env["ANSIBLE_RETRY_FILES_ENABLED"] = "0"
if debug:
env["ANSIBLE_STDOUT_CALLBACK"] = "debug"
if environment:
env.update(environment)
if a_cfg_path:
env["ANSIBLE_CONFIG"] = a_cfg_path
if try_unshare and os.getuid() != 0:
logger.info("we are running rootless, prepending `buildah unshare`")
# rootless, we need to `buildah unshare` for sake of `buildah mount`
# https://github.com/containers/buildah/issues/1271
# the need for `--` https://github.com/containers/buildah/issues/1374
cmd_args = ["buildah", "unshare", "--"] + cmd_args
# ansible has no official python API, the API they have is internal and said to break compat
try:
return run_cmd(
cmd_args,
print_output=provide_output,
save_output_in_exc=True,
env=env,
return_all_output=provide_output,
log_stderr=log_stderr,
)
except subprocess.CalledProcessError as ex:
raise ABBuildUnsuccesful("ansible-playbook execution failed: %s" % ex,
ex.output)
def pull_buildah_image(container_image):
run_cmd(["podman", "pull", container_image])
