def buildah_run_cmd(container_image: str, host_name: str, cmd: List[str], log_stderr: bool = True, log_output: bool = False): """ run provided command in selected container image using buildah; raise exc when command fails :param container_image: str :param host_name: str :param cmd: list of str :param log_stderr: bool, log errors to stdout as ERROR level :param log_output: bool, print output of the command to logs """ container_name = "{}-{}".format( host_name, datetime.datetime.now().strftime(TIMESTAMP_FORMAT_TOGETHER)) # was the temporary container created? if so, remove it created = False try: create_buildah_container(container_image, container_name, None, None, False) created = True run_cmd(["buildah", "run", container_name] + cmd, log_stderr=log_stderr, log_output=log_output) except subprocess.CalledProcessError: logger.error( f"Unable to create or run a container using {container_image} with buildah" ) raise finally: if created: run_cmd(["buildah", "rm", container_name], log_stderr=log_stderr)
def sanity_check(self): """ invoke container tooling and thus verify they work well """ # doing podman info would be super-handy, but it will immensely clutter the output logger.debug("checking that podman command works") run_cmd(["podman", "version"], log_stderr=True, log_output=True) logger.debug("checking that buildah command works") run_cmd(["buildah", "version"], log_stderr=True, log_output=True)
def test_clean(target_image, tmpdir): cmd = ["build", basic_playbook_path, base_image, target_image] ab(cmd, str(tmpdir)) run_cmd(["podman", "rmi", target_image], print_output=True) cmd = ["clean"] ab(cmd, str(tmpdir)) out = ab(["clean"], str(tmpdir), return_output=True) assert out.startswith("Cleaning images from database which are no longer present on the disk...") assert out.endswith("Done!\n")
def test_multiplay(build, application): im = "multiplay" build.playbook_path = multiplay_path build.target_image = im application.build(build) try: build = application.db.get_build(build.build_id) podman_run_cmd(im, ["ls", "/queen"]) # the file has to be in there assert len(build.layers) == 3 finally: run_cmd(["buildah", "rmi", im], ignore_status=True, print_output=True)
def sanity_check(self): """ invoke container tooling and thus verify they work well """ logger.debug("checking that docker command works") run_cmd(["docker", "version"], log_stderr=True, log_output=True) logger.debug("Checking container creation using docker") docker_run_cmd_in_container( self.build.base_image, self.ansible_host, ["true"], log_stderr=True, extra_from_args=self.build.buildah_from_extra_args)
def sanity_check(self): """ invoke container tooling and thus verify they work well """ # doing podman info would be super-handy, but it will immensely clutter the output logger.debug("checking that podman command works") run_cmd(["podman", "version"], log_stderr=True, log_output=True) logger.debug("checking that buildah command works") run_cmd(["buildah", "version"], log_stderr=True, log_output=True) logger.debug("Checking container creation using buildah") buildah_run_cmd(self.build.base_image, self.ansible_host, ["true"], log_stderr=True)
def push(self, build, target, force=False): """ push built image into a remote location using `podman push` :param target: str, transport:details :param build: instance of Build :param force: bool, bypass checks if True :return: None """ built_image = build.get_target_image_id() cmd = ["buildah", "push", built_image, target] # podman prints progress to stderr run_cmd(cmd, print_output=False, log_stderr=False)
def test_pb_with_role(build, application): im = "image-built-with-role" build.playbook_path = role_pb_path build.target_image = im os.environ["ANSIBLE_ROLES_PATH"] = roles_dir application.build(build) try: build = application.db.get_build(build.build_id) podman_run_cmd(im, ["ls", "/officer"]) # the file has to be in there # base image + 2 from roles: [] + 2 from import_role # + 3 from include_role (include_role is a task) assert len(build.layers) == 8 finally: run_cmd(["buildah", "rmi", im], ignore_status=True, print_output=True)
def buildah(command, args_and_opts, print_output=False, debug=False): cmd = ["buildah"] # if debug: # cmd += ["--debug"] cmd += [command] + args_and_opts logger.debug("running command: %s", command) return run_cmd(cmd, print_output=print_output, log_stderr=False)
def _clean(self): builds = self.app.list_builds() print( "Cleaning images from database which are no longer present on the disk..." ) for b in builds: try: run_cmd(["podman", "inspect", b.target_image], return_output=False, log_output=False) except subprocess.CalledProcessError: self.app.remove_build(b.build_id) print( f"Build entry with ID {b.build_id} has been removed from DB as it no longer has it's corresponding image" ) continue print("Done!")
def buildah_with_output(command, args_and_opts, debug=False): cmd = ["buildah"] # if debug: # cmd += ["--debug"] cmd += [command] + args_and_opts output = run_cmd(cmd, return_output=True) logger.debug("output: %s", output) return output
def docker(command, args_and_opts, print_output=False, debug=False, log_stderr=False): cmd = ["docker"] cmd += [command] + args_and_opts logger.debug("running command: %s", command) return run_cmd(cmd, print_output=print_output, log_stderr=log_stderr)
def run_playbook(playbook_path, inventory_path, a_cfg_path, connection, extra_variables=None, ansible_args=None, debug=False, environment=None): """ run selected ansible playbook and return output from ansible-playbook run :param playbook_path: :param inventory_path: :param a_cfg_path: :param connection: :param extra_variables: :param ansible_args: :param debug: :param environment: :return: output """ ap = ap_command_exists() cmd_args = [ ap, "-i", inventory_path, "-c", connection, ] if debug: cmd_args += ["-vvv"] if extra_variables: cmd_args += ["--extra-vars"] + \ [" ".join( ["{}={}".format(k, v) for k, v in extra_variables.items()] )] cmd_args += [playbook_path] logger.debug("%s", " ".join(cmd_args)) env = os.environ.copy() if environment: env.update(environment) env["ANSIBLE_CONFIG"] = a_cfg_path # TODO: does ansible have an API? try: return run_cmd( cmd_args, print_output=True, env=env, return_all_output=True, ) except subprocess.CalledProcessError as ex: raise AbBuildUnsuccesful("ansible-playbook execution failed: %s" % ex, ex.output)
def inspect_buildah_resource(resource_type, resource_id): try: i = run_cmd(["buildah", "inspect", "-t", resource_type, resource_id], return_output=True, log_output=False) except subprocess.CalledProcessError: logger.info("no such %s %s", resource_type, resource_id) return None metadata = json.loads(i) return metadata
def run(self, image_name, command): """ run provided command in the selected image and return output :param image_name: str :param command: list of str :return: str (output) """ cmd = ["podman", "run", "--rm", image_name] + command return run_cmd(cmd, return_output=True)
def test_caching_non_ex_image(tmpdir, application, build): """ scenario: we perform a build, we remove an image from cache, we perform the build again, ab should recover """ t = str(tmpdir) non_ex_pb_basename = os.path.basename(non_ex_pb) p = os.path.join(t, non_ex_pb_basename) shutil.copy(non_ex_pb, p) with open(p) as fd: d = yaml.safe_load(fd) d[0]["tasks"][0]["debug"]["msg"] = f"Hello {random_str()}" with open(p, "w") as fd: yaml.safe_dump(d, fd) image_name = random_str(5) build.playbook_path = p build.target_image = image_name application.build(build) build = application.db.get_build(build.build_id) # for debugging layers = build.layers final_layer_id = build.final_layer_id import subprocess subprocess.call(["podman", "images", "--all"]) subprocess.call(["podman", "inspect", build.target_image]) # FIXME: this command fails in CI, which is super weird run_cmd(["buildah", "rmi", build.target_image], ignore_status=True, print_output=True) run_cmd(["buildah", "rmi", build.final_layer_id], ignore_status=True, print_output=True) # now remove all images from the cache layers = build.layers[1:] layers.reverse() for l in layers: if l.base_image_id: run_cmd(["buildah", "rmi", l.layer_id], ignore_status=True, print_output=True) second_build = Build.from_json(build.to_dict()) second_build.build_id = "33" application.build(second_build) run_cmd(["buildah", "rmi", build.target_image], ignore_status=True, print_output=True)
def find_python_interpreter(self): """ find python executable in the base image :return: str, path to python interpreter """ for i in self.python_interpr_prio: cmd = ["ls", i] try: run_cmd(["podman", "run", "--rm", self.build.base_image] + cmd, log_stderr=False, log_output=True) except subprocess.CalledProcessError: logger.info("python interpreter %s does not exist", i) continue else: logger.info("using python interpreter %s", i) return i logger.error("couldn't locate python interpreter, tried these paths: %s", self.python_interpr_prio) raise RuntimeError(f"no python interpreter was found in the base image \"{self.build.base_image}\"" ", you can specify the path via CLI option --python-interpreter")
def podman_run_cmd(container_image, cmd, log_stderr=True, return_output=False): """ run provided command in selected container image using podman; raise exc when command fails :param container_image: str :param cmd: list of str :param log_stderr: bool, log errors to stdout as ERROR level :param return_output: bool, if True, return output of the command :return: stdout output """ return run_cmd(["podman", "run", "--rm", container_image] + cmd, return_output=return_output, log_stderr=log_stderr)
def get_buildah_version(self): out = run_cmd(["buildah", "version"], log_stderr=True, return_output=True, log_output=False) version = re.findall(r"Version:\s*([\d\.]+)", out)[0].split(".") logger.debug("buildah version = %s", version) # buildah version = ['1', '11', '3'] try: return tuple(map(int, version)) except (IndexError, ValueError) as ex: logger.error("Unable to parse buildah's version: %s", ex) return 0, 0, 0
def docker_run_cmd_in_container( container_image: str, host_name: str, cmd: List[str], log_stderr: bool = True, log_output: bool = False, extra_from_args: Optional[List[str]] = None, ): """ run provided command in selected container image using docker; raise exc when command fails :param container_image: str :param host_name: str :param cmd: list of str :param log_stderr: bool, log errors to stdout as ERROR level :param log_output: bool, print output of the command to logs :param extra_from_args: a list of extra arguments for `buildah from` """ container_name = "{}-{}".format( host_name, datetime.datetime.now().strftime(TIMESTAMP_FORMAT_TOGETHER)) # was the temporary container created? if so, remove it created = False try: create_docker_container(container_image, container_name, build_volumes=None, extra_from_args=extra_from_args, command=cmd, debug=False) created = True except subprocess.CalledProcessError: logger.error( f"Unable to create or run a container using {container_image} with docker" ) raise finally: if created: run_cmd(["docker", "rm", container_name], log_stderr=log_stderr)
def inspect_resource(resource_type, resource_id): try: i = run_cmd( ["docker", "inspect", "--type", resource_type, resource_id], return_output=True, log_output=False) except subprocess.CalledProcessError: logger.info("no such %s %s", resource_type, resource_id) return None try: metadata = json.loads(i) except IndexError: logger.info("no such %s %s", resource_type, resource_id) return None return metadata
def run(self, image_name, command): """ run provided command in the selected image and return output :param image_name: str :param command: list of str :return: str (output) """ print("HUY") # let's apply configuration before execing the playbook, except for user # configure_docker_container( # self.ansible_host, working_dir=self.build.metadata.working_dir, # user=self.build.build_user, # env_vars=self.build.metadata.env_vars, # ports=self.build.metadata.ports, # labels=self.build.metadata.labels, # labels are not applied when they are configured # # before doing commit # annotations=self.build.metadata.annotations, # debug=self.debug # ) cmd = ["podman", "run", "--rm", image_name] + command return run_cmd(cmd, return_output=True)
def test_run_cmd(): assert "etc" in run_cmd(["ls", "-1", "/"], return_all_output=True)
def does_image_exist(container_image): # cmd = ["podman", "image", "exists", container_image] # https://github.com/containers/libpod/issues/2924 # https://github.com/ansible-community/ansible-bender/issues/114 cmd = ["buildah", "inspect", "-t", "image", container_image] run_cmd(cmd, print_output=False)
def pull_buildah_image(container_image): run_cmd(["buildah", "pull", container_image], save_output_in_exc=False, log_stderr=False, print_output=True, log_output=False)
def pull_docker_image(container_image): run_cmd(["docker", "pull", "--quiet", container_image], save_output_in_exc=False, log_stderr=False, print_output=True, log_output=False)
def does_image_exist(container_image): cmd = ["docker", "inspect", "--type", "image", container_image] run_cmd(cmd, print_output=False)
def does_image_exist(container_image): run_cmd(["podman", "image", "exists", container_image])
def run_playbook(playbook_path, inventory_path, a_cfg_path, connection, extra_variables=None, ansible_args=None, debug=False, environment=None, try_unshare=True, provide_output=True, log_stderr=False): """ run selected ansible playbook and return output from ansible-playbook run :param playbook_path: :param inventory_path: :param a_cfg_path: :param connection: :param extra_variables: :param ansible_args: list of str, extra arguments for a-p :param debug: :param environment: :param try_unshare: bool, do `buildah unshare` if uid > 0 :param provide_output: bool, present output to user :param log_stderr: bool, log errors coming from stderr to our logger :return: output """ ap = ap_command_exists() if is_ansibles_python_2(ap): # I just realized it could work, we would just had to disable the # callback plugin: no caching and layering raise RuntimeError( "ansible-bender is written in python 3 and does not work in python 2,\n" f"it seems that {ap} is using python 2 - ansible-bender will not" "work in such environment\n") cmd_args = [ ap, "-c", connection, ] if inventory_path: cmd_args += ["-i", inventory_path] if debug: cmd_args += ["-vvv"] if extra_variables: cmd_args += ["--extra-vars"] + \ [" ".join( ["{}={}".format(k, v) for k, v in extra_variables.items()] )] if ansible_args: cmd_args += ansible_args cmd_args += [playbook_path] logger.debug("%s", " ".join(cmd_args)) env = os.environ.copy() env["ANSIBLE_RETRY_FILES_ENABLED"] = "0" if debug: env["ANSIBLE_STDOUT_CALLBACK"] = "debug" if environment: env.update(environment) if a_cfg_path: env["ANSIBLE_CONFIG"] = a_cfg_path if try_unshare and os.getuid() != 0: logger.info("we are running rootless, prepending `buildah unshare`") # rootless, we need to `buildah unshare` for sake of `buildah mount` # https://github.com/containers/buildah/issues/1271 # the need for `--` https://github.com/containers/buildah/issues/1374 cmd_args = ["buildah", "unshare", "--"] + cmd_args # ansible has no official python API, the API they have is internal and said to break compat try: return run_cmd( cmd_args, print_output=provide_output, save_output_in_exc=True, env=env, return_all_output=provide_output, log_stderr=log_stderr, ) except subprocess.CalledProcessError as ex: raise ABBuildUnsuccesful("ansible-playbook execution failed: %s" % ex, ex.output)
def pull_buildah_image(container_image): run_cmd(["podman", "pull", container_image])