def push_container(self, dest_tag: ImageName, *, insecure: bool = False) -> None: """Push the built container (named dest_tag) to the registry (as dest_tag). Push the container as v2s2 (Docker v2 schema 2) regardless of the original format. :param dest_tag: the name of the built container, and the destination for the push :param insecure: disable --tls-verify? """ options = ["--format=v2s2"] if self._registries_authfile: options.append(f"--authfile={self._registries_authfile}") if insecure: options.append("--tls-verify=false") push_cmd = [*self._podman_remote_cmd, "push", *options, str(dest_tag)] try: retries.run_cmd(push_cmd) except subprocess.CalledProcessError as e: raise PushError( f"Push failed (rc={e.returncode}). Check the logs for more details." ) from e
def test_run_cmd_failure(caplog): cmd = ["skopeo", "copy", "docker://a", "docker://b"] total_tries = SUBPROCESS_MAX_RETRIES + 1 (flexmock(subprocess).should_receive('run').with_args( cmd, check=True, capture_output=True).times(total_tries).and_raise( subprocess.CalledProcessError(1, cmd, output=b'', stderr=b'something went wrong'))) flexmock(time).should_receive('sleep').times(SUBPROCESS_MAX_RETRIES) with pytest.raises(subprocess.CalledProcessError): retries.run_cmd(cmd) assert caplog.text.count( 'Running skopeo copy docker://a docker://b') == total_tries assert caplog.text.count('skopeo failed:\n' 'STDOUT:\n' '\n' 'STDERR:\n' 'something went wrong') == total_tries for n in range(SUBPROCESS_MAX_RETRIES): wait = SUBPROCESS_BACKOFF_FACTOR * 2**n assert f'Backing off run_cmd(...) for {wait:.1f}s' in caplog.text assert f'Giving up run_cmd(...) after {total_tries} tries' in caplog.text
def push_with_skopeo(self, image: Dict[str, str], registry_image: ImageName, insecure: bool, docker_push_secret: str) -> None: cmd = ['skopeo', 'copy'] if docker_push_secret is not None: dockercfg = Dockercfg(docker_push_secret) cmd.append('--authfile=' + dockercfg.json_secret_path) if insecure: cmd.append('--dest-tls-verify=false') if image['type'] == IMAGE_TYPE_OCI: # ref_name is added by 'flatpak_create_oci' # we have to be careful when changing the source container image type # since assumption here is that source container image will always be 'docker-archive' source_img = 'oci:{path}:{ref_name}'.format(**image) cmd.append('--format=v2s2') elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE: source_img = 'docker-archive://{path}'.format(**image) else: raise RuntimeError("Attempt to push unsupported image type %s with skopeo" % image['type']) dest_img = 'docker://' + registry_image.to_str() cmd += [source_img, dest_img] try: retries.run_cmd(cmd) except subprocess.CalledProcessError as e: self.log.error("push failed with output:\n%s", e.output) raise
def build_flatpak_image(self, source, build_dir: BuildDir) -> Dict[str, Any]: builder = FlatpakBuilder(source, build_dir.path, 'var/tmp/flatpak-build', parse_manifest=parse_rpm_output, flatpak_metadata=self.flatpak_metadata) df_labels = build_dir.dockerfile_with_parent_env( self.workflow.imageutil.base_image_inspect() ).labels builder.add_labels(df_labels) tmp_dir = tempfile.mkdtemp(dir=build_dir.path) image_filesystem = self.workflow.imageutil.extract_filesystem_layer( str(build_dir.exported_squashed_image), str(tmp_dir)) build_dir.exported_squashed_image.unlink() filesystem_path = os.path.join(tmp_dir, image_filesystem) with open(filesystem_path, 'rb') as f: # this part is 'not ideal' but this function seems to be a prerequisite # for building flatpak image since it does the setup for it flatpak_filesystem, flatpak_manifest = builder._export_from_stream(f) os.remove(filesystem_path) self.log.info('filesystem tarfile written to %s', flatpak_filesystem) image_rpm_components = builder.get_components(flatpak_manifest) ref_name, outfile, outfile_tarred = builder.build_container(flatpak_filesystem) os.remove(outfile_tarred) metadata = get_exported_image_metadata(outfile, IMAGE_TYPE_OCI) metadata['ref_name'] = ref_name cmd = ['skopeo', 'copy', 'oci:{path}:{ref_name}'.format(**metadata), '--format=v2s2', 'docker-archive:{}'.format(str(build_dir.exported_squashed_image))] try: retries.run_cmd(cmd) except subprocess.CalledProcessError as e: self.log.error("skopeo copy failed with output:\n%s", e.output) raise RuntimeError("skopeo copy failed with output:\n{}".format(e.output)) from e self.log.info('OCI image is available as %s', outfile) shutil.rmtree(tmp_dir) self.workflow.data.image_components[build_dir.platform] = image_rpm_components return metadata
def download_image_archive_tarball(self, image: Union[str, ImageName], path: str) -> None: """Downloads image archive tarball to path. :param image: Union[str, ImageName], image pullspec to download :param path: str, path including the filename of the tarball """ cmd = ['skopeo', 'copy', f'docker://{image}', f'docker-archive:{path}'] try: retries.run_cmd(cmd) except subprocess.CalledProcessError as e: logger.error("Image archive download failed:\n%s", e.output) raise
def export_image(self, image_output_dir: Path) -> Dict[str, Union[str, int]]: output_path = self.workflow.build_dir.any_platform.exported_squashed_image cmd = ['skopeo', 'copy'] source_img = 'oci:{}'.format(image_output_dir) dest_img = 'docker-archive:{}'.format(output_path) cmd += [source_img, dest_img] try: retries.run_cmd(cmd) except subprocess.CalledProcessError as e: self.log.error("failed to save docker-archive :\n%s", e.output) raise return get_exported_image_metadata(str(output_path), IMAGE_TYPE_DOCKER_ARCHIVE)
def test_run_cmd_success(retries_needed, caplog): cmd = ["skopeo", "copy", "docker://a", "docker://b"] n_tries = 0 def mock_check_output(*args, **kwargs): nonlocal n_tries n_tries += 1 if n_tries > retries_needed: return b'some output' raise subprocess.CalledProcessError(1, cmd, output=b'something went wrong') (flexmock(subprocess).should_receive('check_output').with_args( cmd, stderr=subprocess.STDOUT).times(retries_needed + 1).replace_with(mock_check_output)) flexmock(time).should_receive('sleep').times(retries_needed) assert retries.run_cmd(cmd) == b'some output' assert caplog.text.count( 'Running skopeo copy docker://a docker://b') == retries_needed + 1 assert caplog.text.count( 'skopeo failed with:\nsomething went wrong') == retries_needed for n in range(retries_needed): wait = SUBPROCESS_BACKOFF_FACTOR * 2**n assert f'Backing off run_cmd(...) for {wait:.1f}s' in caplog.text
def extract_file_from_image(self, image: Union[str, ImageName], src_path: str, dst_path: str) -> None: """ Extract file or directory from image at src_path to dst_path using 'oc image extract' command. This command has some peculiar behaviour that the users of this method should be aware of. - the dst_path must be an existing empty dir, otherwise the extraction fails - file permissions of the extracted files are not preserved - trying to extract nonexistent file fails silently - when extracting whole dir, it matters if src_path ends in / (e.g. /usr/local/bin vs /usr/local/bin/). If slash is used, only the files in the directory will be extraced. Else, the directory together will the files will be extracted :param image: Union[str, ImageName], image pullspec from which to extract :param src_path: str, path inside the image that points to file or directory that will be extracted :param dst_path: str, path where to export file/dir """ if any(Path(dst_path).iterdir()): raise NonEmptyDestinationError( f'the destination directory {dst_path} must be empty') cmd = [ 'oc', 'image', 'extract', f'{image}', '--path', f'{src_path}:{dst_path}' ] try: retries.run_cmd(cmd) except subprocess.CalledProcessError as e: raise ExtractionError( f"Image file extraction failed\n{e.output}") from e # check if something was extracted, as the extraction can fail # silently when extracting nonexisting files if not any(Path(dst_path).iterdir()): raise NothingExtractedError( f'Extraction failed, files at path {src_path}' ' not found in the image')
def test_run_cmd_success(retries_needed, caplog): cmd = ["skopeo", "copy", "docker://a", "docker://b"] n_tries = 0 def mock_run(*args, **kwargs): nonlocal n_tries n_tries += 1 if n_tries > retries_needed: return subprocess.CompletedProcess(cmd, 0, stdout=b'some output', stderr=b'some warning') else: raise subprocess.CalledProcessError(1, cmd, output=b'some output', stderr=b'some error') (flexmock(subprocess).should_receive('run').with_args( cmd, check=True, capture_output=True).times(retries_needed + 1).replace_with(mock_run)) flexmock(time).should_receive('sleep').times(retries_needed) # important: the output shouldn't include 'some warning' assert retries.run_cmd(cmd) == b'some output' assert caplog.text.count( 'Running skopeo copy docker://a docker://b') == retries_needed + 1 assert caplog.text.count('skopeo failed:\n' 'STDOUT:\n' 'some output\n' 'STDERR:\n' 'some error') == retries_needed assert caplog.text.count('skopeo STDERR:\nsome warning') == 1 for n in range(retries_needed): wait = SUBPROCESS_BACKOFF_FACTOR * 2**n assert f'Backing off run_cmd(...) for {wait:.1f}s' in caplog.text
def get_image_size(self, dest_tag: ImageName) -> int: inspect_cmd = [ *self._podman_remote_cmd, 'image', 'inspect', str(dest_tag) ] try: output = retries.run_cmd(inspect_cmd) inspect_json = json.loads(output) image_inspect = inspect_json[0] image_size = image_inspect['Size'] except subprocess.CalledProcessError as e: raise InspectError( f"Couldn't check image size for image:{str(dest_tag)}." f" (rc={e.returncode})") from e except IndexError as e: raise InspectError( "Image inspect didn't return any results for image:" f"{str(dest_tag)}") from e except JSONDecodeError as e: raise InspectError("Image inspect returned invalid JSON for image:" f"{str(dest_tag)}") from e return image_size