def push_container(self,
dest_tag: ImageName,
*,
insecure: bool = False) -> None:
"""Push the built container (named dest_tag) to the registry (as dest_tag).
Push the container as v2s2 (Docker v2 schema 2) regardless of the original format.
:param dest_tag: the name of the built container, and the destination for the push
:param insecure: disable --tls-verify?
"""
options = ["--format=v2s2"]
if self._registries_authfile:
options.append(f"--authfile={self._registries_authfile}")
if insecure:
options.append("--tls-verify=false")
push_cmd = [*self._podman_remote_cmd, "push", *options, str(dest_tag)]
try:
retries.run_cmd(push_cmd)
except subprocess.CalledProcessError as e:
raise PushError(
f"Push failed (rc={e.returncode}). Check the logs for more details."
) from e
def test_run_cmd_failure(caplog):
cmd = ["skopeo", "copy", "docker://a", "docker://b"]
total_tries = SUBPROCESS_MAX_RETRIES + 1
(flexmock(subprocess).should_receive('run').with_args(
cmd, check=True, capture_output=True).times(total_tries).and_raise(
subprocess.CalledProcessError(1,
cmd,
output=b'',
stderr=b'something went wrong')))
flexmock(time).should_receive('sleep').times(SUBPROCESS_MAX_RETRIES)
with pytest.raises(subprocess.CalledProcessError):
retries.run_cmd(cmd)
assert caplog.text.count(
'Running skopeo copy docker://a docker://b') == total_tries
assert caplog.text.count('skopeo failed:\n'
'STDOUT:\n'
'\n'
'STDERR:\n'
'something went wrong') == total_tries
for n in range(SUBPROCESS_MAX_RETRIES):
wait = SUBPROCESS_BACKOFF_FACTOR * 2**n
assert f'Backing off run_cmd(...) for {wait:.1f}s' in caplog.text
assert f'Giving up run_cmd(...) after {total_tries} tries' in caplog.text
def push_with_skopeo(self, image: Dict[str, str], registry_image: ImageName, insecure: bool,
docker_push_secret: str) -> None:
cmd = ['skopeo', 'copy']
if docker_push_secret is not None:
dockercfg = Dockercfg(docker_push_secret)
cmd.append('--authfile=' + dockercfg.json_secret_path)
if insecure:
cmd.append('--dest-tls-verify=false')
if image['type'] == IMAGE_TYPE_OCI:
# ref_name is added by 'flatpak_create_oci'
# we have to be careful when changing the source container image type
# since assumption here is that source container image will always be 'docker-archive'
source_img = 'oci:{path}:{ref_name}'.format(**image)
cmd.append('--format=v2s2')
elif image['type'] == IMAGE_TYPE_DOCKER_ARCHIVE:
source_img = 'docker-archive://{path}'.format(**image)
else:
raise RuntimeError("Attempt to push unsupported image type %s with skopeo" %
image['type'])
dest_img = 'docker://' + registry_image.to_str()
cmd += [source_img, dest_img]
try:
retries.run_cmd(cmd)
except subprocess.CalledProcessError as e:
self.log.error("push failed with output:\n%s", e.output)
raise
def build_flatpak_image(self, source, build_dir: BuildDir) -> Dict[str, Any]:
builder = FlatpakBuilder(source, build_dir.path,
'var/tmp/flatpak-build',
parse_manifest=parse_rpm_output,
flatpak_metadata=self.flatpak_metadata)
df_labels = build_dir.dockerfile_with_parent_env(
self.workflow.imageutil.base_image_inspect()
).labels
builder.add_labels(df_labels)
tmp_dir = tempfile.mkdtemp(dir=build_dir.path)
image_filesystem = self.workflow.imageutil.extract_filesystem_layer(
str(build_dir.exported_squashed_image), str(tmp_dir))
build_dir.exported_squashed_image.unlink()
filesystem_path = os.path.join(tmp_dir, image_filesystem)
with open(filesystem_path, 'rb') as f:
# this part is 'not ideal' but this function seems to be a prerequisite
# for building flatpak image since it does the setup for it
flatpak_filesystem, flatpak_manifest = builder._export_from_stream(f)
os.remove(filesystem_path)
self.log.info('filesystem tarfile written to %s', flatpak_filesystem)
image_rpm_components = builder.get_components(flatpak_manifest)
ref_name, outfile, outfile_tarred = builder.build_container(flatpak_filesystem)
os.remove(outfile_tarred)
metadata = get_exported_image_metadata(outfile, IMAGE_TYPE_OCI)
metadata['ref_name'] = ref_name
cmd = ['skopeo', 'copy', 'oci:{path}:{ref_name}'.format(**metadata), '--format=v2s2',
'docker-archive:{}'.format(str(build_dir.exported_squashed_image))]
try:
retries.run_cmd(cmd)
except subprocess.CalledProcessError as e:
self.log.error("skopeo copy failed with output:\n%s", e.output)
raise RuntimeError("skopeo copy failed with output:\n{}".format(e.output)) from e
self.log.info('OCI image is available as %s', outfile)
shutil.rmtree(tmp_dir)
self.workflow.data.image_components[build_dir.platform] = image_rpm_components
return metadata
def download_image_archive_tarball(self, image: Union[str, ImageName],
path: str) -> None:
"""Downloads image archive tarball to path.
:param image: Union[str, ImageName], image pullspec to download
:param path: str, path including the filename of the tarball
"""
cmd = ['skopeo', 'copy', f'docker://{image}', f'docker-archive:{path}']
try:
retries.run_cmd(cmd)
except subprocess.CalledProcessError as e:
logger.error("Image archive download failed:\n%s", e.output)
raise
def export_image(self, image_output_dir: Path) -> Dict[str, Union[str, int]]:
output_path = self.workflow.build_dir.any_platform.exported_squashed_image
cmd = ['skopeo', 'copy']
source_img = 'oci:{}'.format(image_output_dir)
dest_img = 'docker-archive:{}'.format(output_path)
cmd += [source_img, dest_img]
try:
retries.run_cmd(cmd)
except subprocess.CalledProcessError as e:
self.log.error("failed to save docker-archive :\n%s", e.output)
raise
return get_exported_image_metadata(str(output_path), IMAGE_TYPE_DOCKER_ARCHIVE)
def test_run_cmd_success(retries_needed, caplog):
cmd = ["skopeo", "copy", "docker://a", "docker://b"]
n_tries = 0
def mock_check_output(*args, **kwargs):
nonlocal n_tries
n_tries += 1
if n_tries > retries_needed:
return b'some output'
raise subprocess.CalledProcessError(1,
cmd,
output=b'something went wrong')
(flexmock(subprocess).should_receive('check_output').with_args(
cmd,
stderr=subprocess.STDOUT).times(retries_needed +
1).replace_with(mock_check_output))
flexmock(time).should_receive('sleep').times(retries_needed)
assert retries.run_cmd(cmd) == b'some output'
assert caplog.text.count(
'Running skopeo copy docker://a docker://b') == retries_needed + 1
assert caplog.text.count(
'skopeo failed with:\nsomething went wrong') == retries_needed
for n in range(retries_needed):
wait = SUBPROCESS_BACKOFF_FACTOR * 2**n
assert f'Backing off run_cmd(...) for {wait:.1f}s' in caplog.text
def extract_file_from_image(self, image: Union[str, ImageName],
src_path: str, dst_path: str) -> None:
"""
Extract file or directory from image at src_path to dst_path
using 'oc image extract' command. This command has some peculiar
behaviour that the users of this method should be aware of.
- the dst_path must be an existing empty dir, otherwise
the extraction fails
- file permissions of the extracted files are not preserved
- trying to extract nonexistent file fails silently
- when extracting whole dir, it matters if src_path ends
in / (e.g. /usr/local/bin vs /usr/local/bin/).
If slash is used, only the files in the directory will
be extraced. Else, the directory together will the files
will be extracted
:param image: Union[str, ImageName], image pullspec from which to extract
:param src_path: str, path inside the image that points to file or directory
that will be extracted
:param dst_path: str, path where to export file/dir
"""
if any(Path(dst_path).iterdir()):
raise NonEmptyDestinationError(
f'the destination directory {dst_path} must be empty')
cmd = [
'oc', 'image', 'extract', f'{image}', '--path',
f'{src_path}:{dst_path}'
]
try:
retries.run_cmd(cmd)
except subprocess.CalledProcessError as e:
raise ExtractionError(
f"Image file extraction failed\n{e.output}") from e
# check if something was extracted, as the extraction can fail
# silently when extracting nonexisting files
if not any(Path(dst_path).iterdir()):
raise NothingExtractedError(
f'Extraction failed, files at path {src_path}'
' not found in the image')
def test_run_cmd_success(retries_needed, caplog):
cmd = ["skopeo", "copy", "docker://a", "docker://b"]
n_tries = 0
def mock_run(*args, **kwargs):
nonlocal n_tries
n_tries += 1
if n_tries > retries_needed:
return subprocess.CompletedProcess(cmd,
0,
stdout=b'some output',
stderr=b'some warning')
else:
raise subprocess.CalledProcessError(1,
cmd,
output=b'some output',
stderr=b'some error')
(flexmock(subprocess).should_receive('run').with_args(
cmd, check=True,
capture_output=True).times(retries_needed + 1).replace_with(mock_run))
flexmock(time).should_receive('sleep').times(retries_needed)
# important: the output shouldn't include 'some warning'
assert retries.run_cmd(cmd) == b'some output'
assert caplog.text.count(
'Running skopeo copy docker://a docker://b') == retries_needed + 1
assert caplog.text.count('skopeo failed:\n'
'STDOUT:\n'
'some output\n'
'STDERR:\n'
'some error') == retries_needed
assert caplog.text.count('skopeo STDERR:\nsome warning') == 1
for n in range(retries_needed):
wait = SUBPROCESS_BACKOFF_FACTOR * 2**n
assert f'Backing off run_cmd(...) for {wait:.1f}s' in caplog.text
def get_image_size(self, dest_tag: ImageName) -> int:
inspect_cmd = [
*self._podman_remote_cmd, 'image', 'inspect',
str(dest_tag)
]
try:
output = retries.run_cmd(inspect_cmd)
inspect_json = json.loads(output)
image_inspect = inspect_json[0]
image_size = image_inspect['Size']
except subprocess.CalledProcessError as e:
raise InspectError(
f"Couldn't check image size for image:{str(dest_tag)}."
f" (rc={e.returncode})") from e
except IndexError as e:
raise InspectError(
"Image inspect didn't return any results for image:"
f"{str(dest_tag)}") from e
except JSONDecodeError as e:
raise InspectError("Image inspect returned invalid JSON for image:"
f"{str(dest_tag)}") from e
return image_size
