def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
if base_image:
Path.create(self.oci.container_dir)
image_tar = ArchiveTar(base_image)
image_tar.extract(self.oci.container_dir)
self.oci.init_layout(bool(base_image))
self.oci.unpack()
self.oci.sync_rootfs(''.join([self.root_dir, os.sep]), exclude_list)
self.oci.repack(self.oci_config)
if 'additional_tags' in self.oci_config:
for tag in self.oci_config['additional_tags']:
self.oci.add_tag(tag)
self.oci.set_config(self.oci_config, bool(base_image))
self.oci.garbage_collect()
return self.pack_image_to_file(filename)
def test_create_from_dir_with_excludes(self, mock_command):
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', '.', '--exclude', './foo', '--exclude', './bar'
])
def import_overlay_files(self,
follow_links=False,
preserve_owner_group=False):
"""
Copy overlay files from the image description to
the image root tree. Supported are a root/ directory
or a root.tar.gz tarball. The root/ directory takes
precedence over the tarball
:param bool follow_links: follow symlinks true|false
:param bool preserve_owner_group: preserve permissions true|false
"""
overlay_directory = self.description_dir + '/root/'
overlay_archive = self.description_dir + '/root.tar.gz'
if os.path.exists(overlay_directory):
log.info('Copying user defined files to image tree')
sync_options = [
'-r', '-p', '-t', '-D', '-H', '-X', '-A', '--one-file-system'
]
if follow_links:
sync_options.append('--copy-links')
else:
sync_options.append('--links')
if preserve_owner_group:
sync_options.append('-o')
sync_options.append('-g')
data = DataSync(overlay_directory, self.root_dir)
data.sync_data(options=sync_options)
elif os.path.exists(overlay_archive):
log.info(
'Extracting user defined files from archive to image tree')
archive = ArchiveTar(overlay_archive)
archive.extract(self.root_dir)
def create_image_format(self) -> None:
"""
Create GCE disk format and manifest
"""
gce_tar_ball_file_list = []
temp_image_dir = Temporary(prefix='kiwi_gce_subformat.',
dir=self.target_dir).new_dir()
diskname = ''.join([
self.target_dir, '/',
self.xml_state.xml_data.get_name(), '.' + self.arch,
'-' + self.xml_state.get_image_version(), '.raw'
])
if self.tag:
with open(temp_image_dir.name + '/manifest.json', 'w') as manifest:
manifest.write('{"licenses": ["%s"]}' % self.tag)
gce_tar_ball_file_list.append('manifest.json')
Command.run(['cp', diskname, temp_image_dir.name + '/disk.raw'])
gce_tar_ball_file_list.append('disk.raw')
archive_name = os.path.basename(
self.get_target_file_path_for_format(self.image_format))
# delete the '.gz' suffix from the name. The suffix is appended by
# the archive creation method depending on the creation type.
archive_name = archive_name.replace('.gz', '')
archive = ArchiveTar(filename=self.target_dir + '/' + archive_name,
file_list=gce_tar_ball_file_list)
archive.create_gnu_gzip_compressed(temp_image_dir.name)
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image',
'{0}:base_layer'.format(self.oci_layout_dir), self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep])
)
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image', self.oci_layout_dir,
self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep]))
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
class TestArchiveTar(object):
def setup(self):
self.archive = ArchiveTar('foo.tar')
@patch('kiwi.archive.tar.Command.run')
def test_extract(self, mock_command):
self.archive.extract('destination')
mock_command.assert_called_once_with(
['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar'])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create('source-dir')
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
def test_create_from_dir_with_excludes(self, mock_command):
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', '.', '--exclude', './foo', '--exclude', './bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create_xz_compressed('source-dir')
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-J', '-f', 'foo.tar.xz', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create_gnu_gzip_compressed('source-dir')
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--format=gnu', '-cSz', '-f',
'foo.tar.gz', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_exclude(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create('source-dir', ['foo'])
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', 'bar'
])
def pack_image_to_file(self, filename):
"""
Packs the oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
oci_tarfile = ArchiveTar(filename.replace('.xz', ''))
oci_tarfile.create_xz_compressed(image_dir, xz_options=self.xz_options)
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains the raw disk image and its checksum
as well as an install initrd and kernel plus the required
kernel commandline information which needs to be added
as append line in the pxelinux config file on the boot
server
Image types which triggers this builder are:
* installpxe="true|false"
"""
self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.',
dir=self.target_dir)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.xz'])
compress = Compress(source_filename=self.diskname,
keep_source_on_compress=True)
compress.xz(self.xz_options)
Command.run(['mv', compress.compressed_filename, pxe_image_filename])
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.md5'])
checksum = Checksum(self.diskname)
checksum.md5(pxe_md5_filename)
# the kiwi initrd code triggers the install by trigger files
self._create_pxe_install_trigger_files()
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.append'])
cmdline = 'pxe=1'
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self._create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(self.pxename.replace('.xz', ''))
archive.create_xz_compressed(self.pxe_dir, xz_options=self.xz_options)
def create(self) -> Result:
"""
Create a root archive tarball
Build a simple XZ compressed root tarball from the image root tree
Image types which triggers this builder are:
* image="tbz"
:return: result
:rtype: instance of :class:`Result`
"""
supported_archives = Defaults.get_archive_image_types()
if self.requested_archive_type not in supported_archives:
raise KiwiArchiveSetupError('Unknown archive type: %s' %
self.requested_archive_type)
if self.requested_archive_type == 'tbz':
log.info('Creating XZ compressed tar archive')
archive = ArchiveTar(self._target_file_for('tar'))
archive.create_xz_compressed(
self.root_dir,
xz_options=self.xz_options,
exclude=Defaults.get_exclude_list_for_root_data_sync() +
Defaults.get_exclude_list_from_custom_exclude_files(
self.root_dir))
Result.verify_image_size(
self.runtime_config.get_max_size_constraint(), self.filename)
if self.bundle_format:
self.result.add_bundle_format(self.bundle_format)
self.result.add(key='root_archive',
filename=self.filename,
use_for_bundle=True,
compress=False,
shasum=True)
self.result.add(key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(key='image_changes',
filename=self.system_setup.export_package_changes(
self.target_dir),
use_for_bundle=True,
compress=True,
shasum=False)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
def test_create_from_dir_with_excludes(self, mock_command):
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.',
'--exclude', './foo', '--exclude', './bar'
]
)
def pack_image_to_file(self, filename):
"""
Packs the oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
oci_tarfile = ArchiveTar(filename.replace('.xz', ''))
oci_tarfile.create_xz_compressed(
image_dir, xz_options=self.xz_options
)
def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')
container_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
container_name = ':'.join([container_dir, self.container_tag])
if base_image:
Path.create(container_dir)
image_tar = ArchiveTar(base_image)
image_tar.extract(container_dir)
Command.run([
'umoci', 'config', '--image',
'{0}:base_layer'.format(container_dir), '--tag',
self.container_tag
])
else:
Command.run(['umoci', 'init', '--layout', container_dir])
Command.run(['umoci', 'new', '--image', container_name])
Command.run(
['umoci', 'unpack', '--image', container_name, self.oci_root_dir])
oci_root = DataSync(''.join([self.root_dir, os.sep]),
os.sep.join([self.oci_root_dir, 'rootfs']))
oci_root.sync_data(options=['-a', '-H', '-X', '-A', '--delete'],
exclude=exclude_list)
Command.run(
['umoci', 'repack', '--image', container_name, self.oci_root_dir])
for tag in self.additional_tags:
Command.run(
['umoci', 'config', '--image', container_name, '--tag', tag])
Command.run(['umoci', 'config'] + self.maintainer + self.user +
self.workingdir + self.entry_command +
self.entry_subcommand + self.expose_ports + self.volumes +
self.environment + self.labels + [
'--image', container_name, '--created',
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00')
])
Command.run(['umoci', 'gc', '--layout', container_dir])
return self.pack_image_to_file(filename)
def create(self, filename, base_image=None, ensure_empty_tmpdirs=None):
"""
Create WSL/Appx archive
:param string filename: archive file name
:param string base_image: not-supported
:param string ensure_empty_tmpdirs: not-supported
"""
exclude_list = Defaults.\
get_exclude_list_for_root_data_sync() + Defaults.\
get_exclude_list_from_custom_exclude_files(self.root_dir)
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
# The C code of WSL-DistroLauncher harcodes the name for the
# root tarball to be install.tar.gz. Thus we have to use this
# name for the root tarball
archive_file_name = os.sep.join(
[self.meta_data_path, 'install.tar']
)
archive = ArchiveTar(
archive_file_name
)
archive_file_name = archive.create(
self.root_dir, exclude=exclude_list
)
compressor = Compress(archive_file_name)
archive_file_name = compressor.gzip()
filemap_file = Temporary().new_file()
with open(filemap_file.name, 'w') as filemap:
filemap.write('[Files]{0}'.format(os.linesep))
for topdir, dirs, files in sorted(os.walk(self.meta_data_path)):
for entry in sorted(dirs + files):
if entry in files:
mapfile = os.sep.join([topdir, entry])
mapfile_relative = os.path.relpath(mapfile, start=self.meta_data_path)
log.info(
'Adding {0} to Appx filemap as relative path {1}'.format(mapfile, mapfile_relative)
)
filemap.write(
'"{0}" "{1}"{2}'.format(
mapfile, mapfile_relative, os.linesep
)
)
Command.run(
['appx', '-o', filename, '-f', filemap_file.name]
)
return filename
def pack_image_to_file(self, filename):
"""
Packs the oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
oci_tarfile = ArchiveTar(filename)
container_compressor = self.runtime_config.get_container_compression()
if container_compressor:
return oci_tarfile.create_xz_compressed(
image_dir, xz_options=self.runtime_config.get_xz_options())
else:
return oci_tarfile.create(image_dir)
def pack_image_to_file(self, filename):
"""
Packs the oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
oci_tarfile = ArchiveTar(filename)
container_compressor = self.runtime_config.get_container_compression()
if container_compressor:
return oci_tarfile.create_xz_compressed(
image_dir, xz_options=self.runtime_config.get_xz_options()
)
else:
return oci_tarfile.create(image_dir)
def import_cdroot_files(self, target_dir):
"""
Copy cdroot files from the image description to the
specified target directory. Supported is a tar
archive named config-cdroot.tar[.compression-postfix]
:param str target_dir: directory to unpack archive to
"""
glob_match = self.description_dir + '/config-cdroot.tar*'
for cdroot_archive in sorted(glob.iglob(glob_match)):
log.info('Extracting ISO user config archive: {0} to: {1}'.format(
cdroot_archive, target_dir))
archive = ArchiveTar(cdroot_archive)
archive.extract(target_dir)
break
def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
command = mock.Mock()
command.output = 'version 1.26.1'
mock_command.return_value = command
archive = ArchiveTar('foo.tar')
mock_os_dir.return_value = ['foo', 'bar']
assert archive.create('source-dir') == 'foo.tar'
calls = [
call(['tar', '--version']),
call([
'tar', '-C', 'source-dir', '-c', '-f', 'foo.tar', 'foo', 'bar'
])
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
def test_create_from_dir_with_excludes(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
archive = ArchiveTar('foo.tar', False)
assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
calls = [
call(['tar', '--version']),
call([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.', '--exclude', './foo', '--exclude',
'./bar'
])
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
def create(self):
"""
Create a root archive tarball
Build a simple XZ compressed root tarball from the image root tree
Image types which triggers this builder are:
* image="tbz"
"""
supported_archives = Defaults.get_archive_image_types()
if self.requested_archive_type not in supported_archives:
raise KiwiArchiveSetupError('Unknown archive type: %s' %
self.requested_archive_type)
if self.requested_archive_type == 'tbz':
log.info('Creating XZ compressed tar archive')
archive = ArchiveTar(self._target_file_for('tar'))
archive.create_xz_compressed(
self.root_dir,
xz_options=self.xz_options,
exclude=Defaults.get_exclude_list_for_root_data_sync())
checksum = Checksum(self.filename)
log.info('--> Creating archive checksum')
checksum.md5(self.checksum)
self.result.add(key='root_archive',
filename=self.filename,
use_for_bundle=True,
compress=False,
shasum=True)
self.result.add(key='root_archive_md5',
filename=self.checksum,
use_for_bundle=False)
self.result.add(key='image_packages',
filename=self.system_setup.export_rpm_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(
key='image_verified',
filename=self.system_setup.export_rpm_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
def _install_archives(self, archive_list):
log.info("Installing archives")
for archive in archive_list:
log.info("--> archive: %s", archive)
description_dir = \
self.xml_state.xml_data.description_dir
derived_description_dir = \
self.xml_state.xml_data.derived_description_dir
archive_is_absolute = archive.startswith('/')
if archive_is_absolute:
archive_file = archive
else:
archive_file = '/'.join([description_dir, archive])
archive_exists = os.path.exists(archive_file)
if not archive_is_absolute and not archive_exists and derived_description_dir:
archive_file = '/'.join([derived_description_dir, archive])
tar = ArchiveTar(archive_file)
tar.extract(self.root_bind.root_dir)
def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
command = mock.Mock()
command.output = 'version 1.26.1'
mock_command.return_value = command
archive = ArchiveTar('foo.tar')
mock_os_dir.return_value = ['foo', 'bar']
assert archive.create('source-dir') == 'foo.tar'
calls = [
call(['tar', '--version']),
call(
[
'tar', '-C', 'source-dir',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
def test_create_from_dir_with_excludes(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
archive = ArchiveTar('foo.tar', False)
assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
calls = [
call(['tar', '--version']),
call(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.',
'--exclude', './foo', '--exclude', './bar'
]
)
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
def import_overlay_files(
self, follow_links: bool = False, preserve_owner_group: bool = False
) -> None:
"""
Copy overlay files from the image description to
the image root tree. Supported are a root/ directory
or a root.tar.gz tarball. The root/ directory takes
precedence over the tarball.
In addition the method also supports profile specific
overlay files which are searched in a directory of the
same name as the profile name.
The overall order for including overlay files is as
follows:
1. root/ dir or root.tar.gz
2. PROFILE_NAME/ dir(s) in the order of the selected
profiles
:param bool follow_links: follow symlinks true|false
:param bool preserve_owner_group: preserve permissions true|false
"""
overlay_directory = self.description_dir + '/root/'
overlay_archive = self.description_dir + '/root.tar.gz'
if os.path.exists(overlay_directory):
self._sync_overlay_files(
overlay_directory, follow_links, preserve_owner_group
)
elif os.path.exists(overlay_archive):
log.info('Extracting user defined files from archive to image tree')
archive = ArchiveTar(overlay_archive)
archive.extract(self.root_dir)
for profile in self.xml_state.profiles:
overlay_directory = os.path.join(
self.description_dir, profile
) + os.sep
if os.path.exists(overlay_directory):
self._sync_overlay_files(
overlay_directory, follow_links, preserve_owner_group,
profile
)
def create_image_format(self):
"""
Create GCE disk format and manifest
"""
gce_tar_ball_file_list = []
self.temp_image_dir = mkdtemp(
prefix='kiwi_gce_subformat.', dir=self.target_dir
)
diskname = ''.join(
[
self.target_dir, '/',
self.xml_state.xml_data.get_name(),
'.' + self.arch,
'-' + self.xml_state.get_image_version(),
'.raw'
]
)
if self.tag:
with open(self.temp_image_dir + '/manifest.json', 'w') as manifest:
manifest.write('{"licenses": ["%s"]}' % self.tag)
gce_tar_ball_file_list.append('manifest.json')
Command.run(
['cp', diskname, self.temp_image_dir + '/disk.raw']
)
gce_tar_ball_file_list.append('disk.raw')
archive_name = os.path.basename(
self.get_target_file_path_for_format(self.image_format)
)
# delete the '.gz' suffix from the name. The suffix is appended by
# the archive creation method depending on the creation type.
archive_name = archive_name.replace('.gz', '')
archive = ArchiveTar(
filename=self.target_dir + '/' + archive_name,
file_list=gce_tar_ball_file_list
)
archive.create_gnu_gzip_compressed(
self.temp_image_dir
)
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
oci = OCI('base_layer', self.oci_layout_dir)
oci.unpack()
oci.import_rootfs(self.root_dir)
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def extract_oci_image(self):
"""
Extract the image from the provided image file to a temporary
location to KIWI can work with it.
"""
if not self.unknown_uri:
tar = ArchiveTar(self.image_file)
self.uncompressed_image = mkdtemp(prefix='kiwi_uncompressed.')
tar.extract(self.uncompressed_image)
if self.tag:
skopeo_uri = 'oci:{0}:{1}'.format(self.uncompressed_image,
self.tag)
else:
skopeo_uri = 'oci:{0}'.format(self.uncompressed_image)
else:
log.warning('Bypassing base image URI to skopeo tool')
skopeo_uri = self.unknown_uri
Command.run([
'skopeo', 'copy', skopeo_uri, 'oci:{0}'.format(self.oci_layout_dir)
])
def _install_archives(self, archive_list):
log.info("Installing archives")
for archive in archive_list:
log.info("--> archive: %s", archive)
description_dir = \
self.xml_state.xml_data.description_dir
derived_description_dir = \
self.xml_state.xml_data.derived_description_dir
archive_is_absolute = archive.startswith('/')
if archive_is_absolute:
archive_file = archive
else:
archive_file = '/'.join(
[description_dir, archive]
)
archive_exists = os.path.exists(archive_file)
if not archive_is_absolute and not archive_exists and derived_description_dir:
archive_file = '/'.join(
[derived_description_dir, archive]
)
tar = ArchiveTar(archive_file)
tar.extract(self.root_bind.root_dir)
def extract_oci_image(self):
"""
Extract the contents from the provided image file to a temporary
location KIWI can work with.
"""
if not self.unknown_uri:
tar = ArchiveTar(self.image_file)
self.uncompressed_image = mkdtemp(prefix='kiwi_uncompressed.')
tar.extract(self.uncompressed_image)
if self.tag:
skopeo_uri = 'oci:{0}:{1}'.format(
self.uncompressed_image, self.tag
)
else:
skopeo_uri = 'oci:{0}'.format(self.uncompressed_image)
else:
log.warning('Bypassing base image URI to skopeo tool')
skopeo_uri = self.unknown_uri
Command.run([
'skopeo', 'copy', skopeo_uri,
'oci:{0}:base_layer'.format(self.oci_layout_dir)
])
def import_overlay_files(
self, follow_links=False, preserve_owner_group=False
):
"""
Copy overlay files from the image description to
the image root tree. Supported are a root/ directory
or a root.tar.gz tarball. The root/ directory takes
precedence over the tarball
:param bool follow_links: follow symlinks true|false
:param bool preserve_owner_group: preserve permissions true|false
"""
overlay_directory = self.description_dir + '/root/'
overlay_archive = self.description_dir + '/root.tar.gz'
if os.path.exists(overlay_directory):
log.info('Copying user defined files to image tree')
sync_options = [
'-r', '-p', '-t', '-D', '-H', '-X', '-A', '--one-file-system'
]
if follow_links:
sync_options.append('--copy-links')
else:
sync_options.append('--links')
if preserve_owner_group:
sync_options.append('-o')
sync_options.append('-g')
data = DataSync(
overlay_directory, self.root_dir
)
data.sync_data(
options=sync_options
)
elif os.path.exists(overlay_archive):
log.info('Extracting user defined files from archive to image tree')
archive = ArchiveTar(overlay_archive)
archive.extract(self.root_dir)
class TestArchiveTar(object):
def setup(self):
self.archive = ArchiveTar('foo.tar')
@patch('kiwi.archive.tar.Command.run')
def test_extract(self, mock_command):
self.archive.extract('destination')
mock_command.assert_called_once_with(
['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar']
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create('source-dir')
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
def test_create_from_dir_with_excludes(self, mock_command):
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.',
'--exclude', './foo', '--exclude', './bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create_xz_compressed('source-dir')
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--xattrs', '--xattrs-include=*',
'-c', '-J', '-f', 'foo.tar.xz', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create_gnu_gzip_compressed('source-dir')
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--format=gnu', '-cSz', '-f', 'foo.tar.gz', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_exclude(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create('source-dir', ['foo'])
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'bar'
]
)
def setup(self):
self.archive = ArchiveTar('foo.tar')
def setup(self):
self.archive = ArchiveTar('foo.tar')
def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')
container_dir = os.sep.join(
[self.oci_dir, 'umoci_layout']
)
container_name = ':'.join(
[container_dir, self.container_tag]
)
if base_image:
Path.create(container_dir)
image_tar = ArchiveTar(base_image)
image_tar.extract(container_dir)
Command.run([
'umoci', 'config', '--image',
'{0}:base_layer'.format(container_dir),
'--tag', self.container_tag
])
else:
Command.run(
['umoci', 'init', '--layout', container_dir]
)
Command.run(
['umoci', 'new', '--image', container_name]
)
Command.run(
['umoci', 'unpack', '--image', container_name, self.oci_root_dir]
)
oci_root = DataSync(
''.join([self.root_dir, os.sep]),
os.sep.join([self.oci_root_dir, 'rootfs'])
)
oci_root.sync_data(
options=['-a', '-H', '-X', '-A', '--delete'], exclude=exclude_list
)
Command.run(
['umoci', 'repack', '--image', container_name, self.oci_root_dir]
)
for tag in self.additional_tags:
Command.run(
['umoci', 'config', '--image', container_name, '--tag', tag]
)
Command.run(
[
'umoci', 'config'
] +
self.maintainer +
self.user +
self.workingdir +
self.entry_command +
self.entry_subcommand +
self.expose_ports +
self.volumes +
self.environment +
self.labels +
[
'--image', container_name,
'--created', datetime.utcnow().strftime(
'%Y-%m-%dT%H:%M:%S+00:00'
)
]
)
Command.run(
['umoci', 'gc', '--layout', container_dir]
)
return self.pack_image_to_file(filename)
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains:
* The raw system image xz compressed
* The raw system image checksum metadata file
* The append file template for the boot server
* The system image initrd for kexec
* The install initrd
* The kernel
Image types which triggers this builder are:
* installpxe="true|false"
"""
self.pxe_dir = mkdtemp(
prefix='kiwi_pxe_install_media.', dir=self.target_dir
)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.xz'
]
)
compress = Compress(
source_filename=self.diskname,
keep_source_on_compress=True
)
compress.xz(self.xz_options)
Command.run(
['mv', compress.compressed_filename, pxe_image_filename]
)
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.md5'
]
)
checksum = Checksum(self.diskname)
checksum.md5(pxe_md5_filename)
# the install image name is stored in a config file
if self.initrd_system == 'kiwi':
self._write_install_image_info_to_boot_image()
# the kexec required system image initrd is stored for dracut kiwi-dump
if self.initrd_system == 'dracut':
boot_names = self.boot_image_task.get_boot_names()
system_image_initrd = os.sep.join(
[self.root_dir, 'boot', boot_names.initrd_name]
)
target_initrd_name = '{0}/{1}.initrd'.format(
self.pxe_dir, self.xml_state.xml_data.get_name()
)
shutil.copy(
system_image_initrd, target_initrd_name
)
os.chmod(target_initrd_name, 420)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.append'
]
)
if self.initrd_system == 'kiwi':
cmdline = 'pxe=1'
else:
cmdline = ' '.join(
[
'rd.kiwi.install.pxe',
'rd.kiwi.install.image=http://example.com/image.xz'
]
)
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self._create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(
self.pxename.replace('.xz', '')
)
archive.create_xz_compressed(
self.pxe_dir, xz_options=self.xz_options
)
class TestArchiveTar(object):
@patch('kiwi.archive.tar.Command.run')
def setup(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
@raises(KiwiCommandCapabilitiesError)
@patch('kiwi.archive.tar.Command.run')
def test_invalid_tar_command_version(self, mock_command):
command = mock.Mock()
command.output = 'version cannot be parsed'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
@patch('kiwi.archive.tar.Command.run')
def test_extract(self, mock_command):
self.archive.extract('destination')
mock_command.assert_called_once_with(
['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar']
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir') == 'foo.tar'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
def test_append_files(self, mock_command):
assert self.archive.append_files('source-dir', ['foo', 'bar']) \
== 'foo.tar'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '-r',
'--file=' + self.archive.filename,
'--xattrs', '--xattrs-include=*',
'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_with_options(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir', options=[
'--fake-option', 'fake_arg'
]) == 'foo.tar'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--fake-option', 'fake_arg',
'--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
command = mock.Mock()
command.output = 'version 1.26.1'
mock_command.return_value = command
archive = ArchiveTar('foo.tar')
mock_os_dir.return_value = ['foo', 'bar']
assert archive.create('source-dir') == 'foo.tar'
calls = [
call(['tar', '--version']),
call(
[
'tar', '-C', 'source-dir',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
@patch('kiwi.archive.tar.Command.run')
def test_create_from_dir_with_excludes(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
archive = ArchiveTar('foo.tar', False)
assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
calls = [
call(['tar', '--version']),
call(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.',
'--exclude', './foo', '--exclude', './bar'
]
)
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_xz_compressed('source-dir') == 'foo.tar.xz'
mock_command.assert_called_once_with(
[
'bash', '-c',
' '.join([
'tar', '-C', 'source-dir', '--xattrs',
'--xattrs-include=*', '-c', '--to-stdout',
'foo', 'bar', '|', 'xz', '-f', '--threads=0',
'>', 'foo.tar.xz'
])
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed_with_custom_xz_options(
self, mock_os_dir, mock_command
):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_xz_compressed(
'source-dir', xz_options=['-a', '-b']
) == 'foo.tar.xz'
mock_command.assert_called_once_with(
[
'bash', '-c',
' '.join([
'tar', '-C', 'source-dir', '--xattrs',
'--xattrs-include=*', '-c', '--to-stdout',
'foo', 'bar', '|', 'xz', '-f', '-a', '-b',
'>', 'foo.tar.xz'
])
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_gnu_gzip_compressed('source-dir') \
== 'foo.tar.gz'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--format=gnu', '-cSz', '-f', 'foo.tar.gz', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_exclude(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir', ['foo']) == 'foo.tar'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'bar'
]
)
def create_recovery_archive(self):
"""
Create a compressed recovery archive from the root tree
for use with kiwi's recvoery system. The method creates
additional data into the image root filesystem which is
deleted prior to the creation of a new recovery data set
"""
# cleanup
bash_comand = [
'rm', '-f', self.root_dir + '/recovery.*'
]
Command.run(['bash', '-c', ' '.join(bash_comand)])
if not self.oemconfig['recovery']:
return
# recovery.tar
log.info('Creating recovery tar archive')
metadata = {
'archive_name':
self.root_dir + '/recovery.tar',
'archive_filecount':
self.root_dir + '/recovery.tar.files',
'archive_size':
self.root_dir + '/recovery.tar.size',
'partition_size':
self.root_dir + '/recovery.partition.size',
'partition_filesystem':
self.root_dir + '/recovery.tar.filesystem'
}
recovery_archive = NamedTemporaryFile(
delete=False
)
archive = ArchiveTar(
filename=recovery_archive.name,
create_from_file_list=False
)
archive.create(
source_dir=self.root_dir,
exclude=['dev', 'proc', 'sys'],
options=[
'--numeric-owner',
'--hard-dereference',
'--preserve-permissions'
]
)
Command.run(
['mv', recovery_archive.name, metadata['archive_name']]
)
# recovery.tar.filesystem
recovery_filesystem = self.xml_state.build_type.get_filesystem()
with open(metadata['partition_filesystem'], 'w') as partfs:
partfs.write('%s' % recovery_filesystem)
log.info(
'--> Recovery partition filesystem: %s', recovery_filesystem
)
# recovery.tar.files
bash_comand = [
'tar', '-tf', metadata['archive_name'], '|', 'wc', '-l'
]
tar_files_call = Command.run(
['bash', '-c', ' '.join(bash_comand)]
)
tar_files_count = int(tar_files_call.output.rstrip('\n'))
with open(metadata['archive_filecount'], 'w') as files:
files.write('%d\n' % tar_files_count)
log.info(
'--> Recovery file count: %d files', tar_files_count
)
# recovery.tar.size
recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
with open(metadata['archive_size'], 'w') as size:
size.write('%d' % recovery_archive_size_bytes)
log.info(
'--> Recovery uncompressed size: %d mbytes',
int(recovery_archive_size_bytes / 1048576)
)
# recovery.tar.gz
log.info('--> Compressing recovery archive')
compress = Compress(self.root_dir + '/recovery.tar')
compress.gzip()
# recovery.partition.size
recovery_archive_gz_size_mbytes = int(
os.path.getsize(metadata['archive_name'] + '.gz') / 1048576
)
recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
+ Defaults.get_recovery_spare_mbytes()
with open(metadata['partition_size'], 'w') as gzsize:
gzsize.write('%d' % recovery_partition_mbytes)
log.info(
'--> Recovery partition size: %d mbytes',
recovery_partition_mbytes
)
# delete recovery archive if inplace recovery is requested
# In this mode the recovery archive is created at install time
# and not at image creation time. However the recovery metadata
# is preserved in order to be able to check if enough space
# is available on the disk to create the recovery archive.
if self.oemconfig['recovery_inplace']:
log.info(
'--> Inplace recovery requested, deleting archive'
)
Path.wipe(metadata['archive_name'] + '.gz')
def setup(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
def test_invalid_tar_command_version(self, mock_command):
command = mock.Mock()
command.output = 'version cannot be parsed'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
def create(self) -> Result:
"""
Build a component image consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum.
Image types which triggers this builder are:
* image="kis"
* image="pxe"
:raises KiwiKisBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
if self.filesystem:
log.info('Creating root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating root filesystem MD5 checksum')
checksum = Checksum(self.image)
checksum.md5(self.checksum_name)
# prepare boot(initrd) root system
log.info('Creating boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiKisBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
hypervisor_data = kernel.get_xen_hypervisor()
if hypervisor_data:
self.hypervisor_filename = ''.join(
[
os.path.basename(self.image_name),
'-', hypervisor_data.name
]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiKisBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd
self.boot_image_task.create_initrd()
# create append information
# this information helps to configure the deployment infrastructure
if self.filesystem and self.filesystem.root_uuid \
and self.initrd_system == 'dracut':
cmdline = 'root=UUID={}'.format(self.filesystem.root_uuid)
if self.custom_cmdline:
cmdline += ' {}'.format(self.custom_cmdline)
with open(self.append_file, 'w') as append:
append.write(cmdline)
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
kis_tarball_files = [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.checksum_name),
]
if self.image:
kis_tarball_files.append(os.path.basename(self.image))
if self.filesystem and self.filesystem.root_uuid \
and self.initrd_system == 'dracut':
kis_tarball_files.append(os.path.basename(self.append_file))
kis_tarball = ArchiveTar(
self.archive_name,
create_from_file_list=True,
file_list=kis_tarball_files
)
if self.compressed:
self.archive_name = kis_tarball.create(self.target_dir)
else:
self.archive_name = kis_tarball.create_xz_compressed(
self.target_dir, xz_options=self.xz_options
)
Result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.archive_name
)
# store results
self.result.add(
key='kis_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(
default=False
),
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_changes',
filename=self.system_setup.export_package_changes(
self.target_dir
),
use_for_bundle=True,
compress=True,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result
class TestArchiveTar(object):
@patch('kiwi.archive.tar.Command.run')
def setup(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
@raises(KiwiCommandCapabilitiesError)
@patch('kiwi.archive.tar.Command.run')
def test_invalid_tar_command_version(self, mock_command):
command = mock.Mock()
command.output = 'version cannot be parsed'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
@patch('kiwi.archive.tar.Command.run')
def test_extract(self, mock_command):
self.archive.extract('destination')
mock_command.assert_called_once_with(
['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar'])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir') == 'foo.tar'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
def test_append_files(self, mock_command):
assert self.archive.append_files('source-dir', ['foo', 'bar']) \
== 'foo.tar'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '-r', '--file=' + self.archive.filename,
'--xattrs', '--xattrs-include=*', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_with_options(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir',
options=['--fake-option',
'fake_arg']) == 'foo.tar'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--fake-option', 'fake_arg', '--xattrs',
'--xattrs-include=*', '-c', '-f', 'foo.tar', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
command = mock.Mock()
command.output = 'version 1.26.1'
mock_command.return_value = command
archive = ArchiveTar('foo.tar')
mock_os_dir.return_value = ['foo', 'bar']
assert archive.create('source-dir') == 'foo.tar'
calls = [
call(['tar', '--version']),
call([
'tar', '-C', 'source-dir', '-c', '-f', 'foo.tar', 'foo', 'bar'
])
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
@patch('kiwi.archive.tar.Command.run')
def test_create_from_dir_with_excludes(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
archive = ArchiveTar('foo.tar', False)
assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
calls = [
call(['tar', '--version']),
call([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.', '--exclude', './foo', '--exclude',
'./bar'
])
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_xz_compressed('source-dir') == 'foo.tar.xz'
mock_command.assert_called_once_with([
'bash', '-c', ' '.join([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '--to-stdout', 'foo', 'bar', '|', 'xz', '-f',
'--threads=0', '>', 'foo.tar.xz'
])
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed_with_custom_xz_options(
self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_xz_compressed('source-dir',
xz_options=['-a', '-b'
]) == 'foo.tar.xz'
mock_command.assert_called_once_with([
'bash', '-c', ' '.join([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '--to-stdout', 'foo', 'bar', '|', 'xz', '-f', '-a', '-b',
'>', 'foo.tar.xz'
])
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_gnu_gzip_compressed('source-dir') \
== 'foo.tar.gz'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--format=gnu', '-cSz', '-f',
'foo.tar.gz', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_exclude(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir', ['foo']) == 'foo.tar'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', 'bar'
])
def test_invalid_tar_command_version(self, mock_command):
command = mock.Mock()
command.output = 'version cannot be parsed'
mock_command.return_value = command
with raises(KiwiCommandCapabilitiesError):
self.archive = ArchiveTar('foo.tar')
def test_invalid_tar_command_version(self, mock_command):
command = mock.Mock()
command.output = 'version cannot be parsed'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
def setup(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
def create_recovery_archive(self):
"""
Create a compressed recovery archive from the root tree
for use with kiwi's recvoery system. The method creates
additional data into the image root filesystem which is
deleted prior to the creation of a new recovery data set
"""
# cleanup
bash_comand = ['rm', '-f', self.root_dir + '/recovery.*']
Command.run(['bash', '-c', ' '.join(bash_comand)])
if not self.oemconfig['recovery']:
return
# recovery.tar
log.info('Creating recovery tar archive')
metadata = {
'archive_name': self.root_dir + '/recovery.tar',
'archive_filecount': self.root_dir + '/recovery.tar.files',
'archive_size': self.root_dir + '/recovery.tar.size',
'partition_size': self.root_dir + '/recovery.partition.size',
'partition_filesystem': self.root_dir + '/recovery.tar.filesystem'
}
recovery_archive = NamedTemporaryFile(delete=False)
archive = ArchiveTar(filename=recovery_archive.name,
create_from_file_list=False)
archive.create(source_dir=self.root_dir,
exclude=['dev', 'proc', 'sys'],
options=[
'--numeric-owner', '--hard-dereference',
'--preserve-permissions'
])
Command.run(['mv', recovery_archive.name, metadata['archive_name']])
# recovery.tar.filesystem
recovery_filesystem = self.xml_state.build_type.get_filesystem()
with open(metadata['partition_filesystem'], 'w') as partfs:
partfs.write('%s' % recovery_filesystem)
log.info('--> Recovery partition filesystem: %s', recovery_filesystem)
# recovery.tar.files
bash_comand = ['tar', '-tf', metadata['archive_name'], '|', 'wc', '-l']
tar_files_call = Command.run(['bash', '-c', ' '.join(bash_comand)])
tar_files_count = int(tar_files_call.output.rstrip('\n'))
with open(metadata['archive_filecount'], 'w') as files:
files.write('%d\n' % tar_files_count)
log.info('--> Recovery file count: %d files', tar_files_count)
# recovery.tar.size
recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
with open(metadata['archive_size'], 'w') as size:
size.write('%d' % recovery_archive_size_bytes)
log.info('--> Recovery uncompressed size: %d mbytes',
int(recovery_archive_size_bytes / 1048576))
# recovery.tar.gz
log.info('--> Compressing recovery archive')
compress = Compress(self.root_dir + '/recovery.tar')
compress.gzip()
# recovery.partition.size
recovery_archive_gz_size_mbytes = int(
os.path.getsize(metadata['archive_name'] + '.gz') / 1048576)
recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
+ Defaults.get_recovery_spare_mbytes()
with open(metadata['partition_size'], 'w') as gzsize:
gzsize.write('%d' % recovery_partition_mbytes)
log.info('--> Recovery partition size: %d mbytes',
recovery_partition_mbytes)
# delete recovery archive if inplace recovery is requested
# In this mode the recovery archive is created at install time
# and not at image creation time. However the recovery metadata
# is preserved in order to be able to check if enough space
# is available on the disk to create the recovery archive.
if self.oemconfig['recovery_inplace']:
log.info('--> Inplace recovery requested, deleting archive')
Path.wipe(metadata['archive_name'] + '.gz')
def create(self):
"""
Create a root archive tarball
Build a simple XZ compressed root tarball from the image root tree
Image types which triggers this builder are:
* image="tbz"
"""
supported_archives = Defaults.get_archive_image_types()
if self.requested_archive_type not in supported_archives:
raise KiwiArchiveSetupError(
'Unknown archive type: %s' % self.requested_archive_type
)
if self.requested_archive_type == 'tbz':
log.info('Creating XZ compressed tar archive')
archive = ArchiveTar(
self._target_file_for('tar')
)
archive.create_xz_compressed(
self.root_dir, xz_options=self.xz_options,
exclude=Defaults.get_exclude_list_for_root_data_sync()
)
checksum = Checksum(self.filename)
log.info('--> Creating archive checksum')
checksum.md5(self.checksum)
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.filename
)
self.result.add(
key='root_archive',
filename=self.filename,
use_for_bundle=True,
compress=False,
shasum=True
)
self.result.add(
key='root_archive_md5',
filename=self.checksum,
use_for_bundle=False
)
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains:
* The raw system image xz compressed
* The raw system image checksum metadata file
* The append file template for the boot server
* The system image initrd for kexec
* The install initrd
* The kernel
Image types which triggers this builder are:
* installpxe="true|false"
"""
self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.',
dir=self.target_dir)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.xz'])
compress = Compress(source_filename=self.diskname,
keep_source_on_compress=True)
compress.xz(self.xz_options)
Command.run(['mv', compress.compressed_filename, pxe_image_filename])
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.md5'])
checksum = Checksum(self.diskname)
checksum.md5(pxe_md5_filename)
# the install image name is stored in a config file
if self.initrd_system == 'kiwi':
self._write_install_image_info_to_boot_image()
# the kexec required system image initrd is stored for dracut kiwi-dump
if self.initrd_system == 'dracut':
boot_names = self.boot_image_task.get_boot_names()
system_image_initrd = os.sep.join(
[self.root_dir, 'boot', boot_names.initrd_name])
target_initrd_name = '{0}/{1}.initrd'.format(
self.pxe_dir, self.xml_state.xml_data.get_name())
shutil.copy(system_image_initrd, target_initrd_name)
os.chmod(target_initrd_name, 420)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.append'])
if self.initrd_system == 'kiwi':
cmdline = 'pxe=1'
else:
cmdline = ' '.join([
'rd.kiwi.install.pxe',
'rd.kiwi.install.image=http://example.com/image.xz'
])
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self._create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(self.pxename.replace('.xz', ''))
archive.create_xz_compressed(self.pxe_dir, xz_options=self.xz_options)
def create(self):
"""
Build a pxe image set consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum. The result can be used within the kiwi
PXE boot infrastructure
Image types which triggers this builder are:
* image="pxe"
:raises KiwiPxeBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
log.info('Creating PXE root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
checksum = Checksum(self.image)
checksum.md5(self.checksum_name)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[os.path.basename(self.image_name), '-', kernel_data.name]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
pxe_tarball_files = [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.image),
os.path.basename(self.checksum_name)
]
pxe_tarball = ArchiveTar(
self.archive_name,
create_from_file_list=True,
file_list=pxe_tarball_files
)
if self.compressed:
self.archive_name = pxe_tarball.create(self.target_dir)
else:
self.archive_name = pxe_tarball.create_xz_compressed(
self.target_dir, xz_options=self.xz_options
)
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.archive_name
)
# store results
self.result.add(
key='pxe_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(
default=False
),
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result