def import_overlay_files(self,
follow_links=False,
preserve_owner_group=False):
"""
Copy overlay files from the image description to
the image root tree. Supported are a root/ directory
or a root.tar.gz tarball. The root/ directory takes
precedence over the tarball
:param bool follow_links: follow symlinks true|false
:param bool preserve_owner_group: preserve permissions true|false
"""
overlay_directory = self.description_dir + '/root/'
overlay_archive = self.description_dir + '/root.tar.gz'
if os.path.exists(overlay_directory):
log.info('Copying user defined files to image tree')
sync_options = [
'-r', '-p', '-t', '-D', '-H', '-X', '-A', '--one-file-system'
]
if follow_links:
sync_options.append('--copy-links')
else:
sync_options.append('--links')
if preserve_owner_group:
sync_options.append('-o')
sync_options.append('-g')
data = DataSync(overlay_directory, self.root_dir)
data.sync_data(options=sync_options)
elif os.path.exists(overlay_archive):
log.info(
'Extracting user defined files from archive to image tree')
archive = ArchiveTar(overlay_archive)
archive.extract(self.root_dir)
class TestDataSync:
def setup(self):
self.sync = DataSync('source_dir', 'target_dir')
@patch('kiwi.utils.sync.Command.run')
@patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes')
@patch('kiwi.logger.log.warning')
@patch('os.chmod')
@patch('os.stat')
def test_sync_data(self, mock_stat, mock_chmod, mock_warn,
mock_xattr_support, mock_command):
mock_stat.return_value = os.stat('.')
mock_xattr_support.return_value = False
self.sync.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=['exclude_me'])
mock_command.assert_called_once_with([
'rsync', '-a', '-H', '--one-file-system', '--exclude',
'/exclude_me', 'source_dir', 'target_dir'
])
mock_chmod.assert_called_once_with('target_dir',
mock_stat.return_value[ST_MODE])
assert mock_warn.called
@patch('xattr.getxattr')
def test_target_supports_extended_attributes(self, mock_getxattr):
assert self.sync.target_supports_extended_attributes() is True
mock_getxattr.assert_called_once_with('target_dir', 'user.mime_type')
@patch('xattr.getxattr')
def test_target_does_not_support_extended_attributes(self, mock_getxattr):
mock_getxattr.side_effect = OSError(
"""[Errno 95] Operation not supported: b'/boot/efi""")
assert self.sync.target_supports_extended_attributes() is False
def _sync_overlay_files(
self, overlay_directory, follow_links=False,
preserve_owner_group=False, profile=None
):
log.info(
'Copying user defined {0} to image tree'.format(
'files for profile: {0}'.format(profile) if profile else 'files'
)
)
sync_options = [
'-r', '-p', '-t', '-D', '-H', '-X', '-A', '--one-file-system'
]
if follow_links:
sync_options.append('--copy-links')
else:
sync_options.append('--links')
if preserve_owner_group:
sync_options.append('-o')
sync_options.append('-g')
data = DataSync(
overlay_directory, self.root_dir
)
data.sync_data(
options=sync_options
)
def _copy_loader_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
loader_data = lookup_path + '/image/loader/'
Path.create(self._get_iso_boot_path())
data = DataSync(loader_data, self._get_iso_boot_path())
data.sync_data(options=['-z', '-a'])
def sync_data(self, exclude=None):
"""
Copy root data tree into filesystem
:param list exclude: list of exclude dirs/files
"""
if not self.root_dir:
raise KiwiFileSystemSyncError(
'no root directory specified'
)
if not os.path.exists(self.root_dir):
raise KiwiFileSystemSyncError(
'given root directory %s does not exist' % self.root_dir
)
self.filesystem_mount = MountManager(
device=self.device_provider.get_device()
)
self.filesystem_mount.mount(
self.custom_args['mount_options']
)
data = DataSync(
self.root_dir, self.filesystem_mount.mountpoint
)
data.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=exclude
)
self.filesystem_mount.umount()
def create(self) -> None:
"""
Create new system root directory
The method creates a temporary directory and initializes it
for the purpose of building a system image from it. This
includes the following setup:
* create core system paths
* create static core device nodes
On success the contents of the temporary location are
synced to the specified root_dir and the temporary location
will be deleted. That way we never work on an incomplete
initial setup
:raises KiwiRootInitCreationError: if the init creation fails
at some point
"""
root = Temporary(prefix='kiwi_root.').new_dir()
Path.create(self.root_dir)
try:
self._create_base_directories(root.name)
self._create_base_links(root.name)
data = DataSync(root.name + '/', self.root_dir)
data.sync_data(options=['-a', '--ignore-existing'])
if Defaults.is_buildservice_worker():
copy(os.sep + Defaults.get_buildservice_env_name(),
self.root_dir)
except Exception as e:
self.delete()
raise KiwiRootInitCreationError('%s: %s' %
(type(e).__name__, format(e)))
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image', self.oci_layout_dir,
self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep]))
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image',
'{0}:base_layer'.format(self.oci_layout_dir), self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep])
)
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def create(self):
"""
Create new system root directory
The method creates a temporary directory and initializes it
for the purpose of building a system image from it. This
includes the following setup:
* create static core device nodes
* create core system paths
On success the contents of the temporary location are
synced to the specified root_dir and the temporary location
will be deleted. That way we never work on an incomplete
initial setup
"""
root = mkdtemp(prefix='kiwi_root.')
Path.create(self.root_dir)
try:
self._create_base_directories(root)
self._create_device_nodes(root)
self._create_base_links(root)
self._setup_config_templates(root)
data = DataSync(root + '/', self.root_dir)
data.sync_data(options=['-a', '--ignore-existing'])
except Exception as e:
self.delete()
raise KiwiRootInitCreationError('%s: %s' %
(type(e).__name__, format(e)))
finally:
rmtree(root, ignore_errors=True)
def create_initrd(self,
mbrid: Optional[SystemIdentifier] = None,
basename: Optional[str] = None,
install_initrd: bool = False) -> None:
"""
Create initrd from prepared boot system tree and compress the result
:param SystemIdentifier mbrid: instance of ImageIdentifier
:param str basename: base initrd file name
:param bool install_initrd: installation media initrd
"""
if self.is_prepared():
log.info('Creating initrd cpio archive')
# we can't simply exclude boot when building the archive
# because the file boot/mbrid must be preserved. Because of
# that we create a copy of the boot directory and remove
# everything in boot/ except for boot/mbrid. The original
# boot directory should not be changed because we rely
# on other data in boot/ e.g the kernel to be available
# for the entire image building process
if basename:
kiwi_initrd_basename = basename
else:
kiwi_initrd_basename = self.initrd_base_name
temp_boot_root = Temporary(prefix='kiwi_boot_root_copy.').new_dir()
temp_boot_root_directory = temp_boot_root.name
os.chmod(temp_boot_root_directory, 0o755)
data = DataSync(self.boot_root_directory + '/',
temp_boot_root_directory)
data.sync_data(options=['-a'])
boot_directory = temp_boot_root_directory + '/boot'
Path.wipe(boot_directory)
if mbrid:
log.info('--> Importing mbrid: %s', mbrid.get_id())
Path.create(boot_directory)
image_identifier = boot_directory + '/mbrid'
mbrid.write(image_identifier)
cpio = ArchiveCpio(
os.sep.join([self.target_dir, kiwi_initrd_basename]))
# the following is a list of directories which were needed
# during the process of creating an image but not when the
# image is actually booting with this initrd
exclude_from_archive = [
'/' + Defaults.get_shared_cache_location(), '/image',
'/usr/lib/grub*'
]
# the following is a list of directories to exclude which
# are not needed inside of the initrd
exclude_from_archive += [
'/usr/share/doc', '/usr/share/man', '/home', '/media', '/srv'
]
cpio.create(source_dir=temp_boot_root_directory,
exclude=exclude_from_archive)
log.info('--> xz compressing archive')
compress = Compress(
os.sep.join([self.target_dir, kiwi_initrd_basename]))
compress.xz(['--check=crc32', '--lzma2=dict=1MiB', '--threads=0'])
self.initrd_filename = compress.compressed_filename
def sync_data(self, exclude: List[str] = []):
"""
Copy root data tree into filesystem
:param list exclude: list of exclude dirs/files
"""
if not self.root_dir:
raise KiwiFileSystemSyncError(
'no root directory specified'
)
if not os.path.exists(self.root_dir):
raise KiwiFileSystemSyncError(
'given root directory %s does not exist' % self.root_dir
)
self.filesystem_mount = MountManager(
device=self.device_provider.get_device()
)
self.filesystem_mount.mount(
self.custom_args['mount_options']
)
self._apply_attributes()
data = DataSync(
self.root_dir, self.filesystem_mount.mountpoint
)
data.sync_data(
exclude=exclude, options=Defaults.get_sync_options()
)
class TestDataSync(object):
def setup(self):
self.sync = DataSync('source_dir', 'target_dir')
@patch('kiwi.utils.sync.Command.run')
@patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes')
@patch('kiwi.logger.log.warning')
def test_sync_data(self, mock_warn, mock_xattr_support, mock_command):
mock_xattr_support.return_value = False
self.sync.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=['exclude_me']
)
mock_command.assert_called_once_with(
[
'rsync', '-a', '-H', '--one-file-system',
'--exclude', '/exclude_me', 'source_dir', 'target_dir'
]
)
assert mock_warn.called
@patch('xattr.getxattr')
def test_target_supports_extended_attributes(self, mock_getxattr):
assert self.sync.target_supports_extended_attributes() is True
mock_getxattr.assert_called_once_with(
'target_dir', 'user.mime_type'
)
@patch('xattr.getxattr')
def test_target_supports_extended_attributes(self, mock_getxattr):
mock_getxattr.side_effect = OSError(
"""[Errno 95] Operation not supported: b'/boot/efi"""
)
assert self.sync.target_supports_extended_attributes() is False
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured'
)
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution
)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
dev_mount = MountManager(
device='/dev', mountpoint=self.root_dir + '/dev'
)
dev_mount.umount()
if self.repository.unauthenticated == 'false':
log.warning(
'KIWI does not support signature checks for apt-get '
'package manager during the bootstrap procedure, any '
'provided key will only be used inside the chroot '
'environment'
)
Command.run(
[
'debootstrap', '--no-check-gpg', self.distribution,
bootstrap_dir, self.distribution_path
], self.command_env
)
data = DataSync(
bootstrap_dir + '/', self.root_dir
)
data.sync_data(
options=['-a', '-H', '-X', '-A']
)
for key in self.repository.signing_keys:
Command.run([
'chroot', self.root_dir, 'apt-key', 'add', key
], self.command_env)
except Exception as e:
raise KiwiDebootstrapError(
'%s: %s' % (type(e).__name__, format(e))
)
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def _sync_data(cls, origin, destination, exclude_list=None, options=None):
"""
Synchronizes the origin and destination paths to be equivalent
:param string origin: the source path
:param string destination: the destination path
"""
sync = DataSync(origin, destination)
sync.sync_data(options=options, exclude=exclude_list)
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
:raises KiwiDebootstrapError: if no main distribution repository
is configured, if the debootstrap script is not found or if the
debootstrap script execution fails
:return: process results in command type
:rtype: namedtuple
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured')
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
dev_mount = MountManager(device='/dev',
mountpoint=self.root_dir + '/dev')
dev_mount.umount()
if self.repository.unauthenticated == 'false':
log.warning(
'KIWI does not support signature checks for apt-get '
'package manager during the bootstrap procedure, any '
'provided key will only be used inside the chroot '
'environment')
cmd = ['debootstrap', '--no-check-gpg']
if self.deboostrap_minbase:
cmd.append('--variant=minbase')
if self.repository.components:
cmd.append('--components={0}'.format(','.join(
self.repository.components)))
cmd.extend(
[self.distribution, bootstrap_dir, self.distribution_path])
Command.run(cmd, self.command_env)
data = DataSync(bootstrap_dir + '/', self.root_dir)
data.sync_data(options=['-a', '-H', '-X', '-A'])
for key in self.repository.signing_keys:
Command.run(['chroot', self.root_dir, 'apt-key', 'add', key],
self.command_env)
except Exception as e:
raise KiwiDebootstrapError('%s: %s' %
(type(e).__name__, format(e)))
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def _copy_modules_to_boot_directory_from(self, module_path):
boot_module_path = \
self._get_grub2_boot_path() + '/' + os.path.basename(module_path)
try:
data = DataSync(module_path + '/', boot_module_path)
data.sync_data(options=['-a'], exclude=['*.module'])
except Exception as e:
raise KiwiBootLoaderGrubModulesError(
'Module synchronisation failed with: %s' % format(e))
def setup_media_loader_directory(lookup_path: str, media_path: str,
boot_theme: str):
loader_data = lookup_path + '/image/loader/'
media_boot_path = os.sep.join(
[media_path, Defaults.get_iso_boot_path(), 'loader'])
Path.wipe(loader_data)
Path.create(loader_data)
grub_image_file_names = [
Defaults.get_isolinux_bios_grub_loader(), 'boot_hybrid.img'
]
loader_files = []
for grub_image_file_name in grub_image_file_names:
grub_file = Defaults.get_grub_path(
lookup_path,
'i386-pc/{0}'.format(grub_image_file_name),
raise_on_error=False)
if grub_file and os.path.exists(grub_file):
loader_files.append(grub_file)
for syslinux_file_name in Defaults.get_syslinux_modules():
for syslinux_dir in Defaults.get_syslinux_search_paths():
syslinux_file = os.path.normpath(
os.sep.join(
[lookup_path, syslinux_dir, syslinux_file_name]))
if os.path.exists(syslinux_file):
loader_files.append(syslinux_file)
log.debug('Copying loader files to {0}'.format(loader_data))
for loader_file in loader_files:
log.debug('--> Copying {0}'.format(loader_file))
shutil.copy(loader_file, loader_data)
bash_command = ' '.join(
['cp', lookup_path + '/boot/memtest*', loader_data + '/memtest'])
Command.run(command=['bash', '-c', bash_command], raise_on_error=False)
if boot_theme:
theme_path = ''.join(
[lookup_path, '/etc/bootsplash/themes/', boot_theme])
if os.path.exists(theme_path + '/cdrom/gfxboot.cfg'):
bash_command = ' '.join(
['cp', theme_path + '/cdrom/*', loader_data])
Command.run(['bash', '-c', bash_command])
# don't move down one menu entry the first time a F-key is used
Command.run([
'gfxboot', '--config-file', loader_data + '/gfxboot.cfg',
'--change-config', 'install::autodown=0'
])
if os.path.exists(theme_path + '/bootloader/message'):
Command.run(
['cp', theme_path + '/bootloader/message', loader_data])
Path.create(media_boot_path)
data = DataSync(loader_data, media_boot_path)
data.sync_data(options=['-a'])
def _setup_EFI_path(self, lookup_path):
"""
Copy efi boot data from lookup_path to the root directory
"""
if not lookup_path:
lookup_path = self.boot_dir
efi_path = lookup_path + '/boot/efi/'
if os.path.exists(efi_path):
efi_data = DataSync(efi_path, self.boot_dir)
efi_data.sync_data(options=['-a'])
def _setup_EFI_path(self, lookup_path):
"""
Copy efi boot data from lookup_path to the root directory
"""
if not lookup_path:
lookup_path = self.root_dir
efi_path = lookup_path + '/boot/efi/'
if os.path.exists(efi_path):
efi_data = DataSync(efi_path, self.root_dir)
efi_data.sync_data(options=['-a'])
def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')
container_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
container_name = ':'.join([container_dir, self.container_tag])
if base_image:
Path.create(container_dir)
image_tar = ArchiveTar(base_image)
image_tar.extract(container_dir)
Command.run([
'umoci', 'config', '--image',
'{0}:base_layer'.format(container_dir), '--tag',
self.container_tag
])
else:
Command.run(['umoci', 'init', '--layout', container_dir])
Command.run(['umoci', 'new', '--image', container_name])
Command.run(
['umoci', 'unpack', '--image', container_name, self.oci_root_dir])
oci_root = DataSync(''.join([self.root_dir, os.sep]),
os.sep.join([self.oci_root_dir, 'rootfs']))
oci_root.sync_data(options=['-a', '-H', '-X', '-A', '--delete'],
exclude=exclude_list)
Command.run(
['umoci', 'repack', '--image', container_name, self.oci_root_dir])
for tag in self.additional_tags:
Command.run(
['umoci', 'config', '--image', container_name, '--tag', tag])
Command.run(['umoci', 'config'] + self.maintainer + self.user +
self.workingdir + self.entry_command +
self.entry_subcommand + self.expose_ports + self.volumes +
self.environment + self.labels + [
'--image', container_name, '--created',
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00')
])
Command.run(['umoci', 'gc', '--layout', container_dir])
return self.pack_image_to_file(filename)
def export_modprobe_setup(self, target_root_dir):
"""
Export etc/modprobe.d to given root_dir
:param string target_root_dir: path name
"""
modprobe_config = self.root_dir + '/etc/modprobe.d'
if os.path.exists(modprobe_config):
log.info('Export modprobe configuration')
Path.create(target_root_dir + '/etc')
data = DataSync(modprobe_config, target_root_dir + '/etc/')
data.sync_data(options=['-z', '-a'])
def sync_data(self, exclude=None):
"""
Implements sync of root directory to mounted volumes
:param list exclude: file patterns to exclude
"""
if self.mountpoint:
root_mount = MountManager(device=None, mountpoint=self.mountpoint)
if not root_mount.is_mounted():
self.mount_volumes()
data = DataSync(self.root_dir, self.mountpoint)
data.sync_data(options=Defaults.get_sync_options(),
exclude=exclude)
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
:raises KiwiDebootstrapError: if no main distribution repository
is configured, if the debootstrap script is not found or if the
debootstrap script execution fails
:return: process results in command type
:rtype: namedtuple
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured')
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
cmd = ['debootstrap']
if self.repository.unauthenticated == 'false' and \
os.path.exists(self.repository.keyring):
cmd.append('--keyring={}'.format(self.repository.keyring))
else:
cmd.append('--no-check-gpg')
if self.deboostrap_minbase:
cmd.append('--variant=minbase')
if self.repository.components:
cmd.append('--components={0}'.format(','.join(
self.repository.components)))
cmd.extend(
[self.distribution, bootstrap_dir, self.distribution_path])
Command.run(cmd, self.command_env)
data = DataSync(bootstrap_dir + '/', self.root_dir)
data.sync_data(options=Defaults.get_sync_options(),
exclude=['proc', 'sys', 'dev'])
except Exception as e:
raise KiwiDebootstrapError('%s: %s' %
(type(e).__name__, format(e)))
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def _copy_modules_to_boot_directory_from(self, module_path):
boot_module_path = \
self._get_grub2_boot_path() + '/' + os.path.basename(module_path)
try:
data = DataSync(
module_path + '/', boot_module_path
)
data.sync_data(
options=['-z', '-a'], exclude=['*.module']
)
except Exception as e:
raise KiwiBootLoaderGrubModulesError(
'Module synchronisation failed with: %s' % format(e)
)
def sync_data(self, exclude=None):
"""
Implements sync of root directory to mounted volumes
:param list exclude: file patterns to exclude
"""
if self.mountpoint:
root_mount = MountManager(device=None, mountpoint=self.mountpoint)
if not root_mount.is_mounted():
self.mount_volumes()
data = DataSync(self.root_dir, self.mountpoint)
data.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=exclude)
def _copy_theme_data_to_boot_directory(self, lookup_path, target):
if not lookup_path:
lookup_path = self.boot_dir
font_name = 'unicode.pf2'
efi_font_dir = Defaults.get_grub_efi_font_directory(lookup_path)
boot_fonts_dir = os.path.normpath(
os.sep.join([
self.boot_dir,
self.get_boot_path(target), self.boot_directory_name, 'fonts'
]))
try:
unicode_font = Defaults.get_grub_path(lookup_path, font_name)
if not os.path.exists(os.sep.join([boot_fonts_dir, font_name])):
Path.create(boot_fonts_dir)
Command.run(['cp', unicode_font, boot_fonts_dir])
if efi_font_dir:
Command.run(['cp', unicode_font, efi_font_dir])
except Exception as issue:
raise KiwiBootLoaderGrubFontError(
'Setting up unicode font failed with {0}'.format(issue))
boot_theme_dir = os.sep.join(
[self.boot_dir, 'boot', self.boot_directory_name, 'themes'])
Path.create(boot_theme_dir)
if self.theme:
theme_dir = Defaults.get_grub_path(lookup_path,
'themes/' + self.theme,
raise_on_error=False)
boot_theme_background_file = self._find_theme_background_file(
lookup_path)
if theme_dir and os.path.exists(theme_dir):
if boot_theme_background_file:
# A background file was found. Preserve a copy of the
# file which was created at install time of the theme
# package by the activate-theme script
boot_theme_background_backup_file = os.sep.join(
[self.boot_dir, 'background.png'])
Command.run([
'cp', boot_theme_background_file,
boot_theme_background_backup_file
])
# sync theme data from install path to boot path
data = DataSync(theme_dir, boot_theme_dir)
data.sync_data(options=['-a'])
if boot_theme_background_file:
# Install preserved background file to the theme
Command.run([
'mv', boot_theme_background_backup_file,
os.sep.join([boot_theme_dir, self.theme])
])
elif boot_theme_background_file:
# assume all theme data is in the directory of the
# background file and just sync that directory to the
# boot path
data = DataSync(os.path.dirname(boot_theme_background_file),
boot_theme_dir)
data.sync_data(options=['-a'])
self._check_boot_theme_exists()
def create_initrd(self, mbrid=None):
"""
Create initrd from prepared boot system tree and compress the result
:param object mbrid: instance of ImageIdentifier
"""
if self.is_prepared():
log.info('Creating initrd cpio archive')
# we can't simply exclude boot when building the archive
# because the file boot/mbrid must be preserved. Because of
# that we create a copy of the boot directory and remove
# everything in boot/ except for boot/mbrid. The original
# boot directory should not be changed because we rely
# on other data in boot/ e.g the kernel to be available
# for the entire image building process
temp_boot_root_directory = mkdtemp(prefix='kiwi_boot_root_copy.')
self.temp_directories.append(temp_boot_root_directory)
data = DataSync(self.boot_root_directory + '/',
temp_boot_root_directory)
data.sync_data(options=['-z', '-a'])
boot_directory = temp_boot_root_directory + '/boot'
Path.wipe(boot_directory)
if mbrid:
log.info('--> Importing mbrid: %s', mbrid.get_id())
Path.create(boot_directory)
image_identifier = boot_directory + '/mbrid'
mbrid.write(image_identifier)
cpio = ArchiveCpio(
os.sep.join([self.target_dir, self.initrd_base_name]))
# the following is a list of directories which were needed
# during the process of creating an image but not when the
# image is actually booting with this initrd
exclude_from_archive = [
'/' + Defaults.get_shared_cache_location(), '/image',
'/usr/lib/grub*'
]
# the following is a list of directories to exclude which
# are not needed inside of the initrd
exclude_from_archive += [
'/usr/share/doc', '/usr/share/man', '/home', '/media', '/srv'
]
cpio.create(source_dir=temp_boot_root_directory,
exclude=exclude_from_archive)
log.info('--> xz compressing archive')
compress = Compress(
os.sep.join([self.target_dir, self.initrd_base_name]))
compress.xz()
self.initrd_filename = compress.compressed_filename
def _copy_loader_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
loader_data = lookup_path + '/image/loader/'
Path.wipe(loader_data)
Path.create(loader_data)
syslinux_file_names = [
'isolinux.bin', 'ldlinux.c32', 'libcom32.c32', 'libutil.c32',
'gfxboot.c32', 'gfxboot.com', 'menu.c32', 'chain.c32', 'mboot.c32'
]
syslinux_dirs = [
'/usr/share/syslinux/', '/usr/lib/syslinux/modules/bios/',
'/usr/lib/ISOLINUX/'
]
for syslinux_file_name in syslinux_file_names:
for syslinux_dir in syslinux_dirs:
syslinux_file = ''.join(
[lookup_path, syslinux_dir, syslinux_file_name])
if os.path.exists(syslinux_file):
shutil.copy(syslinux_file, loader_data)
bash_command = ' '.join(
['cp', lookup_path + '/boot/memtest*', loader_data + '/memtest'])
Command.run(command=['bash', '-c', bash_command], raise_on_error=False)
if self.get_boot_theme():
theme_path = ''.join([
lookup_path, '/etc/bootsplash/themes/',
self.get_boot_theme()
])
if os.path.exists(theme_path + '/cdrom/gfxboot.cfg'):
bash_command = ' '.join(
['cp', theme_path + '/cdrom/*', loader_data])
Command.run(['bash', '-c', bash_command])
# don't move down one menu entry the first time a F-key is used
Command.run([
'gfxboot', '--config-file', loader_data + '/gfxboot.cfg',
'--change-config', 'install::autodown=0'
])
if os.path.exists(theme_path + '/bootloader/message'):
Command.run(
['cp', theme_path + '/bootloader/message', loader_data])
Path.create(self._get_iso_boot_path())
data = DataSync(loader_data, self._get_iso_boot_path())
data.sync_data(options=['-z', '-a'])
def export_modprobe_setup(self, target_root_dir):
"""
Export etc/modprobe.d to given root_dir
:param string target_root_dir: path name
"""
modprobe_config = self.root_dir + '/etc/modprobe.d'
if os.path.exists(modprobe_config):
log.info('Export modprobe configuration')
Path.create(target_root_dir + '/etc')
data = DataSync(
modprobe_config, target_root_dir + '/etc/'
)
data.sync_data(
options=['-z', '-a']
)
class TestDataSync(object):
def setup(self):
self.sync = DataSync('source_dir', 'target_dir')
@patch('kiwi.utils.sync.Command.run')
def test_sync_data(self, mock_command):
self.sync.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=['exclude_me']
)
mock_command.assert_called_once_with(
[
'rsync', '-a', '-H', '-X', '-A', '--one-file-system',
'--exclude', '/exclude_me', 'source_dir', 'target_dir'
]
)
def sync_data(self, exclude=None):
"""
Implements sync of root directory to mounted volumes
:param list exclude: file patterns to exclude
"""
if self.mountpoint:
root_mount = MountManager(device=None, mountpoint=self.mountpoint)
if not root_mount.is_mounted():
self.mount_volumes()
data = DataSync(self.root_dir, self.mountpoint)
data.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=exclude
)
self.umount_volumes()
def setup_static_device_nodes(self):
"""
Container device node setup
Without subsystems like udev running in a container it is
required to provide a set of device nodes to let the
system in the container function correctly. This is
done by syncing the host system nodes to the container.
That this will also create device nodes which are not
necessarily present in the container later is a know
limitation of this method and considered harmless
"""
try:
data = DataSync('/dev/', self.root_dir + '/dev/')
data.sync_data(options=['-a', '-x', '--devices', '--specials'])
except Exception as e:
raise KiwiContainerSetupError(
'Failed to create static container nodes %s' % format(e))
def sync_data(self, exclude=None):
"""
Sync data into btrfs filesystem
If snapshots are activated the root filesystem is synced
into the first snapshot
:param list exclude: files to exclude from sync
"""
if self.toplevel_mount:
sync_target = self.mountpoint + '/@'
if self.custom_args['root_is_snapshot']:
sync_target = self.mountpoint + '/@/.snapshots/1/snapshot'
self._create_snapshot_info(''.join(
[self.mountpoint, '/@/.snapshots/1/info.xml']))
data = DataSync(self.root_dir, sync_target)
data.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=exclude)
def sync_data(self, exclude=None):
"""
Sync data into btrfs filesystem
If snapshots are activated the root filesystem is synced
into the first snapshot
"""
if self.toplevel_mount:
sync_target = self.mountpoint + '/@'
if self.custom_args['root_is_snapshot']:
sync_target = self.mountpoint + '/@/.snapshots/1/snapshot'
self._create_snapshot_info(
''.join([self.mountpoint, '/@/.snapshots/1/info.xml'])
)
data = DataSync(self.root_dir, sync_target)
data.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=exclude
)
def sync_data(self, exclude=None):
"""
Sync data into btrfs filesystem
If snapshots are activated the root filesystem is synced
into the first snapshot
:param list exclude: files to exclude from sync
"""
if self.toplevel_mount:
sync_target = self.get_mountpoint()
if self.custom_args['root_is_snapshot']:
self._create_snapshot_info(''.join(
[self.mountpoint, '/@/.snapshots/1/info.xml']))
data = DataSync(self.root_dir, sync_target)
data.sync_data(options=Defaults.get_sync_options(),
exclude=exclude)
if self.custom_args['quota_groups'] and \
self.custom_args['root_is_snapshot']:
self._create_snapper_quota_configuration()
def _copy_theme_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
boot_unicode_font = self.root_dir + '/boot/unicode.pf2'
if not os.path.exists(boot_unicode_font):
unicode_font = self._find_grub_data(lookup_path + '/usr/share') + \
'/unicode.pf2'
try:
Command.run(['cp', unicode_font, boot_unicode_font])
except Exception:
raise KiwiBootLoaderGrubFontError('Unicode font %s not found' %
unicode_font)
boot_theme_dir = os.sep.join(
[self.root_dir, 'boot', self.boot_directory_name, 'themes'])
Path.create(boot_theme_dir)
if self.theme:
theme_dir = self._find_grub_data(lookup_path + '/usr/share') + \
'/themes/' + self.theme
boot_theme_background_file = self._find_theme_background_file(
lookup_path)
if os.path.exists(theme_dir):
if boot_theme_background_file:
# A background file was found. Preserve a copy of the
# file which was created at install time of the theme
# package by the activate-theme script
boot_theme_background_backup_file = os.sep.join(
[self.root_dir, 'background.png'])
Command.run([
'cp', boot_theme_background_file,
boot_theme_background_backup_file
])
# sync theme data from install path to boot path
data = DataSync(theme_dir, boot_theme_dir)
data.sync_data(options=['-z', '-a'])
if boot_theme_background_file:
# Install preserved background file to the theme
Command.run([
'mv', boot_theme_background_backup_file,
os.sep.join([boot_theme_dir, self.theme])
])
elif boot_theme_background_file:
# assume all theme data is in the directory of the
# background file and just sync that directory to the
# boot path
data = DataSync(os.path.dirname(boot_theme_background_file),
boot_theme_dir)
data.sync_data(options=['-z', '-a'])
self._check_boot_theme_exists()
def setup_static_device_nodes(self):
"""
Container device node setup
Without subsystems like udev running in a container it is
required to provide a set of device nodes to let the
system in the container function correctly. This is
done by syncing the host system nodes to the container.
That this will also create device nodes which are not
necessarily present in the container later is a know
limitation of this method and considered harmless
"""
try:
data = DataSync('/dev/', self.root_dir + '/dev/')
data.sync_data(
options=['-z', '-a', '-x', '--devices', '--specials']
)
except Exception as e:
raise KiwiContainerSetupError(
'Failed to create static container nodes %s' % format(e)
)
def create(self):
"""
Create new system root directory
The method creates a temporary directory and initializes it
for the purpose of building a system image from it. This
includes the following setup:
* create static core device nodes
* create core system paths
On success the contents of the temporary location are
synced to the specified root_dir and the temporary location
will be deleted. That way we never work on an incomplete
initial setup
:raises KiwiRootInitCreationError: if the init creation fails
at some point
"""
root = mkdtemp(prefix='kiwi_root.')
Path.create(self.root_dir)
try:
self._create_base_directories(root)
self._create_base_links(root)
self._setup_config_templates(root)
data = DataSync(root + '/', self.root_dir)
data.sync_data(
options=['-a', '--ignore-existing']
)
if Defaults.is_buildservice_worker():
copy(
os.sep + Defaults.get_buildservice_env_name(),
self.root_dir)
except Exception as e:
self.delete()
raise KiwiRootInitCreationError(
'%s: %s' % (type(e).__name__, format(e))
)
finally:
rmtree(root, ignore_errors=True)
def _copy_theme_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
boot_unicode_font = self.root_dir + '/boot/unicode.pf2'
if not os.path.exists(boot_unicode_font):
unicode_font = self._find_grub_data(lookup_path + '/usr/share') + \
'/unicode.pf2'
try:
Command.run(['cp', unicode_font, boot_unicode_font])
except Exception:
raise KiwiBootLoaderGrubFontError('Unicode font %s not found' %
unicode_font)
boot_theme_dir = os.sep.join(
[self.root_dir, 'boot', self.boot_directory_name, 'themes'])
Path.create(boot_theme_dir)
if self.theme:
theme_dir = self._find_grub_data(lookup_path + '/usr/share') + \
'/themes/' + self.theme
# lookup theme background, it is expected that the
# installation of the theme package provided an appropriate
# background file
boot_theme_background_file = self._find_theme_background_file(
lookup_path)
if os.path.exists(theme_dir) and boot_theme_background_file:
# store a backup of the background file
boot_theme_background_backup_file = os.sep.join(
[self.root_dir, 'background.png'])
Command.run([
'cp', boot_theme_background_file,
boot_theme_background_backup_file
])
# sync theme data from install path to boot path
data = DataSync(theme_dir, boot_theme_dir)
data.sync_data(options=['-z', '-a'])
# restore background file
Command.run([
'mv', boot_theme_background_backup_file,
os.sep.join([boot_theme_dir, self.theme])
])
elif boot_theme_background_file:
# assume all theme data is in the directory of the
# background file and just sync that directory to the
# boot path
data = DataSync(os.path.dirname(boot_theme_background_file),
boot_theme_dir)
data.sync_data(options=['-z', '-a'])
self._check_boot_theme_exists()
def _copy_theme_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
boot_unicode_font = self.root_dir + '/boot/unicode.pf2'
if not os.path.exists(boot_unicode_font):
unicode_font = self._find_grub_data(lookup_path + '/usr/share') + \
'/unicode.pf2'
try:
Command.run(['cp', unicode_font, boot_unicode_font])
except Exception:
raise KiwiBootLoaderGrubFontError('Unicode font %s not found' %
unicode_font)
boot_theme_dir = os.sep.join(
[self.root_dir, 'boot', self.boot_directory_name, 'themes'])
Path.create(boot_theme_dir)
if self.theme and not os.listdir(boot_theme_dir):
theme_dir = self._find_grub_data(lookup_path + '/usr/share') + \
'/themes/' + self.theme
if os.path.exists(theme_dir):
data = DataSync(theme_dir, boot_theme_dir)
data.sync_data(options=['-z', '-a'])
self._check_boot_theme_exists()
def import_overlay_files(
self, follow_links=False, preserve_owner_group=False
):
"""
Copy overlay files from the image description to
the image root tree. Supported are a root/ directory
or a root.tar.gz tarball. The root/ directory takes
precedence over the tarball
:param bool follow_links: follow symlinks true|false
:param bool preserve_owner_group: preserve permissions true|false
"""
overlay_directory = self.description_dir + '/root/'
overlay_archive = self.description_dir + '/root.tar.gz'
if os.path.exists(overlay_directory):
log.info('Copying user defined files to image tree')
sync_options = [
'-r', '-p', '-t', '-D', '-H', '-X', '-A', '--one-file-system'
]
if follow_links:
sync_options.append('--copy-links')
else:
sync_options.append('--links')
if preserve_owner_group:
sync_options.append('-o')
sync_options.append('-g')
data = DataSync(
overlay_directory, self.root_dir
)
data.sync_data(
options=sync_options
)
elif os.path.exists(overlay_archive):
log.info('Extracting user defined files from archive to image tree')
archive = ArchiveTar(overlay_archive)
archive.extract(self.root_dir)
class TestDataSync:
@fixture(autouse=True)
def inject_fixtures(self, caplog):
self._caplog = caplog
def setup(self):
self.sync = DataSync('source_dir', 'target_dir')
def setup_method(self, cls):
self.setup()
@patch('kiwi.utils.sync.Command.run')
@patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes')
@patch('os.chmod')
@patch('os.stat')
def test_sync_data(self, mock_stat, mock_chmod, mock_xattr_support,
mock_command):
mock_stat.return_value = os.stat('.')
mock_xattr_support.return_value = False
with self._caplog.at_level(logging.WARNING):
self.sync.sync_data(options=[
'--archive', '--hard-links', '--xattrs', '--acls',
'--one-file-system'
],
exclude=['exclude_me'])
mock_command.assert_called_once_with([
'rsync', '--archive', '--hard-links', '--one-file-system',
'--exclude', '/exclude_me', 'source_dir', 'target_dir'
])
mock_chmod.assert_called_once_with('target_dir',
mock_stat.return_value[ST_MODE])
@patch('kiwi.utils.sync.Command.run')
@patch('os.chmod')
@patch('os.stat')
def test_sync_data_force_trailing_slash(self, mock_stat, mock_chmod,
mock_command):
mock_stat.return_value = os.stat('.')
self.sync.sync_data(force_trailing_slash=True)
mock_command.assert_called_once_with(
['rsync', 'source_dir/', 'target_dir'])
@patch('xattr.getxattr')
def test_target_supports_extended_attributes(self, mock_getxattr):
assert self.sync.target_supports_extended_attributes() is True
mock_getxattr.assert_called_once_with('target_dir', 'user.mime_type')
@patch('xattr.getxattr')
def test_target_does_not_support_extended_attributes(self, mock_getxattr):
mock_getxattr.side_effect = OSError(
"""[Errno 95] Operation not supported: b'/boot/efi""")
assert self.sync.target_supports_extended_attributes() is False
def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')
container_dir = os.sep.join(
[self.oci_dir, 'umoci_layout']
)
container_name = ':'.join(
[container_dir, self.container_tag]
)
if base_image:
Path.create(container_dir)
image_tar = ArchiveTar(base_image)
image_tar.extract(container_dir)
Command.run([
'umoci', 'config', '--image',
'{0}:base_layer'.format(container_dir),
'--tag', self.container_tag
])
else:
Command.run(
['umoci', 'init', '--layout', container_dir]
)
Command.run(
['umoci', 'new', '--image', container_name]
)
Command.run(
['umoci', 'unpack', '--image', container_name, self.oci_root_dir]
)
oci_root = DataSync(
''.join([self.root_dir, os.sep]),
os.sep.join([self.oci_root_dir, 'rootfs'])
)
oci_root.sync_data(
options=['-a', '-H', '-X', '-A', '--delete'], exclude=exclude_list
)
Command.run(
['umoci', 'repack', '--image', container_name, self.oci_root_dir]
)
for tag in self.additional_tags:
Command.run(
['umoci', 'config', '--image', container_name, '--tag', tag]
)
Command.run(
[
'umoci', 'config'
] +
self.maintainer +
self.user +
self.workingdir +
self.entry_command +
self.entry_subcommand +
self.expose_ports +
self.volumes +
self.environment +
self.labels +
[
'--image', container_name,
'--created', datetime.utcnow().strftime(
'%Y-%m-%dT%H:%M:%S+00:00'
)
]
)
Command.run(
['umoci', 'gc', '--layout', container_dir]
)
return self.pack_image_to_file(filename)
def setup(self):
self.sync = DataSync('source_dir', 'target_dir')
def _copy_loader_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
loader_data = lookup_path + '/image/loader/'
Path.wipe(loader_data)
Path.create(loader_data)
syslinux_file_names = [
'isolinux.bin',
'ldlinux.c32',
'libcom32.c32',
'libutil.c32',
'gfxboot.c32',
'gfxboot.com',
'menu.c32',
'chain.c32',
'mboot.c32'
]
syslinux_dirs = [
'/usr/share/syslinux/',
'/usr/lib/syslinux/modules/bios/'
]
for syslinux_file_name in syslinux_file_names:
for syslinux_dir in syslinux_dirs:
syslinux_file = ''.join(
[lookup_path, syslinux_dir, syslinux_file_name]
)
if os.path.exists(syslinux_file):
shutil.copy(syslinux_file, loader_data)
bash_command = ' '.join(
['cp', lookup_path + '/boot/memtest*', loader_data + '/memtest']
)
Command.run(
command=['bash', '-c', bash_command], raise_on_error=False
)
if self.get_boot_theme():
theme_path = ''.join(
[lookup_path, '/etc/bootsplash/themes/', self.get_boot_theme()]
)
if os.path.exists(theme_path + '/cdrom/gfxboot.cfg'):
bash_command = ' '.join(
['cp', theme_path + '/cdrom/*', loader_data]
)
Command.run(
['bash', '-c', bash_command]
)
# don't move down one menu entry the first time a F-key is used
Command.run(
[
'gfxboot',
'--config-file', loader_data + '/gfxboot.cfg',
'--change-config', 'install::autodown=0'
]
)
if os.path.exists(theme_path + '/bootloader/message'):
Command.run(
['cp', theme_path + '/bootloader/message', loader_data]
)
Path.create(self._get_iso_boot_path())
data = DataSync(
loader_data, self._get_iso_boot_path()
)
data.sync_data(
options=['-z', '-a']
)
def _copy_theme_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
boot_unicode_font = self.root_dir + '/boot/unicode.pf2'
if not os.path.exists(boot_unicode_font):
unicode_font = self._find_grub_data(lookup_path + '/usr/share') + \
'/unicode.pf2'
try:
Command.run(
['cp', unicode_font, boot_unicode_font]
)
except Exception:
raise KiwiBootLoaderGrubFontError(
'Unicode font %s not found' % unicode_font
)
boot_theme_dir = os.sep.join(
[self.root_dir, 'boot', self.boot_directory_name, 'themes']
)
Path.create(boot_theme_dir)
if self.theme:
theme_dir = self._find_grub_data(lookup_path + '/usr/share') + \
'/themes/' + self.theme
boot_theme_background_file = self._find_theme_background_file(
lookup_path
)
if os.path.exists(theme_dir):
if boot_theme_background_file:
# A background file was found. Preserve a copy of the
# file which was created at install time of the theme
# package by the activate-theme script
boot_theme_background_backup_file = os.sep.join(
[self.root_dir, 'background.png']
)
Command.run(
[
'cp', boot_theme_background_file,
boot_theme_background_backup_file
]
)
# sync theme data from install path to boot path
data = DataSync(
theme_dir, boot_theme_dir
)
data.sync_data(
options=['-z', '-a']
)
if boot_theme_background_file:
# Install preserved background file to the theme
Command.run(
[
'mv', boot_theme_background_backup_file,
os.sep.join([boot_theme_dir, self.theme])
]
)
elif boot_theme_background_file:
# assume all theme data is in the directory of the
# background file and just sync that directory to the
# boot path
data = DataSync(
os.path.dirname(boot_theme_background_file), boot_theme_dir
)
data.sync_data(
options=['-z', '-a']
)
self._check_boot_theme_exists()
