def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image',
'{0}:base_layer'.format(self.oci_layout_dir), self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep])
)
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def sync_data(self, exclude=None):
"""
Copy root data tree into filesystem
:param list exclude: list of exclude dirs/files
"""
if not self.root_dir:
raise KiwiFileSystemSyncError(
'no root directory specified'
)
if not os.path.exists(self.root_dir):
raise KiwiFileSystemSyncError(
'given root directory %s does not exist' % self.root_dir
)
self.filesystem_mount = MountManager(
device=self.device_provider.get_device()
)
self.filesystem_mount.mount(
self.custom_args['mount_options']
)
data = DataSync(
self.root_dir, self.filesystem_mount.mountpoint
)
data.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=exclude
)
self.filesystem_mount.umount()
class TestDataSync(object):
def setup(self):
self.sync = DataSync('source_dir', 'target_dir')
@patch('kiwi.utils.sync.Command.run')
@patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes')
@patch('kiwi.logger.log.warning')
def test_sync_data(self, mock_warn, mock_xattr_support, mock_command):
mock_xattr_support.return_value = False
self.sync.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=['exclude_me']
)
mock_command.assert_called_once_with(
[
'rsync', '-a', '-H', '--one-file-system',
'--exclude', '/exclude_me', 'source_dir', 'target_dir'
]
)
assert mock_warn.called
@patch('xattr.getxattr')
def test_target_supports_extended_attributes(self, mock_getxattr):
assert self.sync.target_supports_extended_attributes() is True
mock_getxattr.assert_called_once_with(
'target_dir', 'user.mime_type'
)
@patch('xattr.getxattr')
def test_target_supports_extended_attributes(self, mock_getxattr):
mock_getxattr.side_effect = OSError(
"""[Errno 95] Operation not supported: b'/boot/efi"""
)
assert self.sync.target_supports_extended_attributes() is False
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured'
)
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution
)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
dev_mount = MountManager(
device='/dev', mountpoint=self.root_dir + '/dev'
)
dev_mount.umount()
if self.repository.unauthenticated == 'false':
log.warning(
'KIWI does not support signature checks for apt-get '
'package manager during the bootstrap procedure, any '
'provided key will only be used inside the chroot '
'environment'
)
Command.run(
[
'debootstrap', '--no-check-gpg', self.distribution,
bootstrap_dir, self.distribution_path
], self.command_env
)
data = DataSync(
bootstrap_dir + '/', self.root_dir
)
data.sync_data(
options=['-a', '-H', '-X', '-A']
)
for key in self.repository.signing_keys:
Command.run([
'chroot', self.root_dir, 'apt-key', 'add', key
], self.command_env)
except Exception as e:
raise KiwiDebootstrapError(
'%s: %s' % (type(e).__name__, format(e))
)
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def _setup_EFI_path(self, lookup_path):
"""
Copy efi boot data from lookup_path to the root directory
"""
if not lookup_path:
lookup_path = self.root_dir
efi_path = lookup_path + '/boot/efi/'
if os.path.exists(efi_path):
efi_data = DataSync(efi_path, self.root_dir)
efi_data.sync_data(options=['-a'])
def _copy_modules_to_boot_directory_from(self, module_path):
boot_module_path = \
self._get_grub2_boot_path() + '/' + os.path.basename(module_path)
try:
data = DataSync(
module_path + '/', boot_module_path
)
data.sync_data(
options=['-z', '-a'], exclude=['*.module']
)
except Exception as e:
raise KiwiBootLoaderGrubModulesError(
'Module synchronisation failed with: %s' % format(e)
)
class TestDataSync(object):
def setup(self):
self.sync = DataSync('source_dir', 'target_dir')
@patch('kiwi.utils.sync.Command.run')
def test_sync_data(self, mock_command):
self.sync.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=['exclude_me']
)
mock_command.assert_called_once_with(
[
'rsync', '-a', '-H', '-X', '-A', '--one-file-system',
'--exclude', '/exclude_me', 'source_dir', 'target_dir'
]
)
def sync_data(self, exclude=None):
"""
Implements sync of root directory to mounted volumes
:param list exclude: file patterns to exclude
"""
if self.mountpoint:
root_mount = MountManager(device=None, mountpoint=self.mountpoint)
if not root_mount.is_mounted():
self.mount_volumes()
data = DataSync(self.root_dir, self.mountpoint)
data.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=exclude
)
self.umount_volumes()
def export_modprobe_setup(self, target_root_dir):
"""
Export etc/modprobe.d to given root_dir
:param string target_root_dir: path name
"""
modprobe_config = self.root_dir + '/etc/modprobe.d'
if os.path.exists(modprobe_config):
log.info('Export modprobe configuration')
Path.create(target_root_dir + '/etc')
data = DataSync(
modprobe_config, target_root_dir + '/etc/'
)
data.sync_data(
options=['-z', '-a']
)
def sync_data(self, exclude=None):
"""
Sync data into btrfs filesystem
If snapshots are activated the root filesystem is synced
into the first snapshot
"""
if self.toplevel_mount:
sync_target = self.mountpoint + '/@'
if self.custom_args['root_is_snapshot']:
sync_target = self.mountpoint + '/@/.snapshots/1/snapshot'
self._create_snapshot_info(
''.join([self.mountpoint, '/@/.snapshots/1/info.xml'])
)
data = DataSync(self.root_dir, sync_target)
data.sync_data(
options=['-a', '-H', '-X', '-A', '--one-file-system'],
exclude=exclude
)
def setup_static_device_nodes(self):
"""
Container device node setup
Without subsystems like udev running in a container it is
required to provide a set of device nodes to let the
system in the container function correctly. This is
done by syncing the host system nodes to the container.
That this will also create device nodes which are not
necessarily present in the container later is a know
limitation of this method and considered harmless
"""
try:
data = DataSync('/dev/', self.root_dir + '/dev/')
data.sync_data(
options=['-z', '-a', '-x', '--devices', '--specials']
)
except Exception as e:
raise KiwiContainerSetupError(
'Failed to create static container nodes %s' % format(e)
)
def create(self):
"""
Create new system root directory
The method creates a temporary directory and initializes it
for the purpose of building a system image from it. This
includes the following setup:
* create static core device nodes
* create core system paths
On success the contents of the temporary location are
synced to the specified root_dir and the temporary location
will be deleted. That way we never work on an incomplete
initial setup
:raises KiwiRootInitCreationError: if the init creation fails
at some point
"""
root = mkdtemp(prefix='kiwi_root.')
Path.create(self.root_dir)
try:
self._create_base_directories(root)
self._create_base_links(root)
self._setup_config_templates(root)
data = DataSync(root + '/', self.root_dir)
data.sync_data(
options=['-a', '--ignore-existing']
)
if Defaults.is_buildservice_worker():
copy(
os.sep + Defaults.get_buildservice_env_name(),
self.root_dir)
except Exception as e:
self.delete()
raise KiwiRootInitCreationError(
'%s: %s' % (type(e).__name__, format(e))
)
finally:
rmtree(root, ignore_errors=True)
def import_overlay_files(
self, follow_links=False, preserve_owner_group=False
):
"""
Copy overlay files from the image description to
the image root tree. Supported are a root/ directory
or a root.tar.gz tarball. The root/ directory takes
precedence over the tarball
:param bool follow_links: follow symlinks true|false
:param bool preserve_owner_group: preserve permissions true|false
"""
overlay_directory = self.description_dir + '/root/'
overlay_archive = self.description_dir + '/root.tar.gz'
if os.path.exists(overlay_directory):
log.info('Copying user defined files to image tree')
sync_options = [
'-r', '-p', '-t', '-D', '-H', '-X', '-A', '--one-file-system'
]
if follow_links:
sync_options.append('--copy-links')
else:
sync_options.append('--links')
if preserve_owner_group:
sync_options.append('-o')
sync_options.append('-g')
data = DataSync(
overlay_directory, self.root_dir
)
data.sync_data(
options=sync_options
)
elif os.path.exists(overlay_archive):
log.info('Extracting user defined files from archive to image tree')
archive = ArchiveTar(overlay_archive)
archive.extract(self.root_dir)
def _copy_theme_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
boot_unicode_font = self.root_dir + '/boot/unicode.pf2'
if not os.path.exists(boot_unicode_font):
unicode_font = self._find_grub_data(lookup_path + '/usr/share') + \
'/unicode.pf2'
try:
Command.run(
['cp', unicode_font, boot_unicode_font]
)
except Exception:
raise KiwiBootLoaderGrubFontError(
'Unicode font %s not found' % unicode_font
)
boot_theme_dir = os.sep.join(
[self.root_dir, 'boot', self.boot_directory_name, 'themes']
)
Path.create(boot_theme_dir)
if self.theme:
theme_dir = self._find_grub_data(lookup_path + '/usr/share') + \
'/themes/' + self.theme
boot_theme_background_file = self._find_theme_background_file(
lookup_path
)
if os.path.exists(theme_dir):
if boot_theme_background_file:
# A background file was found. Preserve a copy of the
# file which was created at install time of the theme
# package by the activate-theme script
boot_theme_background_backup_file = os.sep.join(
[self.root_dir, 'background.png']
)
Command.run(
[
'cp', boot_theme_background_file,
boot_theme_background_backup_file
]
)
# sync theme data from install path to boot path
data = DataSync(
theme_dir, boot_theme_dir
)
data.sync_data(
options=['-z', '-a']
)
if boot_theme_background_file:
# Install preserved background file to the theme
Command.run(
[
'mv', boot_theme_background_backup_file,
os.sep.join([boot_theme_dir, self.theme])
]
)
elif boot_theme_background_file:
# assume all theme data is in the directory of the
# background file and just sync that directory to the
# boot path
data = DataSync(
os.path.dirname(boot_theme_background_file), boot_theme_dir
)
data.sync_data(
options=['-z', '-a']
)
self._check_boot_theme_exists()
def _copy_loader_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
loader_data = lookup_path + '/image/loader/'
Path.wipe(loader_data)
Path.create(loader_data)
syslinux_file_names = [
'isolinux.bin',
'ldlinux.c32',
'libcom32.c32',
'libutil.c32',
'gfxboot.c32',
'gfxboot.com',
'menu.c32',
'chain.c32',
'mboot.c32'
]
syslinux_dirs = [
'/usr/share/syslinux/',
'/usr/lib/syslinux/modules/bios/'
]
for syslinux_file_name in syslinux_file_names:
for syslinux_dir in syslinux_dirs:
syslinux_file = ''.join(
[lookup_path, syslinux_dir, syslinux_file_name]
)
if os.path.exists(syslinux_file):
shutil.copy(syslinux_file, loader_data)
bash_command = ' '.join(
['cp', lookup_path + '/boot/memtest*', loader_data + '/memtest']
)
Command.run(
command=['bash', '-c', bash_command], raise_on_error=False
)
if self.get_boot_theme():
theme_path = ''.join(
[lookup_path, '/etc/bootsplash/themes/', self.get_boot_theme()]
)
if os.path.exists(theme_path + '/cdrom/gfxboot.cfg'):
bash_command = ' '.join(
['cp', theme_path + '/cdrom/*', loader_data]
)
Command.run(
['bash', '-c', bash_command]
)
# don't move down one menu entry the first time a F-key is used
Command.run(
[
'gfxboot',
'--config-file', loader_data + '/gfxboot.cfg',
'--change-config', 'install::autodown=0'
]
)
if os.path.exists(theme_path + '/bootloader/message'):
Command.run(
['cp', theme_path + '/bootloader/message', loader_data]
)
Path.create(self._get_iso_boot_path())
data = DataSync(
loader_data, self._get_iso_boot_path()
)
data.sync_data(
options=['-z', '-a']
)
def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')
container_dir = os.sep.join(
[self.oci_dir, 'umoci_layout']
)
container_name = ':'.join(
[container_dir, self.container_tag]
)
if base_image:
Path.create(container_dir)
image_tar = ArchiveTar(base_image)
image_tar.extract(container_dir)
Command.run([
'umoci', 'config', '--image',
'{0}:base_layer'.format(container_dir),
'--tag', self.container_tag
])
else:
Command.run(
['umoci', 'init', '--layout', container_dir]
)
Command.run(
['umoci', 'new', '--image', container_name]
)
Command.run(
['umoci', 'unpack', '--image', container_name, self.oci_root_dir]
)
oci_root = DataSync(
''.join([self.root_dir, os.sep]),
os.sep.join([self.oci_root_dir, 'rootfs'])
)
oci_root.sync_data(
options=['-a', '-H', '-X', '-A', '--delete'], exclude=exclude_list
)
Command.run(
['umoci', 'repack', '--image', container_name, self.oci_root_dir]
)
for tag in self.additional_tags:
Command.run(
['umoci', 'config', '--image', container_name, '--tag', tag]
)
Command.run(
[
'umoci', 'config'
] +
self.maintainer +
self.user +
self.workingdir +
self.entry_command +
self.entry_subcommand +
self.expose_ports +
self.volumes +
self.environment +
self.labels +
[
'--image', container_name,
'--created', datetime.utcnow().strftime(
'%Y-%m-%dT%H:%M:%S+00:00'
)
]
)
Command.run(
['umoci', 'gc', '--layout', container_dir]
)
return self.pack_image_to_file(filename)
def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = [
'image', '.profile', '.kconfig', 'boot', 'dev', 'sys', 'proc',
Defaults.get_shared_cache_location()
]
self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')
container_dir = os.sep.join(
[self.oci_dir, 'umoci_layout']
)
container_name = ':'.join(
[container_dir, self.container_tag]
)
if base_image:
Path.create(container_dir)
image_tar = ArchiveTar(base_image)
image_tar.extract(container_dir)
Command.run([
'umoci', 'config', '--image',
container_dir, '--tag', self.container_tag
])
else:
Command.run(
['umoci', 'init', '--layout', container_dir]
)
Command.run(
['umoci', 'new', '--image', container_name]
)
Command.run(
['umoci', 'unpack', '--image', container_name, self.oci_root_dir]
)
oci_root = DataSync(
''.join([self.root_dir, os.sep]),
os.sep.join([self.oci_root_dir, 'rootfs'])
)
oci_root.sync_data(
options=['-a', '-H', '-X', '-A', '--delete'], exclude=exclude_list
)
Command.run(
['umoci', 'repack', '--image', container_name, self.oci_root_dir]
)
Command.run(
[
'umoci', 'config'
] +
self.maintainer +
self.user +
self.workingdir +
self.entry_command +
self.entry_subcommand +
self.expose_ports +
self.volumes +
self.environment +
self.labels +
[
'--image', container_name,
'--created', datetime.utcnow().strftime(
'%Y-%m-%dT%H:%M:%S+00:00'
)
]
)
Command.run(
['umoci', 'gc', '--layout', container_dir]
)
self.pack_image_to_file(filename)
def _copy_theme_data_to_boot_directory(self, lookup_path, target):
if not lookup_path:
lookup_path = self.boot_dir
boot_fonts_dir = os.path.normpath(
os.sep.join(
[
self.boot_dir,
self.get_boot_path(target),
self.boot_directory_name,
'fonts'
]
)
)
Path.create(boot_fonts_dir)
boot_unicode_font = boot_fonts_dir + '/unicode.pf2'
if not os.path.exists(boot_unicode_font):
try:
unicode_font = Defaults.get_grub_path(
lookup_path, 'unicode.pf2'
)
Command.run(
['cp', unicode_font, boot_unicode_font]
)
except Exception as issue:
raise KiwiBootLoaderGrubFontError(
'Setting up unicode font failed with {0}'.format(issue)
)
boot_theme_dir = os.sep.join(
[self.boot_dir, 'boot', self.boot_directory_name, 'themes']
)
Path.create(boot_theme_dir)
if self.theme:
theme_dir = Defaults.get_grub_path(
lookup_path, 'themes/' + self.theme, raise_on_error=False
)
boot_theme_background_file = self._find_theme_background_file(
lookup_path
)
if theme_dir and os.path.exists(theme_dir):
if boot_theme_background_file:
# A background file was found. Preserve a copy of the
# file which was created at install time of the theme
# package by the activate-theme script
boot_theme_background_backup_file = os.sep.join(
[self.boot_dir, 'background.png']
)
Command.run(
[
'cp', boot_theme_background_file,
boot_theme_background_backup_file
]
)
# sync theme data from install path to boot path
data = DataSync(
theme_dir, boot_theme_dir
)
data.sync_data(
options=['-a']
)
if boot_theme_background_file:
# Install preserved background file to the theme
Command.run(
[
'mv', boot_theme_background_backup_file,
os.sep.join([boot_theme_dir, self.theme])
]
)
elif boot_theme_background_file:
# assume all theme data is in the directory of the
# background file and just sync that directory to the
# boot path
data = DataSync(
os.path.dirname(boot_theme_background_file), boot_theme_dir
)
data.sync_data(
options=['-a']
)
self._check_boot_theme_exists()
