def test_config_sections_from_home_base_config(self):
with patch.dict('os.environ', {'HOME': '../data/kiwi_config/ok'}):
runtime_config = RuntimeConfig(reread=True)
assert runtime_config.get_xz_options() == ['-a', '-b', 'xxx']
assert runtime_config.is_obs_public() is True
assert runtime_config.get_bundle_compression() is True
assert runtime_config.get_obs_download_server_url() == \
'http://example.com'
assert runtime_config.get_obs_api_server_url() == \
'https://api.example.com'
assert runtime_config.get_container_compression() is False
assert runtime_config.get_iso_tool_category() == 'xorriso'
assert runtime_config.get_oci_archive_tool() == 'umoci'
assert runtime_config.get_package_changes() is True
assert runtime_config.get_disabled_runtime_checks() == [
'check_dracut_module_for_oem_install_in_package_list',
'check_container_tool_chain_installed'
]
assert runtime_config.get_obs_api_credentials() == [
{'user_name': 'user_credentials'}
]
class DiskBuilder:
"""
**Disk image builder**
:param object xml_state: Instance of :class:`XMLState`
:param str target_dir: Target directory path name
:param str root_dir: Root directory path name
:param dict custom_args: Custom processing arguments defined as hash keys:
* signing_keys: list of package signing keys
* xz_options: string of XZ compression parameters
"""
def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
self.arch = platform.machine()
if self.arch == 'i686' or self.arch == 'i586':
self.arch = 'ix86'
self.root_dir = root_dir
self.target_dir = target_dir
self.xml_state = xml_state
self.spare_part_mbsize = xml_state.get_build_type_spare_part_size()
self.spare_part_fs = xml_state.build_type.get_spare_part_fs()
self.spare_part_is_last = xml_state.build_type.get_spare_part_is_last()
self.spare_part_mountpoint = \
xml_state.build_type.get_spare_part_mountpoint()
self.persistency_type = xml_state.build_type.get_devicepersistency()
self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot(
)
self.custom_root_mount_args = xml_state.get_fs_mount_option_list()
self.custom_root_creation_args = xml_state.get_fs_create_option_list()
self.build_type_name = xml_state.get_build_type_name()
self.image_format = xml_state.build_type.get_format()
self.install_iso = xml_state.build_type.get_installiso()
self.install_stick = xml_state.build_type.get_installstick()
self.install_pxe = xml_state.build_type.get_installpxe()
self.blocksize = xml_state.build_type.get_target_blocksize()
self.volume_manager_name = xml_state.get_volume_management()
self.volumes = xml_state.get_volumes()
self.volume_group_name = xml_state.get_volume_group_name()
self.mdraid = xml_state.build_type.get_mdraid()
self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr()
self.force_mbr = xml_state.build_type.get_force_mbr()
self.luks = xml_state.build_type.get_luks()
self.luks_os = xml_state.build_type.get_luksOS()
self.xen_server = xml_state.is_xen_server()
self.requested_filesystem = xml_state.build_type.get_filesystem()
self.requested_boot_filesystem = \
xml_state.build_type.get_bootfilesystem()
self.bootloader = xml_state.build_type.get_bootloader()
self.initrd_system = xml_state.get_initrd_system()
self.target_removable = xml_state.build_type.get_target_removable()
self.root_filesystem_is_multipath = \
xml_state.get_oemconfig_oem_multipath_scan()
self.disk_setup = DiskSetup(xml_state, root_dir)
self.unpartitioned_bytes = \
xml_state.get_build_type_unpartitioned_bytes()
self.custom_args = custom_args
self.signing_keys = None
if custom_args and 'signing_keys' in custom_args:
self.signing_keys = custom_args['signing_keys']
self.boot_image = BootImage(xml_state,
target_dir,
root_dir,
signing_keys=self.signing_keys)
self.firmware = FirmWare(xml_state)
self.system_setup = SystemSetup(xml_state=xml_state,
root_dir=self.root_dir)
self.diskname = ''.join([
target_dir, '/',
xml_state.xml_data.get_name(), '.' + self.arch,
'-' + xml_state.get_image_version(), '.raw'
])
self.boot_is_crypto = True if self.luks and not \
self.disk_setup.need_boot_partition() else False
self.install_media = self._install_image_requested()
self.generic_fstab_entries = []
# an instance of a class with the sync_data capability
# representing the entire image system except for the boot/ area
# which could live on another part of the disk
self.system = None
# an instance of a class with the sync_data capability
# representing the boot/ area of the disk if not part of
# self.system
self.system_boot = None
# an instance of a class with the sync_data capability
# representing the boot/efi area of the disk
self.system_efi = None
# an instance of a class with the sync_data capability
# representing the spare_part_mountpoint area of the disk
self.system_spare = None
# result store
self.result = Result(xml_state)
self.runtime_config = RuntimeConfig()
def create(self):
"""
Build a bootable disk image and optional installation image
The installation image is a bootable hybrid ISO image which
embeds the disk image and an image installer
Image types which triggers this builder are:
* image="oem"
* image="vmx"
:return: result
:rtype: instance of :class:`Result`
"""
disk = DiskBuilder(self.xml_state, self.target_dir, self.root_dir,
self.custom_args)
result = disk.create_disk()
# cleanup disk resources taken prior to next steps
del disk
disk_installer = DiskBuilder(self.xml_state, self.target_dir,
self.root_dir)
result = disk_installer.create_install_media(result)
disk_format = DiskBuilder(self.xml_state, self.target_dir,
self.root_dir)
disk_format.append_unpartitioned_space()
result = disk_format.create_disk_format(result)
return result
def create_disk(self): # noqa: C901
"""
Build a bootable raw disk image
:raises KiwiInstallMediaError:
if install media is required and image type is not oem
:raises KiwiVolumeManagerSetupError:
root overlay at the same time volumes are defined is not supported
:return: result
:rtype: instance of :class:`Result`
"""
if self.install_media and self.build_type_name != 'oem':
raise KiwiInstallMediaError(
'Install media requires oem type setup, got %s' %
self.build_type_name)
if self.root_filesystem_is_overlay and self.volume_manager_name:
raise KiwiVolumeManagerSetupError(
'Volume management together with root overlay is not supported'
)
# setup recovery archive, cleanup and create archive if requested
self.system_setup.create_recovery_archive()
# prepare boot(initrd) root system
log.info('Preparing boot system')
self.boot_image.prepare()
# precalculate needed disk size
disksize_mbytes = self.disk_setup.get_disksize_mbytes()
# create the disk
log.info('Creating raw disk image %s', self.diskname)
loop_provider = LoopDevice(self.diskname, disksize_mbytes,
self.blocksize)
loop_provider.create()
self.disk = Disk(self.firmware.get_partition_table_type(),
loop_provider, self.xml_state.get_disk_start_sector())
# create the bootloader instance
self.bootloader_config = BootLoaderConfig(
self.bootloader,
self.xml_state,
root_dir=self.root_dir,
boot_dir=self.root_dir,
custom_args={
'targetbase':
loop_provider.get_device(),
'grub_directory_name':
Defaults.get_grub_boot_directory_name(self.root_dir),
'boot_is_crypto':
self.boot_is_crypto
})
# create disk partitions and instance device map
device_map = self._build_and_map_disk_partitions(disksize_mbytes)
# create raid on current root device if requested
if self.mdraid:
self.raid_root = RaidDevice(device_map['root'])
self.raid_root.create_degraded_raid(raid_level=self.mdraid)
device_map['root'] = self.raid_root.get_device()
self.disk.public_partition_id_map['kiwi_RaidPart'] = \
self.disk.public_partition_id_map['kiwi_RootPart']
self.disk.public_partition_id_map['kiwi_RaidDev'] = \
device_map['root'].get_device()
# create luks on current root device if requested
if self.luks:
self.luks_root = LuksDevice(device_map['root'])
self.luks_boot_keyname = '/.root.keyfile'
self.luks_boot_keyfile = ''.join(
[self.root_dir, self.luks_boot_keyname])
self.luks_root.create_crypto_luks(passphrase=self.luks,
os=self.luks_os,
keyfile=self.luks_boot_keyfile
if self.boot_is_crypto else None)
if self.boot_is_crypto:
self.luks_boot_keyfile_setup = ''.join(
[self.root_dir, '/etc/dracut.conf.d/99-luks-boot.conf'])
self.boot_image.write_system_config_file(
config={'install_items': [self.luks_boot_keyname]},
config_file=self.luks_boot_keyfile_setup)
self.boot_image.include_file(
os.sep + os.path.basename(self.luks_boot_keyfile))
device_map['luks_root'] = device_map['root']
device_map['root'] = self.luks_root.get_device()
# create spare filesystem on spare partition if present
self._build_spare_filesystem(device_map)
# create filesystems on boot partition(s) if any
self._build_boot_filesystems(device_map)
# create volumes and filesystems for root system
if self.volume_manager_name:
volume_manager_custom_parameters = {
'fs_mount_options':
self.custom_root_mount_args,
'fs_create_options':
self.custom_root_creation_args,
'root_label':
self.disk_setup.get_root_label(),
'root_is_snapshot':
self.xml_state.build_type.get_btrfs_root_is_snapshot(),
'root_is_readonly_snapshot':
self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot(
),
'quota_groups':
self.xml_state.build_type.get_btrfs_quota_groups(),
'image_type':
self.xml_state.get_build_type_name()
}
volume_manager = VolumeManager(self.volume_manager_name,
device_map['root'],
self.root_dir + '/', self.volumes,
volume_manager_custom_parameters)
volume_manager.setup(self.volume_group_name)
volume_manager.create_volumes(self.requested_filesystem)
volume_manager.mount_volumes()
self.generic_fstab_entries += volume_manager.get_fstab(
self.persistency_type, self.requested_filesystem)
self.system = volume_manager
device_map['root'] = volume_manager.get_device()['root']
else:
log.info('Creating root(%s) filesystem on %s',
self.requested_filesystem,
device_map['root'].get_device())
filesystem_custom_parameters = {
'mount_options': self.custom_root_mount_args,
'create_options': self.custom_root_creation_args
}
filesystem = FileSystem(self.requested_filesystem,
device_map['root'], self.root_dir + '/',
filesystem_custom_parameters)
filesystem.create_on_device(label=self.disk_setup.get_root_label())
self.system = filesystem
# create a random image identifier
self.mbrid = SystemIdentifier()
self.mbrid.calculate_id()
# create first stage metadata to boot image
self._write_partition_id_config_to_boot_image()
self._write_recovery_metadata_to_boot_image()
self._write_raid_config_to_boot_image()
self._write_generic_fstab_to_boot_image(device_map)
self.system_setup.export_modprobe_setup(
self.boot_image.boot_root_directory)
# create first stage metadata to system image
self._write_image_identifier_to_system_image()
self._write_crypttab_to_system_image()
self._write_generic_fstab_to_system_image(device_map)
if self.initrd_system == 'dracut':
if self.root_filesystem_is_multipath is False:
self.boot_image.omit_module('multipath')
if self.root_filesystem_is_overlay:
self.boot_image.include_module('kiwi-overlay')
self.boot_image.write_system_config_file(
config={'modules': ['kiwi-overlay']})
if self.build_type_name == 'oem':
self.boot_image.include_module('kiwi-repart')
# create initrd cpio archive
self.boot_image.create_initrd(self.mbrid)
# create second stage metadata to system image
self._copy_first_boot_files_to_system_image()
self._write_bootloader_meta_data_to_system_image(device_map)
self.mbrid.write_to_disk(self.disk.storage_provider)
# set SELinux file security contexts if context exists
self._setup_selinux_file_contexts()
# syncing system data to disk image
log.info('Syncing system to image')
if self.system_spare:
self.system_spare.sync_data()
if self.system_efi:
log.info('--> Syncing EFI boot data to EFI partition')
self.system_efi.sync_data()
if self.system_boot:
log.info('--> Syncing boot data at extra partition')
self.system_boot.sync_data(
self._get_exclude_list_for_boot_data_sync())
log.info('--> Syncing root filesystem data')
if self.root_filesystem_is_overlay:
squashed_root_file = NamedTemporaryFile()
squashed_root = FileSystemSquashFs(device_provider=None,
root_dir=self.root_dir)
squashed_root.create_on_file(
filename=squashed_root_file.name,
exclude=self._get_exclude_list_for_root_data_sync(device_map))
Command.run([
'dd',
'if=%s' % squashed_root_file.name,
'of=%s' % device_map['readonly'].get_device()
])
else:
self.system.sync_data(
self._get_exclude_list_for_root_data_sync(device_map))
# install boot loader
self._install_bootloader(device_map)
# set root filesystem properties
self._setup_property_root_is_readonly_snapshot()
# prepare for install media if requested
if self.install_media:
log.info('Saving boot image instance to file')
self.boot_image.dump(self.target_dir + '/boot_image.pickledump')
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(), self.diskname)
# store image file name in result
self.result.add(
key='disk_image',
filename=self.diskname,
use_for_bundle=True if not self.image_format else False,
compress=self.runtime_config.get_bundle_compression(default=True),
shasum=True)
# create image root metadata
self.result.add(key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
def create_disk_format(self, result_instance):
"""
Create a bootable disk format from a previously
created raw disk image
:param object result_instance: instance of :class:`Result`
:return: updated result_instance
:rtype: instance of :class:`Result`
"""
if self.image_format:
log.info('Creating %s Disk Format', self.image_format)
disk_format = DiskFormat(self.image_format, self.xml_state,
self.root_dir, self.target_dir)
disk_format.create_image_format()
disk_format.store_to_result(result_instance)
return result_instance
def append_unpartitioned_space(self):
"""
Extends the raw disk if an unpartitioned area is specified
"""
if self.unpartitioned_bytes:
log.info('Expanding disk with %d bytes of unpartitioned space',
self.unpartitioned_bytes)
disk_format = DiskFormat('raw', self.xml_state, self.root_dir,
self.target_dir)
disk_format.resize_raw_disk(self.unpartitioned_bytes, append=True)
firmware = FirmWare(self.xml_state)
loop_provider = LoopDevice(disk_format.diskname)
loop_provider.create(overwrite=False)
partitioner = Partitioner(firmware.get_partition_table_type(),
loop_provider)
partitioner.resize_table()
def create_install_media(self, result_instance):
"""
Build an installation image. The installation image is a
bootable hybrid ISO image which embeds the raw disk image
and an image installer
:param object result_instance: instance of :class:`Result`
:return: updated result_instance with installation media
:rtype: instance of :class:`Result`
"""
if self.install_media:
install_image = InstallImageBuilder(
self.xml_state, self.root_dir, self.target_dir,
self._load_boot_image_instance(), self.custom_args)
if self.install_iso or self.install_stick:
log.info('Creating hybrid ISO installation image')
install_image.create_install_iso()
result_instance.add(key='installation_image',
filename=install_image.isoname,
use_for_bundle=True,
compress=False,
shasum=True)
if self.install_pxe:
log.info('Creating PXE installation archive')
install_image.create_install_pxe_archive()
result_instance.add(key='installation_pxe_archive',
filename=install_image.pxename,
use_for_bundle=True,
compress=False,
shasum=True)
return result_instance
def _load_boot_image_instance(self):
boot_image_dump_file = self.target_dir + '/boot_image.pickledump'
if not os.path.exists(boot_image_dump_file):
raise KiwiInstallMediaError(
'No boot image instance dump %s found' % boot_image_dump_file)
try:
with open(boot_image_dump_file, 'rb') as boot_image_dump:
boot_image = pickle.load(boot_image_dump)
boot_image.enable_cleanup()
Path.wipe(boot_image_dump_file)
except Exception as e:
raise KiwiInstallMediaError('Failed to load boot image dump: %s' %
type(e).__name__)
return boot_image
def _setup_selinux_file_contexts(self):
security_context = '/etc/selinux/targeted/contexts/files/file_contexts'
if os.path.exists(self.root_dir + security_context):
self.system_setup.set_selinux_file_contexts(security_context)
def _install_image_requested(self):
if self.install_iso or self.install_stick or self.install_pxe:
return True
def _get_exclude_list_for_root_data_sync(self, device_map):
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
if 'spare' in device_map and self.spare_part_mountpoint:
exclude_list.append('{0}/*'.format(
self.spare_part_mountpoint.lstrip(os.sep)))
exclude_list.append('{0}/.*'.format(
self.spare_part_mountpoint.lstrip(os.sep)))
if 'boot' in device_map and self.bootloader == 'grub2_s390x_emu':
exclude_list.append('boot/zipl/*')
exclude_list.append('boot/zipl/.*')
elif 'boot' in device_map:
exclude_list.append('boot/*')
exclude_list.append('boot/.*')
if 'efi' in device_map:
exclude_list.append('boot/efi/*')
exclude_list.append('boot/efi/.*')
return exclude_list
def _get_exclude_list_for_boot_data_sync(self):
return ['efi/*']
def _build_spare_filesystem(self, device_map):
if 'spare' in device_map and self.spare_part_fs:
spare_part_data_path = None
if self.spare_part_mountpoint:
spare_part_data_path = self.root_dir + '{0}/'.format(
self.spare_part_mountpoint)
filesystem = FileSystem(self.spare_part_fs, device_map['spare'],
spare_part_data_path)
filesystem.create_on_device(label='SPARE')
self.system_spare = filesystem
def _build_boot_filesystems(self, device_map):
if 'efi' in device_map:
log.info('Creating EFI(fat16) filesystem on %s',
device_map['efi'].get_device())
filesystem = FileSystem('fat16', device_map['efi'],
self.root_dir + '/boot/efi/')
filesystem.create_on_device(label=self.disk_setup.get_efi_label())
self.system_efi = filesystem
if 'boot' in device_map:
boot_filesystem = self.requested_boot_filesystem
if not boot_filesystem:
boot_filesystem = self.requested_filesystem
boot_directory = self.root_dir + '/boot/'
if self.bootloader == 'grub2_s390x_emu':
boot_directory = self.root_dir + '/boot/zipl/'
boot_filesystem = 'ext2'
log.info('Creating boot(%s) filesystem on %s', boot_filesystem,
device_map['boot'].get_device())
filesystem = FileSystem(boot_filesystem, device_map['boot'],
boot_directory)
filesystem.create_on_device(label=self.disk_setup.get_boot_label())
self.system_boot = filesystem
def _build_and_map_disk_partitions(self, disksize_mbytes): # noqa: C901
self.disk.wipe()
disksize_used_mbytes = 0
if self.firmware.legacy_bios_mode():
log.info('--> creating EFI CSM(legacy bios) partition')
partition_mbsize = self.firmware.get_legacy_bios_partition_size()
self.disk.create_efi_csm_partition(partition_mbsize)
disksize_used_mbytes += partition_mbsize
if self.firmware.efi_mode():
log.info('--> creating EFI partition')
partition_mbsize = self.firmware.get_efi_partition_size()
self.disk.create_efi_partition(partition_mbsize)
disksize_used_mbytes += partition_mbsize
if self.firmware.ofw_mode():
log.info('--> creating PReP partition')
partition_mbsize = self.firmware.get_prep_partition_size()
self.disk.create_prep_partition(partition_mbsize)
disksize_used_mbytes += partition_mbsize
if self.disk_setup.need_boot_partition():
log.info('--> creating boot partition')
partition_mbsize = self.disk_setup.boot_partition_size()
self.disk.create_boot_partition(partition_mbsize)
disksize_used_mbytes += partition_mbsize
if self.spare_part_mbsize and not self.spare_part_is_last:
log.info('--> creating spare partition')
self.disk.create_spare_partition(self.spare_part_mbsize)
if self.root_filesystem_is_overlay:
log.info('--> creating readonly root partition')
squashed_root_file = NamedTemporaryFile()
squashed_root = FileSystemSquashFs(device_provider=None,
root_dir=self.root_dir)
squashed_root.create_on_file(
filename=squashed_root_file.name,
exclude=[Defaults.get_shared_cache_location()])
squashed_rootfs_mbsize = int(
os.path.getsize(squashed_root_file.name) /
1048576) + Defaults.get_min_partition_mbytes()
self.disk.create_root_readonly_partition(squashed_rootfs_mbsize)
disksize_used_mbytes += squashed_rootfs_mbsize
if self.spare_part_mbsize and self.spare_part_is_last:
rootfs_mbsize = disksize_mbytes - disksize_used_mbytes - \
self.spare_part_mbsize - Defaults.get_min_partition_mbytes()
else:
rootfs_mbsize = 'all_free'
if self.volume_manager_name and self.volume_manager_name == 'lvm':
log.info('--> creating LVM root partition')
self.disk.create_root_lvm_partition(rootfs_mbsize)
elif self.mdraid:
log.info('--> creating mdraid root partition')
self.disk.create_root_raid_partition(rootfs_mbsize)
else:
log.info('--> creating root partition')
self.disk.create_root_partition(rootfs_mbsize)
if self.spare_part_mbsize and self.spare_part_is_last:
log.info('--> creating spare partition')
self.disk.create_spare_partition('all_free')
if self.firmware.bios_mode():
log.info('--> setting active flag to primary boot partition')
self.disk.activate_boot_partition()
if self.firmware.ofw_mode():
log.info('--> setting active flag to primary PReP partition')
self.disk.activate_boot_partition()
if self.firmware.efi_mode():
if self.force_mbr:
log.info('--> converting partition table to MBR')
self.disk.create_mbr()
elif self.hybrid_mbr:
log.info('--> converting partition table to hybrid GPT/MBR')
self.disk.create_hybrid_mbr()
self.disk.map_partitions()
return self.disk.get_device()
def _write_partition_id_config_to_boot_image(self):
log.info('Creating config.partids in boot system')
filename = ''.join(
[self.boot_image.boot_root_directory, '/config.partids'])
partition_id_map = self.disk.get_public_partition_id_map()
with open(filename, 'w') as partids:
for id_name, id_value in list(partition_id_map.items()):
partids.write('{0}="{1}"{2}'.format(id_name, id_value,
os.linesep))
self.boot_image.include_file(os.sep + os.path.basename(filename))
def _write_raid_config_to_boot_image(self):
if self.mdraid:
log.info('Creating etc/mdadm.conf in boot system')
filename = ''.join(
[self.boot_image.boot_root_directory, '/etc/mdadm.conf'])
self.raid_root.create_raid_config(filename)
self.boot_image.include_file(
os.sep + os.sep.join(['etc', os.path.basename(filename)]))
def _write_crypttab_to_system_image(self):
if self.luks:
log.info('Creating etc/crypttab')
filename = ''.join([self.root_dir, '/etc/crypttab'])
self.luks_root.create_crypttab(filename)
self.boot_image.include_file(
os.sep + os.sep.join(['etc', os.path.basename(filename)]))
def _write_generic_fstab_to_system_image(self, device_map):
log.info('Creating generic system etc/fstab')
self._write_generic_fstab(device_map, self.system_setup)
def _write_generic_fstab_to_boot_image(self, device_map):
if self.initrd_system == 'kiwi':
log.info('Creating generic boot image etc/fstab')
self._write_generic_fstab(device_map, self.boot_image.setup)
def _write_generic_fstab(self, device_map, setup):
root_is_snapshot = \
self.xml_state.build_type.get_btrfs_root_is_snapshot()
root_is_readonly_snapshot = \
self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()
fs_check_interval = '1 1'
custom_root_mount_args = list(self.custom_root_mount_args)
if root_is_snapshot and root_is_readonly_snapshot:
custom_root_mount_args += ['ro']
fs_check_interval = '0 0'
self._add_generic_fstab_entry(device_map['root'].get_device(), '/',
custom_root_mount_args,
fs_check_interval)
if 'spare' in device_map and \
self.spare_part_fs and self.spare_part_mountpoint:
self._add_generic_fstab_entry(device_map['spare'].get_device(),
self.spare_part_mountpoint)
if 'boot' in device_map:
if self.bootloader == 'grub2_s390x_emu':
boot_mount_point = '/boot/zipl'
else:
boot_mount_point = '/boot'
self._add_generic_fstab_entry(device_map['boot'].get_device(),
boot_mount_point)
if 'efi' in device_map:
self._add_generic_fstab_entry(device_map['efi'].get_device(),
'/boot/efi')
setup.create_fstab(self.generic_fstab_entries)
def _add_generic_fstab_entry(self,
device,
mount_point,
options=None,
check='0 0'):
if not options:
options = ['defaults']
block_operation = BlockID(device)
blkid_type = 'LABEL' if self.persistency_type == 'by-label' else 'UUID'
device_id = block_operation.get_blkid(blkid_type)
fstab_entry = ' '.join([
blkid_type + '=' + device_id, mount_point,
block_operation.get_filesystem(), ','.join(options), check
])
if fstab_entry not in self.generic_fstab_entries:
self.generic_fstab_entries.append(fstab_entry)
def _write_image_identifier_to_system_image(self):
log.info('Creating image identifier: %s', self.mbrid.get_id())
self.mbrid.write(self.root_dir + '/boot/mbrid')
def _write_recovery_metadata_to_boot_image(self):
if os.path.exists(self.root_dir + '/recovery.partition.size'):
log.info('Copying recovery metadata to boot image')
recovery_metadata = ''.join(
[self.root_dir, '/recovery.partition.size'])
Command.run(
['cp', recovery_metadata, self.boot_image.boot_root_directory])
self.boot_image.include_file(os.sep +
os.path.basename(recovery_metadata))
def _write_bootloader_meta_data_to_system_image(self, device_map):
if self.bootloader != 'custom':
log.info('Creating %s bootloader configuration', self.bootloader)
boot_options = []
if self.mdraid:
boot_options.append('rd.auto')
root_device = device_map['root']
boot_device = root_device
if 'boot' in device_map:
boot_device = device_map['boot']
root_uuid = self.disk.get_uuid(device_map['root'].get_device())
boot_uuid = self.disk.get_uuid(boot_device.get_device())
boot_uuid_unmapped = self.disk.get_uuid(
device_map['luks_root'].get_device(
)) if self.luks else boot_uuid
self.bootloader_config.setup_disk_boot_images(boot_uuid_unmapped)
self.bootloader_config.write_meta_data(
root_uuid=root_uuid, boot_options=' '.join(boot_options))
log.info('Creating config.bootoptions')
filename = ''.join(
[self.boot_image.boot_root_directory, '/config.bootoptions'])
kexec_boot_options = ' '.join(
[self.bootloader_config.get_boot_cmdline(root_uuid)] +
boot_options)
with open(filename, 'w') as boot_options:
boot_options.write('{0}{1}'.format(kexec_boot_options,
os.linesep))
partition_id_map = self.disk.get_public_partition_id_map()
boot_partition_id = partition_id_map['kiwi_RootPart']
if 'kiwi_BootPart' in partition_id_map:
boot_partition_id = partition_id_map['kiwi_BootPart']
self.system_setup.call_edit_boot_config_script(
self.requested_filesystem, boot_partition_id)
def _install_bootloader(self, device_map):
root_device = device_map['root']
boot_device = root_device
if 'boot' in device_map:
boot_device = device_map['boot']
if 'readonly' in device_map:
root_device = device_map['readonly']
custom_install_arguments = {
'boot_device': boot_device.get_device(),
'root_device': root_device.get_device(),
'firmware': self.firmware,
'target_removable': self.target_removable
}
if 'efi' in device_map:
efi_device = device_map['efi']
custom_install_arguments.update(
{'efi_device': efi_device.get_device()})
if 'prep' in device_map:
prep_device = device_map['prep']
custom_install_arguments.update(
{'prep_device': prep_device.get_device()})
if self.volume_manager_name:
self.system.umount_volumes()
custom_install_arguments.update(
{'system_volumes': self.system.get_volumes()})
# create bootloader config prior bootloader installation
self.bootloader_config.setup_disk_image_config(
boot_options=custom_install_arguments)
# cleanup bootloader config resources taken prior to next steps
del self.bootloader_config
if self.bootloader != 'custom':
log.debug("custom arguments for bootloader installation %s",
custom_install_arguments)
bootloader = BootLoaderInstall(self.bootloader, self.root_dir,
self.disk.storage_provider,
custom_install_arguments)
if bootloader.install_required():
bootloader.install()
self.system_setup.call_edit_boot_install_script(
self.diskname, boot_device.get_device())
def _setup_property_root_is_readonly_snapshot(self):
if self.volume_manager_name:
root_is_snapshot = \
self.xml_state.build_type.get_btrfs_root_is_snapshot()
root_is_readonly_snapshot = \
self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()
if root_is_snapshot and root_is_readonly_snapshot:
log.info('Setting root filesystem into read-only mode')
self.system.mount_volumes()
self.system.set_property_readonly_root()
self.system.umount_volumes()
def _copy_first_boot_files_to_system_image(self):
boot_names = self.boot_image.get_boot_names()
if self.initrd_system == 'kiwi':
log.info('Copy boot files to system image')
kernel = Kernel(self.boot_image.boot_root_directory)
log.info('--> boot image kernel as %s', boot_names.kernel_name)
kernel.copy_kernel(self.root_dir,
''.join(['/boot/', boot_names.kernel_name]))
if self.xen_server:
if kernel.get_xen_hypervisor():
log.info('--> boot image Xen hypervisor as xen.gz')
kernel.copy_xen_hypervisor(self.root_dir, '/boot/xen.gz')
else:
raise KiwiDiskBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image.boot_root_directory)
log.info('--> initrd archive as %s', boot_names.initrd_name)
Command.run([
'mv', self.boot_image.initrd_filename,
self.root_dir + ''.join(['/boot/', boot_names.initrd_name])
])
class FileSystemBuilder:
"""
**Filesystem image builder**
:param str label: filesystem label
:param str root_uuid: UUID of the created filesystem (on block device only)
:param str root_dir: root directory path name
:param str target_dir: target directory path name
:param str requested_image_type: configured image type
:param str requested_filesystem: requested filesystem name
:param obejct system_setup: instance of :class:`SystemSetup`
:param str filename: file name of the filesystem image
:param int blocksize: configured disk blocksize
:param object filesystem_setup: instance of :class:`FileSystemSetup`
:param object filesystems_no_device_node: List of filesystems which are
created from a data tree and do not require a block device e.g loop
:param dict filesystem_custom_parameters: Configured custom filesystem
mount and creation arguments
:param object result: instance of :class:`Result`
"""
def __init__(self, xml_state, target_dir, root_dir):
self.label = None
self.root_uuid = None
self.root_dir = root_dir
self.target_dir = target_dir
self.requested_image_type = xml_state.get_build_type_name()
if self.requested_image_type == 'pxe':
self.requested_filesystem = xml_state.build_type.get_filesystem()
else:
self.requested_filesystem = self.requested_image_type
if not self.requested_filesystem:
raise KiwiFileSystemSetupError(
'No filesystem configured in %s type' %
self.requested_image_type)
self.filesystem_custom_parameters = {
'mount_options': xml_state.get_fs_mount_option_list(),
'create_options': xml_state.get_fs_create_option_list()
}
self.system_setup = SystemSetup(xml_state=xml_state,
root_dir=self.root_dir)
self.filename = ''.join([
target_dir, '/',
xml_state.xml_data.get_name(), '.' + platform.machine(),
'-' + xml_state.get_image_version(), '.', self.requested_filesystem
])
self.blocksize = xml_state.build_type.get_target_blocksize()
self.filesystem_setup = FileSystemSetup(xml_state, root_dir)
self.filesystems_no_device_node = ['squashfs']
self.result = Result(xml_state)
self.runtime_config = RuntimeConfig()
def create(self):
"""
Build a mountable filesystem image
Image types which triggers this builder are:
* image="ext2"
* image="ext3"
* image="ext4"
* image="btrfs"
* image="xfs"
:return: result
:rtype: instance of :class:`Result`
"""
log.info('Creating %s filesystem', self.requested_filesystem)
supported_filesystems = Defaults.get_filesystem_image_types()
if self.requested_filesystem not in supported_filesystems:
raise KiwiFileSystemSetupError('Unknown filesystem: %s' %
self.requested_filesystem)
if self.requested_filesystem not in self.filesystems_no_device_node:
self._operate_on_loop()
else:
self._operate_on_file()
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(), self.filename)
self.result.add(
key='filesystem_image',
filename=self.filename,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(default=True),
shasum=True)
self.result.add(key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
def _operate_on_loop(self):
filesystem = None
loop_provider = LoopDevice(self.filename,
self.filesystem_setup.get_size_mbytes(),
self.blocksize)
loop_provider.create()
filesystem = FileSystem(self.requested_filesystem, loop_provider,
self.root_dir + os.sep,
self.filesystem_custom_parameters)
filesystem.create_on_device(self.label)
self.root_uuid = loop_provider.get_uuid(loop_provider.get_device())
log.info('--> Syncing data to filesystem on %s',
loop_provider.get_device())
filesystem.sync_data(Defaults.get_exclude_list_for_root_data_sync())
def _operate_on_file(self):
default_provider = DeviceProvider()
filesystem = FileSystem(self.requested_filesystem, default_provider,
self.root_dir,
self.filesystem_custom_parameters)
filesystem.create_on_file(
self.filename, self.label,
Defaults.get_exclude_list_for_root_data_sync())
class TestRuntimeConfig:
def setup(self):
with patch.dict('os.environ', {'HOME': '../data'}):
self.runtime_config = RuntimeConfig()
# pretend that none of the runtime config files exist, even if they do
# (e.g. the system wide config file in /etc/kiwi.yml)
# => this will give us the defaults
with patch('os.path.exists', return_value=False):
self.default_runtime_config = RuntimeConfig()
@patch('os.path.exists')
@patch('yaml.safe_load')
def test_reading_system_wide_config_file(self, mock_yaml, mock_exists):
exists_call_results = [True, False]
def os_path_exists(config):
return exists_call_results.pop()
mock_exists.side_effect = os_path_exists
with patch('builtins.open') as m_open:
self.runtime_config = RuntimeConfig()
m_open.assert_called_once_with('/etc/kiwi.yml', 'r')
def test_invalid_yaml_format(self):
self.runtime_config.config_data = {'xz': None}
with raises(KiwiRuntimeConfigFormatError):
self.runtime_config.get_xz_options()
def test_get_xz_options(self):
assert self.runtime_config.get_xz_options() == ['-a', '-b', 'xxx']
def test_is_obs_public(self):
assert self.runtime_config.is_obs_public() is True
def test_get_bundle_compression(self):
assert self.runtime_config.get_bundle_compression() is True
def test_get_bundle_compression_default(self):
assert self.default_runtime_config.get_bundle_compression(
default=True) is True
assert self.default_runtime_config.get_bundle_compression(
default=False) is False
def test_is_obs_public_default(self):
assert self.default_runtime_config.is_obs_public() is True
def test_get_obs_download_server_url(self):
assert self.runtime_config.get_obs_download_server_url() == \
'http://example.com'
def test_get_obs_download_server_url_default(self):
assert self.default_runtime_config.get_obs_download_server_url() == \
Defaults.get_obs_download_server_url()
def test_get_container_compression(self):
assert self.runtime_config.get_container_compression() is None
def test_get_container_compression_default(self):
assert self.default_runtime_config.get_container_compression() == 'xz'
@patch.object(RuntimeConfig, '_get_attribute')
@patch('kiwi.logger.log.warning')
def test_get_container_compression_invalid(self, mock_warning,
mock_get_attribute):
mock_get_attribute.return_value = 'foo'
assert self.runtime_config.get_container_compression() == 'xz'
mock_warning.assert_called_once_with(
'Skipping invalid container compression: foo')
@patch.object(RuntimeConfig, '_get_attribute')
def test_get_container_compression_xz(self, mock_get_attribute):
mock_get_attribute.return_value = 'xz'
assert self.runtime_config.get_container_compression() == 'xz'
def test_get_iso_tool_category(self):
assert self.runtime_config.get_iso_tool_category() == 'cdrtools'
def test_get_iso_tool_category_default(self):
assert self.default_runtime_config.get_iso_tool_category() == 'xorriso'
@patch.object(RuntimeConfig, '_get_attribute')
@patch('kiwi.logger.log.warning')
def test_get_iso_tool_category_invalid(self, mock_warning,
mock_get_attribute):
mock_get_attribute.return_value = 'foo'
assert self.runtime_config.get_iso_tool_category() == 'xorriso'
mock_warning.assert_called_once_with(
'Skipping invalid iso tool category: foo')
def test_get_oci_archive_tool(self):
assert self.runtime_config.get_oci_archive_tool() == 'umoci'
def test_get_oci_archive_tool_default(self):
assert self.default_runtime_config.get_oci_archive_tool() == 'umoci'
def test_get_disabled_runtime_checks(self):
assert self.runtime_config.get_disabled_runtime_checks() == [
'check_dracut_module_for_oem_install_in_package_list',
'check_container_tool_chain_installed'
]
class FileSystemBuilder:
"""
**Filesystem image builder**
:param obsject xml_state: Instance of :class:`XMLState`
:param str target_dir: target directory path name
:param str root_dir: root directory path name
:param dict custom_args: Custom processing arguments defined as hash keys:
* None
"""
def __init__(
self, xml_state: XMLState, target_dir: str,
root_dir: str, custom_args: Dict = None
):
self.label = None
self.root_uuid = ''
self.root_dir = root_dir
self.target_dir = target_dir
self.bundle_format = xml_state.get_build_type_bundle_format()
self.requested_image_type = xml_state.get_build_type_name()
if self.requested_image_type in Defaults.get_kis_image_types():
self.requested_filesystem = xml_state.build_type.get_filesystem()
else:
self.requested_filesystem = self.requested_image_type
if not self.requested_filesystem:
raise KiwiFileSystemSetupError(
'No filesystem configured in %s type' %
self.requested_image_type
)
self.filesystem_custom_parameters = {
'mount_options': xml_state.get_fs_mount_option_list(),
'create_options': xml_state.get_fs_create_option_list()
}
if self.requested_filesystem == 'squashfs':
self.filesystem_custom_parameters['compression'] = \
xml_state.build_type.get_squashfscompression()
self.system_setup = SystemSetup(
xml_state=xml_state, root_dir=self.root_dir
)
self.filename = ''.join(
[
target_dir, '/',
xml_state.xml_data.get_name(),
'.' + Defaults.get_platform_name(),
'-' + xml_state.get_image_version(),
'.', self.requested_filesystem
]
)
self.blocksize = xml_state.build_type.get_target_blocksize()
self.filesystem_setup = FileSystemSetup(xml_state, root_dir)
self.filesystems_no_device_node = [
'squashfs'
]
self.result = Result(xml_state)
self.runtime_config = RuntimeConfig()
def create(self) -> Result:
"""
Build a mountable filesystem image
Image types which triggers this builder are:
* image="ext2"
* image="ext3"
* image="ext4"
* image="btrfs"
* image="xfs"
:return: result
:rtype: instance of :class:`Result`
"""
log.info(
'Creating %s filesystem', self.requested_filesystem
)
supported_filesystems = Defaults.get_filesystem_image_types()
if self.requested_filesystem not in supported_filesystems:
raise KiwiFileSystemSetupError(
'Unknown filesystem: %s' % self.requested_filesystem
)
if self.requested_filesystem not in self.filesystems_no_device_node:
self._operate_on_loop()
else:
self._operate_on_file()
Result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.filename
)
if self.bundle_format:
self.result.add_bundle_format(self.bundle_format)
self.result.add(
key='filesystem_image',
filename=self.filename,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(
default=True
),
shasum=True
)
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_changes',
filename=self.system_setup.export_package_changes(
self.target_dir
),
use_for_bundle=True,
compress=True,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result
def _operate_on_loop(self) -> None:
filesystem = None
loop_provider = LoopDevice(
self.filename,
self.filesystem_setup.get_size_mbytes(),
self.blocksize
)
loop_provider.create()
filesystem = FileSystem.new(
self.requested_filesystem, loop_provider,
self.root_dir + os.sep, self.filesystem_custom_parameters
)
filesystem.create_on_device(self.label)
self.root_uuid = loop_provider.get_uuid(loop_provider.get_device())
log.info(
f'--> Syncing data to filesystem on {loop_provider.get_device()}'
)
filesystem.sync_data(
Defaults.
get_exclude_list_for_root_data_sync() + Defaults.
get_exclude_list_from_custom_exclude_files(self.root_dir)
)
def _operate_on_file(self) -> None:
default_provider = DeviceProvider()
filesystem = FileSystem.new(
self.requested_filesystem, default_provider,
self.root_dir, self.filesystem_custom_parameters
)
filesystem.create_on_file(
self.filename, self.label,
Defaults.get_exclude_list_for_root_data_sync()
)
class PxeBuilder(object):
"""
**Filesystem based PXE image builder.**
:param object xml_state: instance of :class:`XMLState`
:param str target_dir: target directory path name
:param str root_dir: system image root directory
:param dict custom_args: Custom processing arguments defined as hash keys:
* signing_keys: list of package signing keys
* xz_options: string of XZ compression parameters
"""
def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
self.target_dir = target_dir
self.compressed = xml_state.build_type.get_compressed()
self.xen_server = xml_state.is_xen_server()
self.filesystem = FileSystemBuilder(xml_state, target_dir,
root_dir + '/')
self.system_setup = SystemSetup(xml_state=xml_state, root_dir=root_dir)
self.boot_signing_keys = custom_args['signing_keys'] if custom_args \
and 'signing_keys' in custom_args else None
self.xz_options = custom_args['xz_options'] if custom_args \
and 'xz_options' in custom_args else None
self.boot_image_task = BootImage(xml_state,
target_dir,
signing_keys=self.boot_signing_keys)
self.image_name = ''.join([
target_dir, '/',
xml_state.xml_data.get_name(), '.' + platform.machine(),
'-' + xml_state.get_image_version()
])
self.archive_name = ''.join([self.image_name, '.tar.xz'])
self.checksum_name = ''.join([self.image_name, '.md5'])
self.kernel_filename = None
self.hypervisor_filename = None
self.result = Result(xml_state)
self.runtime_config = RuntimeConfig()
def create(self):
"""
Build a pxe image set consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum. The result can be used within the kiwi
PXE boot infrastructure
Image types which triggers this builder are:
* image="pxe"
:raises KiwiPxeBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
log.info('Creating PXE root filesystem image')
self.filesystem.create()
os.rename(self.filesystem.filename, self.image_name)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
checksum = Checksum(self.image)
checksum.md5(self.checksum_name)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join([
os.path.basename(self.image_name), '-', kernel_data.version,
'.kernel'
])
kernel.copy_kernel(self.target_dir, self.kernel_filename)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[os.path.basename(self.image_name), '-', kernel_data.name])
kernel.copy_xen_hypervisor(self.target_dir,
self.hypervisor_filename)
self.result.add(key='xen_hypervisor',
filename=self.target_dir + '/' +
self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
bash_command = ['tar', '-C', self.target_dir, '-c', '--to-stdout'] + [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.image),
os.path.basename(self.checksum_name)
] + ['|', 'xz', '-f'] + self.xz_options + ['>', self.archive_name]
Command.run(['bash', '-c', ' '.join(bash_command)])
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(), self.archive_name)
# store results
self.result.add(
key='pxe_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(default=False),
shasum=True)
# create image root metadata
self.result.add(key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
class DiskFormatBase:
"""
**Base class to create disk formats from a raw disk image**
:param object xml_state: Instance of XMLState
:param string root_dir: root directory path name
:param string arch: platform.machine
:param string target_dir: target directory path name
:param dict custom_args: custom format options dictionary
"""
def __init__(self, xml_state, root_dir, target_dir, custom_args=None):
self.xml_state = xml_state
self.root_dir = root_dir
self.arch = platform.machine()
self.target_dir = target_dir
self.custom_args = {}
self.temp_image_dir = None
self.image_format = None
self.diskname = self.get_target_file_path_for_format('raw')
self.runtime_config = RuntimeConfig()
self.post_init(custom_args)
def post_init(self, custom_args):
"""
Post initialization method
Implementation in specialized disk format class if required
:param list custom_args: unused
"""
pass
def has_raw_disk(self):
"""
Check if the base raw disk image exists
:return: True or False
:rtype: bool
"""
return os.path.exists(self.diskname)
def resize_raw_disk(self, size_bytes, append=False):
"""
Resize raw disk image to specified size. If the request
would actually shrink the disk an exception is raised.
If the disk got changed the method returns True, if
the new size is the same as the current size nothing
gets resized and the method returns False
:param int size: size in bytes
:return: True or False
:rtype: bool
"""
if not append:
current_byte_size = os.path.getsize(self.diskname)
size_bytes = int(size_bytes)
if size_bytes < current_byte_size:
raise KiwiResizeRawDiskError(
'shrinking {0} disk to {1} bytes corrupts the image'.
format(self.diskname, size_bytes))
elif size_bytes == current_byte_size:
return False
Command.run([
'qemu-img', 'resize', self.diskname,
'+{0}'.format(size_bytes) if append else format(size_bytes)
])
return True
def create_image_format(self):
"""
Create disk format
Implementation in specialized disk format class required
"""
raise NotImplementedError
def get_qemu_option_list(self, custom_args):
"""
Create list of qemu options from custom_args dict
:param dict custom_args: arguments
:return: qemu option list
:rtype: list
"""
options = []
if custom_args:
ordered_args = OrderedDict(sorted(custom_args.items()))
for key, value in list(ordered_args.items()):
if key == 'adapter_type=pvscsi':
# qemu does not support the pvscsi type:
# To build a vmdk image with ddb.adapterType set to
# pvscsi we need to manually change the header but
# have to create a vmdk with lsilogic as scsi adapter
# first. I don't really like this hack but it seems
# there is no way around. For details see
# bsc#1099569
key = 'adapter_type=lsilogic'
options.append('-o')
if value:
options.append('{0}={1}'.format(key, value))
else:
options.append(key)
return options
def get_target_file_path_for_format(self, format_name):
"""
Create target file path name for specified format
:param string format_name: disk format name
:return: file path name
:rtype: str
"""
if format_name != 'raw':
if format_name not in Defaults.get_disk_format_types():
raise KiwiFormatSetupError('unsupported disk format %s' %
format_name)
return ''.join([
self.target_dir, '/',
self.xml_state.xml_data.get_name(), '.' + self.arch,
'-' + self.xml_state.get_image_version(), '.' + format_name
])
def store_to_result(self, result):
"""
Store result file of the format conversion into the
provided result instance.
By default only the converted image file will be stored
as compressed file. Subformats which creates additional
metadata files or want to use other result flags needs
to overwrite this method
:param object result: Instance of Result
"""
result.add(
key='disk_format_image',
filename=self.get_target_file_path_for_format(self.image_format),
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(default=True),
shasum=True)
def __del__(self):
if self.temp_image_dir and os.path.exists(self.temp_image_dir):
log.info('Cleaning up %s instance', type(self).__name__)
Path.wipe(self.temp_image_dir)
class KisBuilder:
"""
**Filesystem based image builder.**
:param object xml_state: instance of :class:`XMLState`
:param str target_dir: target directory path name
:param str root_dir: system image root directory
:param dict custom_args: Custom processing arguments defined as hash keys:
* signing_keys: list of package signing keys
* xz_options: string of XZ compression parameters
"""
def __init__(
self, xml_state: XMLState, target_dir: str,
root_dir: str, custom_args: Dict = None
):
self.target_dir = target_dir
self.compressed = xml_state.build_type.get_compressed()
self.xen_server = xml_state.is_xen_server()
self.custom_cmdline = xml_state.build_type.get_kernelcmdline()
self.filesystem = FileSystemBuilder(
xml_state, target_dir, root_dir + '/'
) if xml_state.build_type.get_filesystem() else None
self.system_setup = SystemSetup(
xml_state=xml_state, root_dir=root_dir
)
self.initrd_system = xml_state.get_initrd_system()
self.boot_signing_keys = custom_args['signing_keys'] if custom_args \
and 'signing_keys' in custom_args else None
self.xz_options = custom_args['xz_options'] if custom_args \
and 'xz_options' in custom_args else None
self.boot_image_task = BootImage.new(
xml_state, target_dir, root_dir,
signing_keys=self.boot_signing_keys
)
self.image_name = ''.join(
[
target_dir, '/',
xml_state.xml_data.get_name(),
'.' + Defaults.get_platform_name(),
'-' + xml_state.get_image_version()
]
)
self.image: str = ''
self.append_file = ''.join([self.image_name, '.append'])
self.archive_name = ''.join([self.image_name, '.tar'])
self.checksum_name = ''.join([self.image_name, '.md5'])
self.kernel_filename: str = ''
self.hypervisor_filename: str = ''
self.result = Result(xml_state)
self.runtime_config = RuntimeConfig()
def create(self) -> Result:
"""
Build a component image consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum.
Image types which triggers this builder are:
* image="kis"
* image="pxe"
:raises KiwiKisBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
if self.filesystem:
log.info('Creating root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating root filesystem MD5 checksum')
checksum = Checksum(self.image)
checksum.md5(self.checksum_name)
# prepare boot(initrd) root system
log.info('Creating boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiKisBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
hypervisor_data = kernel.get_xen_hypervisor()
if hypervisor_data:
self.hypervisor_filename = ''.join(
[
os.path.basename(self.image_name),
'-', hypervisor_data.name
]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiKisBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd
self.boot_image_task.create_initrd()
# create append information
# this information helps to configure the deployment infrastructure
if self.filesystem and self.filesystem.root_uuid \
and self.initrd_system == 'dracut':
cmdline = 'root=UUID={}'.format(self.filesystem.root_uuid)
if self.custom_cmdline:
cmdline += ' {}'.format(self.custom_cmdline)
with open(self.append_file, 'w') as append:
append.write(cmdline)
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
kis_tarball_files = [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.checksum_name),
]
if self.image:
kis_tarball_files.append(os.path.basename(self.image))
if self.filesystem and self.filesystem.root_uuid \
and self.initrd_system == 'dracut':
kis_tarball_files.append(os.path.basename(self.append_file))
kis_tarball = ArchiveTar(
self.archive_name,
create_from_file_list=True,
file_list=kis_tarball_files
)
if self.compressed:
self.archive_name = kis_tarball.create(self.target_dir)
else:
self.archive_name = kis_tarball.create_xz_compressed(
self.target_dir, xz_options=self.xz_options
)
Result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.archive_name
)
# store results
self.result.add(
key='kis_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(
default=False
),
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_changes',
filename=self.system_setup.export_package_changes(
self.target_dir
),
use_for_bundle=True,
compress=True,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result
def test_get_bundle_compression_default(self):
with patch.dict('os.environ', {'HOME': './'}):
runtime_config = RuntimeConfig()
assert runtime_config.get_bundle_compression(default=True) is True
assert runtime_config.get_bundle_compression(
default=False) is False
class DiskBuilder:
"""
**Disk image builder**
:param object xml_state: Instance of :class:`XMLState`
:param str target_dir: Target directory path name
:param str root_dir: Root directory path name
:param dict custom_args: Custom processing arguments defined as hash keys:
* signing_keys: list of package signing keys
* xz_options: string of XZ compression parameters
"""
def __init__(self,
xml_state: XMLState,
target_dir: str,
root_dir: str,
custom_args: Dict = None):
self.arch = Defaults.get_platform_name()
self.root_dir = root_dir
self.target_dir = target_dir
self.xml_state = xml_state
self.spare_part_mbsize = xml_state.get_build_type_spare_part_size()
self.spare_part_fs = xml_state.build_type.get_spare_part_fs()
self.spare_part_is_last = xml_state.build_type.get_spare_part_is_last()
self.spare_part_mountpoint = \
xml_state.build_type.get_spare_part_mountpoint()
self.persistency_type = xml_state.build_type.get_devicepersistency()
self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot(
)
self.custom_root_mount_args = xml_state.get_fs_mount_option_list()
self.custom_root_creation_args = xml_state.get_fs_create_option_list()
self.build_type_name = xml_state.get_build_type_name()
self.image_format = xml_state.build_type.get_format()
self.install_iso = xml_state.build_type.get_installiso()
self.install_stick = xml_state.build_type.get_installstick()
self.install_pxe = xml_state.build_type.get_installpxe()
self.blocksize = xml_state.build_type.get_target_blocksize()
self.volume_manager_name = xml_state.get_volume_management()
self.volumes = xml_state.get_volumes()
self.custom_partitions = xml_state.get_partitions()
self.volume_group_name = xml_state.get_volume_group_name()
self.mdraid = xml_state.build_type.get_mdraid()
self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr()
self.force_mbr = xml_state.build_type.get_force_mbr()
self.luks = xml_state.build_type.get_luks()
self.luks_os = xml_state.build_type.get_luksOS()
self.xen_server = xml_state.is_xen_server()
self.requested_filesystem = xml_state.build_type.get_filesystem()
self.requested_boot_filesystem = \
xml_state.build_type.get_bootfilesystem()
self.bootloader = xml_state.get_build_type_bootloader_name()
self.initrd_system = xml_state.get_initrd_system()
self.target_removable = xml_state.build_type.get_target_removable()
self.root_filesystem_is_multipath = \
xml_state.get_oemconfig_oem_multipath_scan()
self.disk_resize_requested = \
xml_state.get_oemconfig_oem_resize()
self.swap_mbytes = xml_state.get_oemconfig_swap_mbytes()
self.disk_setup = DiskSetup(xml_state, root_dir)
self.unpartitioned_bytes = \
xml_state.get_build_type_unpartitioned_bytes()
self.custom_args = custom_args
self.signing_keys = None
if custom_args and 'signing_keys' in custom_args:
self.signing_keys = custom_args['signing_keys']
self.boot_image = BootImage.new(xml_state,
target_dir,
root_dir,
signing_keys=self.signing_keys)
self.firmware = FirmWare(xml_state)
self.system_setup = SystemSetup(xml_state=xml_state,
root_dir=self.root_dir)
self.bundle_format = xml_state.get_build_type_bundle_format()
self.diskname = ''.join([
target_dir, '/',
xml_state.xml_data.get_name(), '.' + self.arch,
'-' + xml_state.get_image_version(), '.raw'
])
self.boot_is_crypto = True if self.luks and not \
self.disk_setup.need_boot_partition() else False
self.install_media = self._install_image_requested()
self.fstab = Fstab()
# result store
self.result = Result(xml_state)
self.runtime_config = RuntimeConfig()
if not self.boot_image.has_initrd_support():
log.warning('Building without initrd support !')
def create(self) -> Result:
"""
Build a bootable disk image and optional installation image
The installation image is a bootable hybrid ISO image which
embeds the disk image and an image installer
Image types which triggers this builder are:
* image="oem"
:return: result
:rtype: instance of :class:`Result`
"""
result = self.create_disk()
result = self.create_install_media(result)
self.append_unpartitioned_space()
return self.create_disk_format(result)
def create_disk(self) -> Result:
"""
Build a bootable raw disk image
:raises KiwiInstallMediaError:
if install media is required and image type is not oem
:raises KiwiVolumeManagerSetupError:
root overlay at the same time volumes are defined is not supported
:return: result
:rtype: instance of :class:`Result`
"""
# an instance of a class with the sync_data capability
# representing the entire image system except for the boot/ area
# which could live on another part of the disk
system: Any = None
# an instance of a class with the sync_data capability
# representing the boot/ area of the disk if not part of
# self.system
system_boot: Optional[FileSystemBase] = None
# an instance of a class with the sync_data capability
# representing the boot/efi area of the disk
system_efi: Optional[FileSystemBase] = None
# an instance of a class with the sync_data capability
# representing the spare_part_mountpoint area of the disk
system_spare: Optional[FileSystemBase] = None
# a list of instances with the sync_data capability
# representing the custom partitions area of the disk
system_custom_parts: List[FileSystemBase] = []
if self.install_media and self.build_type_name != 'oem':
raise KiwiInstallMediaError(
'Install media requires oem type setup, got {0}'.format(
self.build_type_name))
if self.root_filesystem_is_overlay and self.volume_manager_name:
raise KiwiVolumeManagerSetupError(
'Volume management together with root overlay is not supported'
)
# setup recovery archive, cleanup and create archive if requested
self.system_setup.create_recovery_archive()
# prepare initrd
if self.boot_image.has_initrd_support():
log.info('Preparing boot system')
self.boot_image.prepare()
# precalculate needed disk size
disksize_mbytes = self.disk_setup.get_disksize_mbytes()
# create the disk
log.info('Creating raw disk image %s', self.diskname)
loop_provider = LoopDevice(self.diskname, disksize_mbytes,
self.blocksize)
loop_provider.create()
disk = Disk(self.firmware.get_partition_table_type(), loop_provider,
self.xml_state.get_disk_start_sector())
# create the bootloader instance
if self.bootloader != 'custom':
self.bootloader_config = BootLoaderConfig.new(
self.bootloader,
self.xml_state,
root_dir=self.root_dir,
boot_dir=self.root_dir,
custom_args={
'targetbase':
loop_provider.get_device(),
'grub_directory_name':
Defaults.get_grub_boot_directory_name(self.root_dir),
'crypto_disk':
True if self.luks is not None else False,
'boot_is_crypto':
self.boot_is_crypto
})
# create disk partitions and instance device map
device_map = self._build_and_map_disk_partitions(disk, disksize_mbytes)
# create raid on current root device if requested
raid_root = None
if self.mdraid:
raid_root = RaidDevice(device_map['root'])
raid_root.create_degraded_raid(raid_level=self.mdraid)
device_map['root'] = raid_root.get_device()
disk.public_partition_id_map['kiwi_RaidPart'] = \
disk.public_partition_id_map['kiwi_RootPart']
disk.public_partition_id_map['kiwi_RaidDev'] = \
device_map['root'].get_device()
# create luks on current root device if requested
luks_root = None
if self.luks is not None:
luks_root = LuksDevice(device_map['root'])
self.luks_boot_keyname = '/.root.keyfile'
self.luks_boot_keyfile = ''.join(
[self.root_dir, self.luks_boot_keyname])
# use LUKS key file for the following conditions:
# 1. /boot is encrypted
# In this case grub needs to read from LUKS via the
# cryptodisk module which at the moment always asks
# for the passphrase even when empty. The keyfile
# setup makes sure only one interaction on the grub
# stage is needed
# 2. LUKS passphrase is configured as empty string
# In this case the keyfile allows to open the
# LUKS pool without asking
#
luks_need_keyfile = \
True if self.boot_is_crypto or self.luks == '' else False
luks_root.create_crypto_luks(
passphrase=self.luks,
os=self.luks_os,
keyfile=self.luks_boot_keyfile if luks_need_keyfile else '')
if luks_need_keyfile:
self.luks_boot_keyfile_setup = ''.join(
[self.root_dir, '/etc/dracut.conf.d/99-luks-boot.conf'])
self.boot_image.write_system_config_file(
config={'install_items': [self.luks_boot_keyname]},
config_file=self.luks_boot_keyfile_setup)
self.boot_image.include_file(
os.sep + os.path.basename(self.luks_boot_keyfile))
device_map['luks_root'] = device_map['root']
device_map['root'] = luks_root.get_device()
# create spare filesystem on spare partition if present
system_spare = self._build_spare_filesystem(device_map)
system_custom_parts = self._build_custom_parts_filesystem(
device_map, self.custom_partitions)
# create filesystems on boot partition(s) if any
system_boot, system_efi = self._build_boot_filesystems(device_map)
# create volumes and filesystems for root system
if self.volume_manager_name:
volume_manager_custom_parameters = {
'fs_mount_options':
self.custom_root_mount_args,
'fs_create_options':
self.custom_root_creation_args,
'root_label':
self.disk_setup.get_root_label(),
'root_is_snapshot':
self.xml_state.build_type.get_btrfs_root_is_snapshot(),
'root_is_readonly_snapshot':
self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot(
),
'quota_groups':
self.xml_state.build_type.get_btrfs_quota_groups(),
'resize_on_boot':
self.disk_resize_requested
}
volume_manager = VolumeManager.new(
self.volume_manager_name, device_map, self.root_dir + '/',
self.volumes, volume_manager_custom_parameters)
volume_manager.setup(self.volume_group_name)
volume_manager.create_volumes(self.requested_filesystem)
volume_manager.mount_volumes()
system = volume_manager
device_map['root'] = volume_manager.get_device().get('root')
device_map['swap'] = volume_manager.get_device().get('swap')
else:
log.info('Creating root(%s) filesystem on %s',
self.requested_filesystem,
device_map['root'].get_device())
filesystem_custom_parameters = {
'mount_options': self.custom_root_mount_args,
'create_options': self.custom_root_creation_args
}
filesystem = FileSystem.new(self.requested_filesystem,
device_map['root'],
self.root_dir + '/',
filesystem_custom_parameters)
filesystem.create_on_device(label=self.disk_setup.get_root_label())
system = filesystem
# create swap on current root device if requested
if self.swap_mbytes:
swap = FileSystem.new('swap', device_map['swap'])
swap.create_on_device(label='SWAP')
# store root partition/filesystem uuid for profile
self._preserve_root_partition_uuid(device_map)
self._preserve_root_filesystem_uuid(device_map)
# create a random image identifier
self.mbrid = SystemIdentifier()
self.mbrid.calculate_id()
# create first stage metadata to boot image
self._write_partition_id_config_to_boot_image(disk)
self._write_recovery_metadata_to_boot_image()
self._write_raid_config_to_boot_image(raid_root)
self._write_generic_fstab_to_boot_image(device_map, system)
self.system_setup.export_modprobe_setup(
self.boot_image.boot_root_directory)
# create first stage metadata to system image
self._write_image_identifier_to_system_image()
self._write_crypttab_to_system_image(luks_root)
self._write_generic_fstab_to_system_image(device_map, system)
if self.initrd_system == 'dracut':
if self.root_filesystem_is_multipath is False:
self.boot_image.omit_module('multipath')
if self.root_filesystem_is_overlay:
self.boot_image.include_module('kiwi-overlay')
self.boot_image.write_system_config_file(
config={'modules': ['kiwi-overlay']})
if self.disk_resize_requested:
self.boot_image.include_module('kiwi-repart')
# create initrd
if self.boot_image.has_initrd_support():
self.boot_image.create_initrd(self.mbrid)
# create second stage metadata to system image
self._copy_first_boot_files_to_system_image()
self._write_bootloader_meta_data_to_system_image(device_map, disk)
self.mbrid.write_to_disk(disk.storage_provider)
# set SELinux file security contexts if context exists
self._setup_selinux_file_contexts()
# syncing system data to disk image
self._sync_system_to_image(device_map, system, system_boot, system_efi,
system_spare, system_custom_parts)
# run post sync script hook
if self.system_setup.script_exists(defaults.POST_DISK_SYNC_SCRIPT):
disk_system = SystemSetup(self.xml_state, system.get_mountpoint())
disk_system.import_description()
disk_system.call_disk_script()
disk_system.cleanup()
# install boot loader
self._install_bootloader(device_map, disk, system)
# set root filesystem properties
self._setup_property_root_is_readonly_snapshot(system)
Result.verify_image_size(self.runtime_config.get_max_size_constraint(),
self.diskname)
# store image bundle_format in result
if self.bundle_format:
self.result.add_bundle_format(self.bundle_format)
# store image file name in result
compression = self.runtime_config.get_bundle_compression(default=True)
if self.luks is not None:
compression = False
self.result.add(
key='disk_image',
filename=self.diskname,
use_for_bundle=True if not self.image_format else False,
compress=compression,
shasum=True)
# create image root metadata
self.result.add(key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(key='image_changes',
filename=self.system_setup.export_package_changes(
self.target_dir),
use_for_bundle=True,
compress=True,
shasum=False)
self.result.add(key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
def create_disk_format(self, result_instance: Result) -> Result:
"""
Create a bootable disk format from a previously
created raw disk image
:param object result_instance: instance of :class:`Result`
:return: updated result_instance
:rtype: instance of :class:`Result`
"""
if self.image_format:
log.info('Creating %s Disk Format', self.image_format)
disk_format = DiskFormat.new(self.image_format, self.xml_state,
self.root_dir, self.target_dir)
disk_format.create_image_format()
disk_format.store_to_result(result_instance)
return result_instance
def append_unpartitioned_space(self) -> None:
"""
Extends the raw disk if an unpartitioned area is specified
"""
if self.unpartitioned_bytes:
log.info('Expanding disk with %d bytes of unpartitioned space',
self.unpartitioned_bytes)
disk_format = DiskFormat.new('raw', self.xml_state, self.root_dir,
self.target_dir)
disk_format.resize_raw_disk(self.unpartitioned_bytes, append=True)
firmware = FirmWare(self.xml_state)
loop_provider = LoopDevice(disk_format.diskname)
loop_provider.create(overwrite=False)
partitioner = Partitioner.new(firmware.get_partition_table_type(),
loop_provider)
partitioner.resize_table()
def create_install_media(self, result_instance: Result) -> Result:
"""
Build an installation image. The installation image is a
bootable hybrid ISO image which embeds the raw disk image
and an image installer
:param object result_instance: instance of :class:`Result`
:return: updated result_instance with installation media
:rtype: instance of :class:`Result`
"""
if self.install_media:
boot_image = None
if self.initrd_system == 'kiwi':
boot_image = self.boot_image
install_image = InstallImageBuilder(self.xml_state, self.root_dir,
self.target_dir, boot_image,
self.custom_args)
if self.install_iso or self.install_stick:
log.info('Creating hybrid ISO installation image')
install_image.create_install_iso()
result_instance.add(key='installation_image',
filename=install_image.isoname,
use_for_bundle=True,
compress=False,
shasum=True)
if self.install_pxe:
log.info('Creating PXE installation archive')
install_image.create_install_pxe_archive()
result_instance.add(key='installation_pxe_archive',
filename=install_image.pxetarball,
use_for_bundle=True,
compress=False,
shasum=True)
return result_instance
def _setup_selinux_file_contexts(self) -> None:
security_context = '/etc/selinux/targeted/contexts/files/file_contexts'
if os.path.exists(self.root_dir + security_context):
self.system_setup.set_selinux_file_contexts(security_context)
def _install_image_requested(self) -> bool:
return bool(self.install_iso or self.install_stick or self.install_pxe)
def _get_exclude_list_for_root_data_sync(self, device_map: Dict) -> list:
exclude_list = Defaults.\
get_exclude_list_for_root_data_sync() + Defaults.\
get_exclude_list_from_custom_exclude_files(self.root_dir)
if 'spare' in device_map and self.spare_part_mountpoint:
exclude_list.append('{0}/*'.format(
self.spare_part_mountpoint.lstrip(os.sep)))
exclude_list.append('{0}/.*'.format(
self.spare_part_mountpoint.lstrip(os.sep)))
if 'boot' in device_map and 's390' in self.arch:
exclude_list.append('boot/zipl/*')
exclude_list.append('boot/zipl/.*')
elif 'boot' in device_map:
exclude_list.append('boot/*')
exclude_list.append('boot/.*')
if 'efi' in device_map:
exclude_list.append('boot/efi/*')
exclude_list.append('boot/efi/.*')
if self.custom_partitions:
for map_name in sorted(self.custom_partitions.keys()):
if map_name in device_map:
mountpoint = os.path.normpath(
self.custom_partitions[map_name].mountpoint).lstrip(
os.sep)
exclude_list.append(f'{mountpoint}/*')
exclude_list.append(f'{mountpoint}/.*')
return exclude_list
@staticmethod
def _get_exclude_list_for_boot_data_sync() -> list:
return ['efi/*']
def _build_custom_parts_filesystem(
self, device_map: Dict,
custom_partitions: Dict['str',
ptable_entry_type]) -> List[FileSystemBase]:
filesystem_list = []
if custom_partitions:
for map_name in sorted(custom_partitions.keys()):
if map_name in device_map:
ptable_entry = custom_partitions[map_name]
filesystem = FileSystem.new(
ptable_entry.filesystem, device_map[map_name],
f'{self.root_dir}{ptable_entry.mountpoint}/')
filesystem.create_on_device(label=map_name.upper())
filesystem_list.append(filesystem)
return filesystem_list
def _build_spare_filesystem(self,
device_map: Dict) -> Optional[FileSystemBase]:
if 'spare' in device_map and self.spare_part_fs:
spare_part_data_path = None
spare_part_custom_parameters = {
'fs_attributes':
self.xml_state.get_build_type_spare_part_fs_attributes()
}
if self.spare_part_mountpoint:
spare_part_data_path = self.root_dir + '{0}/'.format(
self.spare_part_mountpoint)
filesystem = FileSystem.new(self.spare_part_fs,
device_map['spare'],
spare_part_data_path,
spare_part_custom_parameters)
filesystem.create_on_device(label='SPARE')
return filesystem
return None
def _build_boot_filesystems(
self, device_map: Dict
) -> Tuple[Optional[FileSystemBase], Optional[FileSystemBase]]:
system_boot = None
system_efi = None
if 'efi' in device_map:
log.info('Creating EFI(fat16) filesystem on %s',
device_map['efi'].get_device())
filesystem = FileSystem.new('fat16', device_map['efi'],
self.root_dir + '/boot/efi/')
filesystem.create_on_device(label=self.disk_setup.get_efi_label())
system_efi = filesystem
if 'boot' in device_map:
boot_filesystem = self.requested_boot_filesystem
if not boot_filesystem:
boot_filesystem = self.requested_filesystem
boot_directory = self.root_dir + '/boot/'
if 's390' in self.arch:
boot_directory = self.root_dir + '/boot/zipl/'
log.info('Creating boot(%s) filesystem on %s', boot_filesystem,
device_map['boot'].get_device())
filesystem = FileSystem.new(boot_filesystem, device_map['boot'],
boot_directory)
filesystem.create_on_device(label=self.disk_setup.get_boot_label())
system_boot = filesystem
return system_boot, system_efi
def _build_and_map_disk_partitions(self, disk: Disk,
disksize_mbytes: float) -> Dict:
disk.wipe()
disksize_used_mbytes = 0
if self.firmware.legacy_bios_mode():
log.info('--> creating EFI CSM(legacy bios) partition')
partition_mbsize = self.firmware.get_legacy_bios_partition_size()
disk.create_efi_csm_partition(partition_mbsize)
disksize_used_mbytes += partition_mbsize
if self.firmware.efi_mode():
log.info('--> creating EFI partition')
partition_mbsize = self.firmware.get_efi_partition_size()
disk.create_efi_partition(partition_mbsize)
disksize_used_mbytes += partition_mbsize
if self.firmware.ofw_mode():
log.info('--> creating PReP partition')
partition_mbsize = self.firmware.get_prep_partition_size()
disk.create_prep_partition(partition_mbsize)
disksize_used_mbytes += partition_mbsize
if self.disk_setup.need_boot_partition():
log.info('--> creating boot partition')
partition_mbsize = self.disk_setup.boot_partition_size()
disk.create_boot_partition(partition_mbsize)
disksize_used_mbytes += partition_mbsize
if self.swap_mbytes:
if not self.volume_manager_name or self.volume_manager_name != 'lvm':
log.info('--> creating SWAP partition')
disk.create_swap_partition(f'{self.swap_mbytes}')
disksize_used_mbytes += self.swap_mbytes
if self.custom_partitions:
log.info('--> creating custom partition(s): {0}'.format(
sorted(self.custom_partitions.keys())))
disk.create_custom_partitions(self.custom_partitions)
if self.spare_part_mbsize and not self.spare_part_is_last:
log.info('--> creating spare partition')
disk.create_spare_partition(f'{self.spare_part_mbsize}')
if self.root_filesystem_is_overlay:
log.info('--> creating readonly root partition')
squashed_root_file = Temporary().new_file()
squashed_root = FileSystemSquashFs(
device_provider=DeviceProvider(),
root_dir=self.root_dir,
custom_args={
'compression':
self.xml_state.build_type.get_squashfscompression()
})
squashed_root.create_on_file(
filename=squashed_root_file.name,
exclude=[Defaults.get_shared_cache_location()])
squashed_rootfs_mbsize = int(
os.path.getsize(squashed_root_file.name) /
1048576) + Defaults.get_min_partition_mbytes()
disk.create_root_readonly_partition(squashed_rootfs_mbsize)
disksize_used_mbytes += squashed_rootfs_mbsize
if self.spare_part_mbsize and self.spare_part_is_last:
rootfs_mbsize = disksize_mbytes - disksize_used_mbytes - \
self.spare_part_mbsize - Defaults.get_min_partition_mbytes()
else:
rootfs_mbsize = 'all_free'
if self.volume_manager_name and self.volume_manager_name == 'lvm':
log.info('--> creating LVM root partition')
disk.create_root_lvm_partition(rootfs_mbsize)
elif self.mdraid:
log.info('--> creating mdraid root partition')
disk.create_root_raid_partition(rootfs_mbsize)
else:
log.info('--> creating root partition')
disk.create_root_partition(rootfs_mbsize)
if self.spare_part_mbsize and self.spare_part_is_last:
log.info('--> creating spare partition')
disk.create_spare_partition('all_free')
if self.firmware.bios_mode():
log.info('--> setting active flag to primary boot partition')
disk.activate_boot_partition()
if self.firmware.ofw_mode():
log.info('--> setting active flag to primary PReP partition')
disk.activate_boot_partition()
if self.firmware.efi_mode():
if self.force_mbr:
log.info('--> converting partition table to MBR')
disk.create_mbr()
elif self.hybrid_mbr:
log.info('--> converting partition table to hybrid GPT/MBR')
disk.create_hybrid_mbr()
disk.map_partitions()
return disk.get_device()
def _write_partition_id_config_to_boot_image(self, disk: Disk) -> None:
log.info('Creating config.partids in boot system')
filename = ''.join(
[self.boot_image.boot_root_directory, '/config.partids'])
partition_id_map = disk.get_public_partition_id_map()
with open(filename, 'w') as partids:
for id_name, id_value in list(partition_id_map.items()):
partids.write('{0}="{1}"{2}'.format(id_name, id_value,
os.linesep))
self.boot_image.include_file(os.sep + os.path.basename(filename))
def _write_raid_config_to_boot_image(
self, raid_root: Optional[RaidDevice]) -> None:
if raid_root is not None:
log.info('Creating etc/mdadm.conf in boot system')
filename = ''.join(
[self.boot_image.boot_root_directory, '/etc/mdadm.conf'])
raid_root.create_raid_config(filename)
self.boot_image.include_file(
os.sep + os.sep.join(['etc', os.path.basename(filename)]))
def _write_crypttab_to_system_image(
self, luks_root: Optional[LuksDevice]) -> None:
if luks_root is not None:
log.info('Creating etc/crypttab')
filename = ''.join([self.root_dir, '/etc/crypttab'])
luks_root.create_crypttab(filename)
self.boot_image.include_file(
os.sep + os.sep.join(['etc', os.path.basename(filename)]))
def _write_generic_fstab_to_system_image(self, device_map: Dict,
system: Any) -> None:
log.info('Creating generic system etc/fstab')
self._write_generic_fstab(device_map, self.system_setup, system)
def _write_generic_fstab_to_boot_image(self, device_map: Dict,
system: Any) -> None:
if self.initrd_system == 'kiwi':
log.info('Creating generic boot image etc/fstab')
self._write_generic_fstab(device_map, self.boot_image.setup,
system)
def _write_generic_fstab(self, device_map: Dict, setup: SystemSetup,
system: Any) -> None:
root_is_snapshot = \
self.xml_state.build_type.get_btrfs_root_is_snapshot()
root_is_readonly_snapshot = \
self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()
fs_check_interval = '0 1'
custom_root_mount_args = list(self.custom_root_mount_args)
if root_is_snapshot and root_is_readonly_snapshot:
custom_root_mount_args += ['ro']
fs_check_interval = '0 0'
self._add_fstab_entry(device_map['root'].get_device(), '/',
custom_root_mount_args, fs_check_interval)
if device_map.get('boot'):
if 's390' in self.arch:
boot_mount_point = '/boot/zipl'
else:
boot_mount_point = '/boot'
self._add_fstab_entry(device_map['boot'].get_device(),
boot_mount_point)
if device_map.get('efi'):
self._add_fstab_entry(device_map['efi'].get_device(), '/boot/efi')
if self.volume_manager_name:
volume_fstab_entries = system.get_fstab(self.persistency_type,
self.requested_filesystem)
for volume_fstab_entry in volume_fstab_entries:
self.fstab.add_entry(volume_fstab_entry)
if device_map.get('spare') and \
self.spare_part_fs and self.spare_part_mountpoint:
self._add_fstab_entry(device_map['spare'].get_device(),
self.spare_part_mountpoint)
if device_map.get('swap'):
self._add_fstab_entry(device_map['swap'].get_device(), 'swap')
if self.custom_partitions:
for map_name in sorted(self.custom_partitions.keys()):
if device_map.get(map_name):
self._add_fstab_entry(
device_map[map_name].get_device(),
self.custom_partitions[map_name].mountpoint)
setup.create_fstab(self.fstab)
def _add_fstab_entry(self,
device: str,
mount_point: str,
options: List = None,
check: str = '0 0') -> None:
if not options:
options = ['defaults']
block_operation = BlockID(device)
if self.volume_manager_name and self.volume_manager_name == 'lvm' \
and (mount_point == '/' or mount_point == 'swap'):
fstab_entry = ' '.join([
device, mount_point,
block_operation.get_filesystem(), ','.join(options), check
])
else:
blkid_type = 'LABEL' if self.persistency_type == 'by-label' \
else 'UUID'
device_id = block_operation.get_blkid(blkid_type)
fstab_entry = ' '.join([
blkid_type + '=' + device_id, mount_point,
block_operation.get_filesystem(), ','.join(options), check
])
self.fstab.add_entry(fstab_entry)
def _preserve_root_partition_uuid(self, device_map: Dict) -> None:
block_operation = BlockID(device_map['root'].get_device())
partition_uuid = block_operation.get_blkid('PARTUUID')
if partition_uuid:
self.xml_state.set_root_partition_uuid(partition_uuid)
def _preserve_root_filesystem_uuid(self, device_map: Dict) -> None:
block_operation = BlockID(device_map['root'].get_device())
rootfs_uuid = block_operation.get_blkid('UUID')
if rootfs_uuid:
self.xml_state.set_root_filesystem_uuid(rootfs_uuid)
def _write_image_identifier_to_system_image(self) -> None:
log.info('Creating image identifier: %s', self.mbrid.get_id())
self.mbrid.write(self.root_dir + '/boot/mbrid')
def _write_recovery_metadata_to_boot_image(self) -> None:
if os.path.exists(self.root_dir + '/recovery.partition.size'):
log.info('Copying recovery metadata to boot image')
recovery_metadata = ''.join(
[self.root_dir, '/recovery.partition.size'])
Command.run(
['cp', recovery_metadata, self.boot_image.boot_root_directory])
self.boot_image.include_file(os.sep +
os.path.basename(recovery_metadata))
def _write_bootloader_meta_data_to_system_image(self, device_map: Dict,
disk: Disk) -> None:
if self.bootloader != 'custom':
log.info('Creating %s bootloader configuration', self.bootloader)
boot_options = []
if self.mdraid:
boot_options.append('rd.auto')
root_device = device_map['root']
boot_device = root_device
if 'boot' in device_map:
boot_device = device_map['boot']
boot_uuid = disk.get_uuid(boot_device.get_device())
boot_uuid_unmapped = disk.get_uuid(
device_map['luks_root'].get_device(
)) if self.luks else boot_uuid
self.bootloader_config.setup_disk_boot_images(boot_uuid_unmapped)
self.bootloader_config.write_meta_data(
root_device=device_map['root'].get_device(),
boot_options=' '.join(boot_options))
log.info('Creating config.bootoptions')
filename = ''.join(
[self.boot_image.boot_root_directory, '/config.bootoptions'])
kexec_boot_options = ' '.join([
self.bootloader_config.get_boot_cmdline(
device_map['root'].get_device())
] + boot_options)
with open(filename, 'w') as boot_optionsfp:
boot_optionsfp.write('{0}{1}'.format(kexec_boot_options,
os.linesep))
partition_id_map = disk.get_public_partition_id_map()
boot_partition_id = partition_id_map['kiwi_RootPart']
if 'kiwi_BootPart' in partition_id_map:
boot_partition_id = partition_id_map['kiwi_BootPart']
self.system_setup.call_edit_boot_config_script(
self.requested_filesystem, int(boot_partition_id))
def _sync_system_to_image(
self, device_map: Dict, system: Any,
system_boot: Optional[FileSystemBase],
system_efi: Optional[FileSystemBase],
system_spare: Optional[FileSystemBase],
system_custom_parts: List[FileSystemBase]) -> None:
log.info('Syncing system to image')
if system_spare:
log.info('--> Syncing spare partition data')
system_spare.sync_data()
for system_custom_part in system_custom_parts:
log.info('--> Syncing custom partition(s) data')
system_custom_part.sync_data()
if system_efi:
log.info('--> Syncing EFI boot data to EFI partition')
system_efi.sync_data()
if system_boot:
log.info('--> Syncing boot data at extra partition')
system_boot.sync_data(self._get_exclude_list_for_boot_data_sync())
log.info('--> Syncing root filesystem data')
if self.root_filesystem_is_overlay:
squashed_root_file = Temporary().new_file()
squashed_root = FileSystemSquashFs(
device_provider=DeviceProvider(),
root_dir=self.root_dir,
custom_args={
'compression':
self.xml_state.build_type.get_squashfscompression()
})
squashed_root.create_on_file(
filename=squashed_root_file.name,
exclude=self._get_exclude_list_for_root_data_sync(device_map))
Command.run([
'dd',
'if=%s' % squashed_root_file.name,
'of=%s' % device_map['readonly'].get_device()
])
else:
system.sync_data(
self._get_exclude_list_for_root_data_sync(device_map))
def _install_bootloader(self, device_map: Dict, disk, system: Any) -> None:
root_device = device_map['root']
boot_device = root_device
if 'boot' in device_map:
boot_device = device_map['boot']
if 'readonly' in device_map:
root_device = device_map['readonly']
custom_install_arguments = {
'boot_device': boot_device.get_device(),
'root_device': root_device.get_device(),
'firmware': self.firmware,
'target_removable': self.target_removable
}
if 'efi' in device_map:
efi_device = device_map['efi']
custom_install_arguments.update(
{'efi_device': efi_device.get_device()})
if 'prep' in device_map:
prep_device = device_map['prep']
custom_install_arguments.update(
{'prep_device': prep_device.get_device()})
if self.volume_manager_name:
system.umount_volumes()
custom_install_arguments.update(
{'system_volumes': system.get_volumes()})
if self.bootloader != 'custom':
# create bootloader config prior bootloader installation
self.bootloader_config.setup_disk_image_config(
boot_options=custom_install_arguments)
if 's390' in self.arch:
self.bootloader_config.write()
# cleanup bootloader config resources taken prior to next steps
del self.bootloader_config
log.debug("custom arguments for bootloader installation %s",
custom_install_arguments)
bootloader = BootLoaderInstall.new(self.bootloader, self.root_dir,
disk.storage_provider,
custom_install_arguments)
if bootloader.install_required():
bootloader.install()
bootloader.secure_boot_install()
self.system_setup.call_edit_boot_install_script(
self.diskname, boot_device.get_device())
def _setup_property_root_is_readonly_snapshot(self, system: Any) -> None:
if self.volume_manager_name:
root_is_snapshot = \
self.xml_state.build_type.get_btrfs_root_is_snapshot()
root_is_readonly_snapshot = \
self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()
if root_is_snapshot and root_is_readonly_snapshot:
log.info('Setting root filesystem into read-only mode')
system.mount_volumes()
system.set_property_readonly_root()
system.umount_volumes()
def _copy_first_boot_files_to_system_image(self) -> None:
boot_names = self.boot_image.get_boot_names()
if self.initrd_system == 'kiwi':
log.info('Copy boot files to system image')
kernel = Kernel(self.boot_image.boot_root_directory)
log.info('--> boot image kernel as %s', boot_names.kernel_name)
kernel.copy_kernel(self.root_dir,
''.join(['/boot/', boot_names.kernel_name]))
if self.xen_server:
if kernel.get_xen_hypervisor():
log.info('--> boot image Xen hypervisor as xen.gz')
kernel.copy_xen_hypervisor(self.root_dir, '/boot/xen.gz')
else:
raise KiwiDiskBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image.boot_root_directory)
if self.boot_image.initrd_filename:
log.info('--> initrd archive as %s', boot_names.initrd_name)
Command.run([
'mv', self.boot_image.initrd_filename,
self.root_dir + ''.join(['/boot/', boot_names.initrd_name])
])