class TestSystemSize: def setup(self): self.size = SystemSize('directory') def setup_method(self, cls): self.setup() def test_customize_ext(self): self.size.accumulate_files = mock.Mock(return_value=10000) assert self.size.customize(42, 'ext3') == 67 def test_customize_btrfs(self): assert self.size.customize(42, 'btrfs') == 63 def test_customize_xfs(self): assert self.size.customize(42, 'xfs') == 63 @patch('kiwi.system.size.Command.run') def test_accumulate_mbyte_file_sizes(self, mock_command): assert isinstance(self.size.accumulate_mbyte_file_sizes(['/foo']), int) mock_command.assert_called_once_with([ 'du', '-s', '--apparent-size', '--block-size', '1', '--exclude', 'directory/proc', '--exclude', 'directory/sys', '--exclude', 'directory/dev', '--exclude', '/foo', 'directory' ]) @patch('kiwi.system.size.Command.run') def test_accumulate_files(self, mock_command): assert isinstance(self.size.accumulate_files(), int) mock_command.assert_called_once_with( ['bash', '-c', 'find directory | wc -l'])
class TestSystemSize(object): def setup(self): self.size = SystemSize('directory') def test_customize_ext(self): self.size.accumulate_files = mock.Mock(return_value=10000) assert self.size.customize(42, 'ext3') == 67 def test_customize_btrfs(self): assert self.size.customize(42, 'btrfs') == 63 def test_customize_xfs(self): assert self.size.customize(42, 'xfs') == 63 @patch('kiwi.system.size.Command.run') def test_accumulate_mbyte_file_sizes(self, mock_command): self.size.accumulate_mbyte_file_sizes() mock_command.assert_called_once_with( ['du', '-s', '--apparent-size', '--block-size', '1', 'directory']) @patch('kiwi.system.size.Command.run') def test_accumulate_files(self, mock_command): self.size.accumulate_files() mock_command.assert_called_once_with( ['bash', '-c', 'find directory | wc -l'])
class TestSystemSize(object): def setup(self): self.size = SystemSize('directory') def test_customize_ext(self): self.size.accumulate_files = mock.Mock( return_value=10000 ) assert self.size.customize(42, 'ext3') == 67 def test_customize_btrfs(self): assert self.size.customize(42, 'btrfs') == 63 def test_customize_xfs(self): assert self.size.customize(42, 'xfs') == 63 @patch('kiwi.system.size.Command.run') def test_accumulate_mbyte_file_sizes(self, mock_command): self.size.accumulate_mbyte_file_sizes(['/foo']) mock_command.assert_called_once_with( [ 'du', '-s', '--apparent-size', '--block-size', '1', '--exclude', '/foo', 'directory' ] ) @patch('kiwi.system.size.Command.run') def test_accumulate_files(self, mock_command): self.size.accumulate_files() mock_command.assert_called_once_with( ['bash', '-c', 'find directory | wc -l'] )
def _calculate_volume_mbytes(self): """ Calculate the number of mbytes each volume path currently consumes and also provide a total number of these values """ volume_mbytes_type = namedtuple( 'volume_mbytes_type', ['volume', 'total'] ) volume_mbytes = {} volume_total = 0 for volume in self.volumes: if volume.realpath and not volume.realpath == '/': path_to_volume = self.root_dir + '/' + volume.realpath if os.path.exists(path_to_volume): volume_size = SystemSize(path_to_volume) volume_mbytes[volume.realpath] = volume_size.customize( volume_size.accumulate_mbyte_file_sizes(), self.filesystem ) volume_total += volume_mbytes[volume.realpath] return volume_mbytes_type( volume=volume_mbytes, total=volume_total )
def _calculate_volume_mbytes(self): """ Calculate the number of mbytes each volume path currently consumes and also provide a total number of these values. """ volume_mbytes_type = namedtuple( 'volume_mbytes_type', ['volume', 'total'] ) volume_mbytes = {} volume_total = 0 for volume in self.volumes: if volume.realpath and not volume.realpath == '/': path_to_volume = self.root_dir + '/' + volume.realpath if os.path.exists(path_to_volume): volume_size = SystemSize(path_to_volume) volume_mbytes[volume.realpath] = volume_size.customize( volume_size.accumulate_mbyte_file_sizes(), self.filesystem ) else: volume_mbytes[volume.realpath] = 0 volume_total += volume_mbytes[volume.realpath] return volume_mbytes_type( volume=volume_mbytes, total=volume_total )
def get_volume_mbsize(self, volume, all_volumes, filesystem_name, resize_on_boot=False): """ Implements size lookup for the given path and desired filesystem according to the specified size type :param tuple volume: volume to check size for :param list all_volumes: list of all volume tuples :param str filesystem_name: filesystem name :param resize_on_boot: specify the time of the resize. If the resize happens at boot time the volume size is only the minimum size to just store the data. If the volume size is fixed and does not change at boot time the returned size is the requested size which can be greater than the minimum needed size. :return: mbsize :rtype: int """ [size_type, mbsize] = volume.size.split(':') lookup_path = volume.realpath lookup_abspath = os.path.normpath( os.sep.join([self.root_dir, lookup_path])) mbsize = int(mbsize) if resize_on_boot: # If resize_on_boot is true, the disk is self expandable and # will resize to the configured sizes on first boot # Therefore the requested size is set to null and we add # the required minimum size for just storing the data size_type = 'freespace' mbsize = Defaults.get_min_volume_mbytes() if size_type == 'freespace' and os.path.exists(lookup_abspath): exclude_paths = [] for volume in all_volumes: volume_path = volume.realpath if lookup_path == volume_path: continue if lookup_path == os.sep: # exclude any sub volume path if lookup_path is / exclude_paths.append( os.path.normpath(self.root_dir + os.sep + volume_path)) elif volume_path.startswith(lookup_path): # exclude any sub volume path below lookup_path exclude_paths.append( os.path.normpath(self.root_dir + os.sep + volume_path)) volume_size = SystemSize(lookup_abspath) if mbsize != Defaults.get_min_volume_mbytes(): mbsize += Defaults.get_min_volume_mbytes() mbsize += volume_size.customize( volume_size.accumulate_mbyte_file_sizes(exclude_paths), filesystem_name) return mbsize
def _calculate_partition_mbytes(self): """ Calculate the number of mbytes each partition path consumes """ partition_mbytes_type = namedtuple( 'partition_mbytes_type', ['partition'] ) partition_mbytes = {} for map_name in sorted(self.custom_partitions.keys()): partition_mount_path = self.custom_partitions[map_name].mountpoint if partition_mount_path: partition_filesystem = self.custom_partitions[map_name].filesystem path_to_partition = os.path.normpath( os.sep.join([self.root_dir, partition_mount_path]) ) if os.path.exists(path_to_partition): partition_size = SystemSize(path_to_partition) partition_mbytes[partition_mount_path] = partition_size.customize( partition_size.accumulate_mbyte_file_sizes(), partition_filesystem ) else: partition_mbytes[partition_mount_path] = 0 return partition_mbytes_type( partition=partition_mbytes )
class FileSystemSetup(object): """ **Implement filesystem setup methods** Methods from this class provides information from the root directory required before building a filesystem image :param object xml_state: Instance of XMLState :param string root_dir: root directory path """ def __init__(self, xml_state, root_dir): self.configured_size = xml_state.get_build_type_size( include_unpartitioned=True) if xml_state.get_build_type_unpartitioned_bytes() > 0: log.warning( 'Unpartitoned size attribute is ignored for filesystem images') self.size = SystemSize(root_dir) self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type in Defaults.get_filesystem_image_types(): self.requested_filesystem = self.requested_image_type else: self.requested_filesystem = xml_state.build_type.get_filesystem() def get_size_mbytes(self, filesystem=None): """ Precalculate the requires size in mbytes to store all data from the root directory in the requested filesystem. Return the configured value if present, if not return the calculated result :param string filesystem: name :return: mbytes :rtype: int """ root_dir_mbytes = self.size.accumulate_mbyte_file_sizes() filesystem_mbytes = self.size.customize( root_dir_mbytes, self.requested_filesystem or filesystem) if not self.configured_size: log.info('Using calculated size: %d MB', filesystem_mbytes) return filesystem_mbytes elif self.configured_size.additive: result_filesystem_mbytes = \ self.configured_size.mbytes + filesystem_mbytes log.info('Using configured size: %d MB + %d MB calculated = %d MB', self.configured_size.mbytes, filesystem_mbytes, result_filesystem_mbytes) return result_filesystem_mbytes else: log.info('Using configured size: %d MB', self.configured_size.mbytes) if self.configured_size.mbytes < filesystem_mbytes: log.warning( '--> Configured size smaller than calculated size: %d MB', filesystem_mbytes) return self.configured_size.mbytes
def __init__(self, path, state, offset=None): self._device = None self._offset = offset self._path = path self._state = state sz = SystemSize(path) self._size = sz.customize(size=sz.accumulate_mbyte_file_sizes(), requested_filesystem='ext4') * 1024 * 1024
def get_volume_mbsize( self, volume, all_volumes, filesystem_name, image_type=None ): """ Implements size lookup for the given path and desired filesystem according to the specified size type :param tuple volume: volume to check size for :param list all_volumes: list of all volume tuples :param str filesystem_name: filesystem name :param image_type: build type name :return: mbsize :rtype: int """ [size_type, mbsize] = volume.size.split(':') lookup_path = volume.realpath if image_type and image_type == 'oem': # only for vmx types we need to create the volumes in the # configured size. oem disks are self expandable and will # resize to the configured sizes on first boot of the disk # image. Therefore the requested size is set to null # and we add the required minimum size to hold the data size_type = 'freespace' mbsize = 0 if size_type == 'freespace': exclude_paths = [] for volume in all_volumes: volume_path = volume.realpath if lookup_path == volume_path: continue if lookup_path == os.sep: # exclude any sub volume path if lookup_path is / [LVRoot] exclude_paths.append( os.path.normpath(self.root_dir + os.sep + volume_path) ) elif volume_path.startswith(lookup_path): # exclude any sub volume path below lookup_path exclude_paths.append( os.path.normpath(self.root_dir + os.sep + volume_path) ) volume_size = SystemSize( os.path.normpath(self.root_dir + os.sep + lookup_path) ) mbsize = int(mbsize) + \ Defaults.get_min_volume_mbytes() mbsize += volume_size.customize( volume_size.accumulate_mbyte_file_sizes(exclude_paths), filesystem_name ) return mbsize
def get_volume_mbsize(self, volume, all_volumes, filesystem_name, image_type=None): """ Implements size lookup for the given path and desired filesystem according to the specified size type :param tuple volume: volume to check size for :param list all_volumes: list of all volume tuples :param str filesystem_name: filesystem name :param image_type: build type name :return: mbsize :rtype: int """ [size_type, mbsize] = volume.size.split(':') lookup_path = volume.realpath lookup_abspath = os.path.normpath( os.sep.join([self.root_dir, lookup_path])) mbsize = int(mbsize) if image_type and image_type == 'oem': # only for vmx types we need to create the volumes in the # configured size. oem disks are self expandable and will # resize to the configured sizes on first boot of the disk # image. Therefore the requested size is set to null # and we add the required minimum size to hold the data size_type = 'freespace' mbsize = Defaults.get_min_volume_mbytes() if size_type == 'freespace' and os.path.exists(lookup_abspath): exclude_paths = [] for volume in all_volumes: volume_path = volume.realpath if lookup_path == volume_path: continue if lookup_path == os.sep: # exclude any sub volume path if lookup_path is / [LVRoot] exclude_paths.append( os.path.normpath(self.root_dir + os.sep + volume_path)) elif volume_path.startswith(lookup_path): # exclude any sub volume path below lookup_path exclude_paths.append( os.path.normpath(self.root_dir + os.sep + volume_path)) volume_size = SystemSize(lookup_abspath) if mbsize != Defaults.get_min_volume_mbytes(): mbsize += Defaults.get_min_volume_mbytes() mbsize += volume_size.customize( volume_size.accumulate_mbyte_file_sizes(exclude_paths), filesystem_name) return mbsize
def get_volume_mbsize(self, mbsize, size_type, realpath, filesystem_name, image_type=None): """ Implements size lookup for the given path and desired filesystem according to the specified size type :param int mbsize: configured volume size :param string size_type: relative or absolute size setup :param string realpath: volume real path name :param string filesystem_name: filesystem name :param image_type: build type name :return: mbsize :rtype: int """ if image_type and image_type == 'oem': # only for vmx types we need to create the volumes in the # configured size. oem disks are self expandable and will # resize to the configured sizes on first boot of the disk # image. Therefore the requested size is set to null # and we add the required minimum size to hold the data size_type = 'freespace' mbsize = 0 if size_type == 'freespace': # Please note for nested volumes which contains other volumes # the freespace calculation is not correct. Example: # /usr is a volume and /usr/lib is a volume. If freespace is # set for the /usr volume the data size calculated also # contains the data of the /usr/lib path which will live in # an extra volume later. The result will be more freespace # than expected ! Especially for the root volume this matters # most because it always nests all other volumes. Thus it is # better to use a fixed size for the root volume if it is not # configured to use all rest space # # You are invited to fix it :) volume_size = SystemSize(self.root_dir + '/' + realpath) mbsize = int(mbsize) + \ Defaults.get_min_volume_mbytes() mbsize += volume_size.customize( volume_size.accumulate_mbyte_file_sizes(), filesystem_name) return mbsize
class DiskSetup(object): """ Implement disk setup methods providing information required before building a disk image Attributes * :attr:`configured_size` Configured size setup * :attr:`build_type_name` Configured build type name * :attr:`filesystem` Configured filesystem name * :attr:`bootpart_requested` Configured request for a boot partition * :attr:`bootpart_mbytes` Configured boot partition size * :attr:`mdraid` Configured raid setup * :attr:`luks` Configured LUKS credentials * :attr:`volume_manager` Configured volume manager name * :attr:`bootloader` Configured bootloader * :attr:`oemconfig` Configured oemconfig section * :attr:`volumes` Configured volumes * :attr:`firmware` Instance of FirmWare * :attr:`rootsize` Instance of SystemSize * :attr:`root_dir` root directory path name * :attr:`xml_state` Instance of XMLState """ def __init__(self, xml_state, root_dir): self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot( ) self.configured_size = xml_state.get_build_type_size() self.build_type_name = xml_state.get_build_type_name() self.filesystem = xml_state.build_type.get_filesystem() self.bootpart_requested = xml_state.build_type.get_bootpartition() self.bootpart_mbytes = xml_state.build_type.get_bootpartsize() self.spare_part_mbytes = xml_state.get_build_type_spare_part_size() self.mdraid = xml_state.build_type.get_mdraid() self.luks = xml_state.build_type.get_luks() self.volume_manager = xml_state.get_volume_management() self.bootloader = xml_state.build_type.get_bootloader() self.oemconfig = xml_state.get_build_type_oemconfig_section() self.volumes = xml_state.get_volumes() self.firmware = FirmWare(xml_state) self.rootsize = SystemSize(root_dir) self.root_dir = root_dir self.xml_state = xml_state def get_disksize_mbytes(self): """ Precalculate disk size requirements in mbytes :return: disk size mbytes :rtype: int """ log.info('Precalculating required disk size') calculated_disk_mbytes = 0 root_filesystem_mbytes = self.rootsize.customize( self.rootsize.accumulate_mbyte_file_sizes(), self.filesystem) calculated_disk_mbytes += root_filesystem_mbytes log.info('--> system data with filesystem overhead needs %s MB', root_filesystem_mbytes) if self.volume_manager and self.volume_manager == 'lvm': if self.build_type_name == 'vmx': # only for vmx types we need to add the configured volume # sizes. oem disks are self expandable and will resize to # the configured sizes on first boot of the disk image volume_mbytes = self._accumulate_volume_size( root_filesystem_mbytes) if volume_mbytes: calculated_disk_mbytes += volume_mbytes log.info('--> volume(s) size setup adding %s MB', volume_mbytes) legacy_bios_mbytes = self.firmware.get_legacy_bios_partition_size() if legacy_bios_mbytes: calculated_disk_mbytes += legacy_bios_mbytes log.info('--> legacy bios boot partition adding %s MB', legacy_bios_mbytes) boot_mbytes = self.boot_partition_size() if boot_mbytes: calculated_disk_mbytes += boot_mbytes log.info('--> boot partition adding %s MB', boot_mbytes) if self.spare_part_mbytes: calculated_disk_mbytes += self.spare_part_mbytes log.info('--> spare partition adding %s MB', self.spare_part_mbytes) efi_mbytes = self.firmware.get_efi_partition_size() if efi_mbytes: calculated_disk_mbytes += efi_mbytes log.info('--> EFI partition adding %s MB', efi_mbytes) prep_mbytes = self.firmware.get_prep_partition_size() if prep_mbytes: calculated_disk_mbytes += prep_mbytes log.info('--> PReP partition adding %s MB', prep_mbytes) recovery_mbytes = self._inplace_recovery_partition_size() if recovery_mbytes: calculated_disk_mbytes += recovery_mbytes log.info('--> In-place recovery partition adding: %s MB', recovery_mbytes) if not self.configured_size: log.info('Using calculated disk size: %d MB', calculated_disk_mbytes) return calculated_disk_mbytes elif self.configured_size.additive: result_disk_mbytes = \ self.configured_size.mbytes + calculated_disk_mbytes log.info( 'Using configured disk size: %d MB + %d MB calculated = %d MB', self.configured_size.mbytes, calculated_disk_mbytes, result_disk_mbytes) return result_disk_mbytes else: log.info('Using configured disk size: %d MB', self.configured_size.mbytes) if self.configured_size.mbytes < calculated_disk_mbytes: log.warning( '--> Configured size smaller than calculated size: %d MB', calculated_disk_mbytes) return self.configured_size.mbytes def need_boot_partition(self): """ Decide if an extra boot partition is needed. This is done with the bootpartition attribute from the type, however if it is not set it depends on some other type configuration parameters if we need a boot partition or not :rtype: bool """ if self.bootpart_requested is True: return True if self.bootpart_requested is False: return False if self.mdraid: return True if self.volume_manager: return True if self.filesystem == 'btrfs': return True if self.filesystem == 'xfs': return True if self.root_filesystem_is_overlay: return True if self.bootloader == 'grub2_s390x_emu': return True if self.luks: return True def get_boot_label(self): """ Filesystem Label to use for the boot partition :return: label name :rtype: string """ label = 'BOOT' if self.bootloader == 'grub2_s390x_emu': label = 'ZIPL' return label def get_root_label(self): """ Filesystem Label to use for the root partition If not specified in the XML configuration the default root label is set to 'ROOT' :return: label name :rtype: string """ root_label = self.xml_state.build_type.get_rootfs_label() if not root_label: root_label = 'ROOT' return root_label def get_efi_label(self): """ Filesystem Label to use for the EFI partition :return: label name :rtype: string """ return 'EFI' def boot_partition_size(self): """ Size of the boot partition in mbytes :return: boot size mbytes :rtype: int """ if self.need_boot_partition(): if self.bootpart_mbytes: return self.bootpart_mbytes else: return Defaults.get_default_boot_mbytes() def _inplace_recovery_partition_size(self): """ In inplace recovery mode the recovery archive is created at install time. This requires free space on the disk. The amount of free space is specified with the oem-recovery-part-size attribute. If specified we add the given size to the disk. If not specified an inplace setup at install time will be moved to the first boot of an oem image when the recovery partition has been created """ if self.oemconfig and self.oemconfig.get_oem_inplace_recovery(): recovery_mbytes = self.oemconfig.get_oem_recovery_part_size() if recovery_mbytes: return int(recovery_mbytes[0] * 1.7) def _accumulate_volume_size(self, root_mbytes): """ Calculate number of mbytes to add to the disk to allow the creaton of the volumes with their configured size """ disk_volume_mbytes = 0 data_volume_mbytes = self._calculate_volume_mbytes() root_volume = self._get_root_volume_configuration() for volume in self.volumes: if volume.realpath and not volume.realpath == '/' and volume.size: [size_type, req_size] = volume.size.split(':') disk_add_mbytes = 0 if size_type == 'freespace': disk_add_mbytes += int(req_size) else: disk_add_mbytes += int(req_size) - \ data_volume_mbytes.volume[volume.realpath] if disk_add_mbytes > 0: disk_volume_mbytes += disk_add_mbytes + \ Defaults.get_min_volume_mbytes() else: log.warning( 'volume size of %s MB for %s is too small, skipped', int(req_size), volume.realpath) if root_volume: if root_volume.size_type == 'freespace': disk_add_mbytes = root_volume.req_size else: disk_add_mbytes = root_volume.req_size - \ root_mbytes + data_volume_mbytes.total if disk_add_mbytes > 0: disk_volume_mbytes += disk_add_mbytes + \ Defaults.get_min_volume_mbytes() else: log.warning('root volume size of %s MB is too small, skipped', root_volume.req_size) return disk_volume_mbytes def _get_root_volume_configuration(self): """ Provide LVRoot volume configuration if present and in use according to the selected volume management. So far this only affects the LVM volume manager """ root_volume_type = namedtuple('root_volume_type', ['size_type', 'req_size']) for volume in self.volumes: if volume.name == 'LVRoot': if volume.size: [size_type, req_size] = volume.size.split(':') return root_volume_type(size_type=size_type, req_size=int(req_size)) def _calculate_volume_mbytes(self): """ Calculate the number of mbytes each volume path currently consumes and also provide a total number of these values """ volume_mbytes_type = namedtuple('volume_mbytes_type', ['volume', 'total']) volume_mbytes = {} volume_total = 0 for volume in self.volumes: if volume.realpath and not volume.realpath == '/': path_to_volume = self.root_dir + '/' + volume.realpath if os.path.exists(path_to_volume): volume_size = SystemSize(path_to_volume) volume_mbytes[volume.realpath] = volume_size.customize( volume_size.accumulate_mbyte_file_sizes(), self.filesystem) volume_total += volume_mbytes[volume.realpath] return volume_mbytes_type(volume=volume_mbytes, total=volume_total)
class FileSystemSetup(object): """ **Implement filesystem setup methods** Methods from this class provides information from the root directory required before building a filesystem image :param object xml_state: Instance of XMLState :param string root_dir: root directory path """ def __init__(self, xml_state, root_dir): self.configured_size = xml_state.get_build_type_size( include_unpartitioned=True ) if xml_state.get_build_type_unpartitioned_bytes() > 0: log.warning( 'Unpartitoned size attribute is ignored for filesystem images' ) self.size = SystemSize(root_dir) self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type in Defaults.get_filesystem_image_types(): self.requested_filesystem = self.requested_image_type else: self.requested_filesystem = xml_state.build_type.get_filesystem() def get_size_mbytes(self, filesystem=None): """ Precalculate the requires size in mbytes to store all data from the root directory in the requested filesystem. Return the configured value if present, if not return the calculated result :param string filesystem: name :return: mbytes :rtype: int """ root_dir_mbytes = self.size.accumulate_mbyte_file_sizes() filesystem_mbytes = self.size.customize( root_dir_mbytes, self.requested_filesystem or filesystem ) if not self.configured_size: log.info( 'Using calculated size: %d MB', filesystem_mbytes ) return filesystem_mbytes elif self.configured_size.additive: result_filesystem_mbytes = \ self.configured_size.mbytes + filesystem_mbytes log.info( 'Using configured size: %d MB + %d MB calculated = %d MB', self.configured_size.mbytes, filesystem_mbytes, result_filesystem_mbytes ) return result_filesystem_mbytes else: log.info( 'Using configured size: %d MB', self.configured_size.mbytes ) if self.configured_size.mbytes < filesystem_mbytes: log.warning( '--> Configured size smaller than calculated size: %d MB', filesystem_mbytes ) return self.configured_size.mbytes
class DiskSetup: """ **Implements disk setup methods** Methods from this class provides information required before building a disk image :param object xml_state: Instance of XMLState :param string root_dir: root directory path name """ def __init__(self, xml_state: XMLState, root_dir: str): self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot( ) self.swap_mbytes = xml_state.get_oemconfig_swap_mbytes() self.configured_size = xml_state.get_build_type_size() self.disk_resize_requested = xml_state.get_oemconfig_oem_resize() self.filesystem = xml_state.build_type.get_filesystem() self.bootpart_requested = xml_state.build_type.get_bootpartition() self.bootpart_mbytes = xml_state.build_type.get_bootpartsize() self.spare_part_mbytes = xml_state.get_build_type_spare_part_size() self.mdraid = xml_state.build_type.get_mdraid() self.luks = xml_state.build_type.get_luks() self.volume_manager = xml_state.get_volume_management() self.bootloader = xml_state.get_build_type_bootloader_name() self.oemconfig = xml_state.get_build_type_oemconfig_section() self.volumes = xml_state.get_volumes() self.custom_partitions = xml_state.get_partitions() self.firmware = FirmWare(xml_state) self.rootsize = SystemSize(root_dir) self.root_dir = root_dir self.xml_state = xml_state def get_disksize_mbytes(self) -> int: """ Precalculate disk size requirements in mbytes :return: disk size mbytes :rtype: int """ log.info('Precalculating required disk size') calculated_disk_mbytes = 0 root_filesystem_mbytes = self.rootsize.customize( self.rootsize.accumulate_mbyte_file_sizes(), self.filesystem) calculated_disk_mbytes += root_filesystem_mbytes log.info('--> system data with filesystem overhead needs %s MB', root_filesystem_mbytes) if self.custom_partitions: partition_mbytes = self._accumulate_partitions_size() if partition_mbytes: calculated_disk_mbytes += partition_mbytes log.info('--> partition(s) size setup adding %s MB', partition_mbytes) if self.volume_manager and self.volume_manager == 'lvm': lvm_overhead_mbytes = Defaults.get_lvm_overhead_mbytes() log.info('--> LVM overhead adding %s MB', lvm_overhead_mbytes) calculated_disk_mbytes += lvm_overhead_mbytes volume_mbytes = self._accumulate_volume_size( root_filesystem_mbytes) if volume_mbytes: calculated_disk_mbytes += volume_mbytes log.info('--> volume(s) size setup adding %s MB', volume_mbytes) elif self.swap_mbytes: calculated_disk_mbytes += self.swap_mbytes log.info('--> swap partition adding %s MB', self.swap_mbytes) legacy_bios_mbytes = self.firmware.get_legacy_bios_partition_size() if legacy_bios_mbytes: calculated_disk_mbytes += legacy_bios_mbytes log.info('--> legacy bios boot partition adding %s MB', legacy_bios_mbytes) boot_mbytes = self.boot_partition_size() if boot_mbytes: calculated_disk_mbytes += boot_mbytes log.info('--> boot partition adding %s MB', boot_mbytes) if self.spare_part_mbytes: calculated_disk_mbytes += self.spare_part_mbytes log.info('--> spare partition adding %s MB', self.spare_part_mbytes) efi_mbytes = self.firmware.get_efi_partition_size() if efi_mbytes: calculated_disk_mbytes += efi_mbytes log.info('--> EFI partition adding %s MB', efi_mbytes) prep_mbytes = self.firmware.get_prep_partition_size() if prep_mbytes: calculated_disk_mbytes += prep_mbytes log.info('--> PReP partition adding %s MB', prep_mbytes) recovery_mbytes = self._inplace_recovery_partition_size() if recovery_mbytes: calculated_disk_mbytes += recovery_mbytes log.info('--> In-place recovery partition adding: %s MB', recovery_mbytes) if not self.configured_size: log.info('Using calculated disk size: %d MB', calculated_disk_mbytes) return calculated_disk_mbytes elif self.configured_size.additive: result_disk_mbytes = \ self.configured_size.mbytes + calculated_disk_mbytes log.info( 'Using configured disk size: %d MB + %d MB calculated = %d MB', self.configured_size.mbytes, calculated_disk_mbytes, result_disk_mbytes) return result_disk_mbytes else: log.info('Using configured disk size: %d MB', self.configured_size.mbytes) if self.configured_size.mbytes < calculated_disk_mbytes: log.warning( '--> Configured size smaller than calculated size: %d MB', calculated_disk_mbytes) return self.configured_size.mbytes def need_boot_partition(self): """ Decide if an extra boot partition is needed. This is done with the bootpartition attribute from the type, however if it is not set it depends on some other type configuration parameters if we need a boot partition or not :return: True or False :rtype: bool """ if self.bootpart_requested is True: return True if self.bootpart_requested is False: return False if self.mdraid: return True if self.volume_manager == 'lvm': return True if self.volume_manager == 'btrfs': return False if self.root_filesystem_is_overlay: return True return False @staticmethod def get_boot_label() -> str: """ Filesystem Label to use for the boot partition :return: label name :rtype: str """ return 'BOOT' def get_root_label(self) -> str: """ Filesystem Label to use for the root partition If not specified in the XML configuration the default root label is set to 'ROOT' :return: label name :rtype: str """ root_label = self.xml_state.build_type.get_rootfs_label() if not root_label: root_label = 'ROOT' return root_label @staticmethod def get_efi_label() -> str: """ Filesystem Label to use for the EFI partition :return: label name :rtype: str """ return 'EFI' def boot_partition_size(self) -> int: """ Size of the boot partition in mbytes :return: boot size mbytes :rtype: int """ if self.need_boot_partition(): if self.bootpart_mbytes: return self.bootpart_mbytes return Defaults.get_default_boot_mbytes() return 0 def _inplace_recovery_partition_size(self): """ In inplace recovery mode the recovery archive is created at install time. This requires free space on the disk. The amount of free space is specified with the oem-recovery-part-size attribute. If specified we add the given size to the disk. If not specified an inplace setup at install time will be moved to the first boot of an oem image when the recovery partition has been created """ if self.oemconfig and self.oemconfig.get_oem_inplace_recovery(): recovery_mbytes = self.oemconfig.get_oem_recovery_part_size() if recovery_mbytes: return int(recovery_mbytes[0] * 1.7) def _accumulate_partitions_size(self): """ Calculate number of mbytes to add to the disk to allow the creaton of the partitions with their configured size """ disk_partition_mbytes = 0 data_partition_mbytes = self._calculate_partition_mbytes() for map_name in sorted(self.custom_partitions.keys()): partition_mount_path = self.custom_partitions[map_name].mountpoint partition_mbsize = self.custom_partitions[map_name].mbsize disk_add_mbytes = int(partition_mbsize) - \ data_partition_mbytes.partition[partition_mount_path] if disk_add_mbytes > 0: disk_partition_mbytes += disk_add_mbytes else: message = dedent('''\n Requested partition size {0}MB for {1!r} is too small The minimum byte value to store the data below the {1!r} path was calculated to be {2}MB ''') raise KiwiPartitionTooSmallError( message.format( partition_mbsize, partition_mount_path, data_partition_mbytes.partition[partition_mount_path])) return disk_partition_mbytes def _accumulate_volume_size(self, root_mbytes): """ Calculate number of mbytes to add to the disk to allow the creaton of the volumes with their configured size """ disk_volume_mbytes = 0 data_volume_mbytes = self._calculate_volume_mbytes() root_volume = self._get_root_volume_configuration() # If disk resize is requested we only add the default min # volume size because their target size request is handled # on first boot of the disk image in oemboot/repart if self.disk_resize_requested: for volume in self.volumes: disk_volume_mbytes += Defaults.get_min_volume_mbytes() return disk_volume_mbytes # For static disk(no resize requested) we need to add the # configured volume sizes because the image is used directly # as it is without being deployed and resized on a target disk for volume in self.volumes: if volume.realpath and not volume.realpath == '/' and volume.size: [size_type, req_size] = volume.size.split(':') disk_add_mbytes = 0 if size_type == 'freespace': disk_add_mbytes += int(req_size) else: disk_add_mbytes += int(req_size) - \ data_volume_mbytes.volume[volume.realpath] if disk_add_mbytes > 0: disk_volume_mbytes += disk_add_mbytes + \ Defaults.get_min_volume_mbytes() else: message = dedent('''\n Requested volume size {0}MB for {1!r} is too small The minimum byte value to store the data below the {1!r} path was calculated to be {2}MB ''') raise KiwiVolumeTooSmallError( message.format( req_size, volume.realpath, data_volume_mbytes.volume[volume.realpath])) if root_volume: if root_volume.size_type == 'freespace': disk_add_mbytes = root_volume.req_size else: disk_add_mbytes = root_volume.req_size - \ root_mbytes + data_volume_mbytes.total if disk_add_mbytes > 0: disk_volume_mbytes += disk_add_mbytes + \ Defaults.get_min_volume_mbytes() else: log.warning('root volume size of %s MB is too small, skipped', root_volume.req_size) return disk_volume_mbytes def _get_root_volume_configuration(self): """ Provide root volume configuration if present and in use according to the selected volume management. So far this only affects the LVM volume manager """ root_volume_type = namedtuple('root_volume_type', ['size_type', 'req_size']) for volume in self.volumes: if volume.is_root_volume: if volume.size: [size_type, req_size] = volume.size.split(':') return root_volume_type(size_type=size_type, req_size=int(req_size)) def _calculate_volume_mbytes(self): """ Calculate the number of mbytes each volume path currently consumes and also provide a total number of these values. """ volume_mbytes_type = namedtuple('volume_mbytes_type', ['volume', 'total']) volume_mbytes = {} volume_total = 0 for volume in self.volumes: if volume.realpath and not volume.realpath == '/': path_to_volume = self.root_dir + '/' + volume.realpath if os.path.exists(path_to_volume): volume_size = SystemSize(path_to_volume) volume_mbytes[volume.realpath] = volume_size.customize( volume_size.accumulate_mbyte_file_sizes(), self.filesystem) else: volume_mbytes[volume.realpath] = 0 volume_total += volume_mbytes[volume.realpath] return volume_mbytes_type(volume=volume_mbytes, total=volume_total) def _calculate_partition_mbytes(self): """ Calculate the number of mbytes each partition path consumes """ partition_mbytes_type = namedtuple('partition_mbytes_type', ['partition']) partition_mbytes = {} for map_name in sorted(self.custom_partitions.keys()): partition_mount_path = self.custom_partitions[map_name].mountpoint partition_filesystem = self.custom_partitions[map_name].filesystem path_to_partition = os.path.normpath( os.sep.join([self.root_dir, partition_mount_path])) if os.path.exists(path_to_partition): partition_size = SystemSize(path_to_partition) partition_mbytes[ partition_mount_path] = partition_size.customize( partition_size.accumulate_mbyte_file_sizes(), partition_filesystem) else: partition_mbytes[partition_mount_path] = 0 return partition_mbytes_type(partition=partition_mbytes)
class FileSystemSetup(object): """ Implement filesystem setup methods providing information from the root directory required before building a filesystem image Attributes * :attr:`configured_size` Configured size section from the build type section * :attr:`size` Instance of Size * :attr:`requested_image_type` Configured image type name * :attr:`requested_filesystem` Configured filesystem name """ def __init__(self, xml_state, root_dir): self.configured_size = xml_state.get_build_type_size() self.size = SystemSize(root_dir) self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type in Defaults.get_filesystem_image_types(): self.requested_filesystem = self.requested_image_type else: self.requested_filesystem = xml_state.build_type.get_filesystem() def get_size_mbytes(self, filesystem=None): """ Precalculate the requires size in mbytes to store all data from the root directory in the requested filesystem. Return the configured value if present, if not return the calculated result :return: mbytes :rtype: int """ root_dir_mbytes = self.size.accumulate_mbyte_file_sizes() filesystem_mbytes = self.size.customize( root_dir_mbytes, self.requested_filesystem or filesystem) if not self.configured_size: log.info('Using calculated size: %d MB', filesystem_mbytes) return filesystem_mbytes elif self.configured_size.additive: result_filesystem_mbytes = \ self.configured_size.mbytes + filesystem_mbytes log.info('Using configured size: %d MB + %d MB calculated = %d MB', self.configured_size.mbytes, filesystem_mbytes, result_filesystem_mbytes) return result_filesystem_mbytes else: log.info('Using configured size: %d MB', self.configured_size.mbytes) if self.configured_size.mbytes < filesystem_mbytes: log.warning( '--> Configured size smaller than calculated size: %d MB', filesystem_mbytes) return self.configured_size.mbytes
class DiskSetup(object): """ **Implements disk setup methods** Methods from this class provides information required before building a disk image :param object xml_state: Instance of XMLState :param string root_dir: root directory path name """ def __init__(self, xml_state, root_dir): self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot() self.configured_size = xml_state.get_build_type_size() self.build_type_name = xml_state.get_build_type_name() self.filesystem = xml_state.build_type.get_filesystem() self.bootpart_requested = xml_state.build_type.get_bootpartition() self.bootpart_mbytes = xml_state.build_type.get_bootpartsize() self.spare_part_mbytes = xml_state.get_build_type_spare_part_size() self.mdraid = xml_state.build_type.get_mdraid() self.luks = xml_state.build_type.get_luks() self.volume_manager = xml_state.get_volume_management() self.bootloader = xml_state.build_type.get_bootloader() self.oemconfig = xml_state.get_build_type_oemconfig_section() self.volumes = xml_state.get_volumes() self.firmware = FirmWare( xml_state ) self.rootsize = SystemSize( root_dir ) self.root_dir = root_dir self.xml_state = xml_state def get_disksize_mbytes(self): """ Precalculate disk size requirements in mbytes :return: disk size mbytes :rtype: int """ log.info('Precalculating required disk size') calculated_disk_mbytes = 0 root_filesystem_mbytes = self.rootsize.customize( self.rootsize.accumulate_mbyte_file_sizes(), self.filesystem ) calculated_disk_mbytes += root_filesystem_mbytes log.info( '--> system data with filesystem overhead needs %s MB', root_filesystem_mbytes ) if self.volume_manager and self.volume_manager == 'lvm': lvm_overhead_mbytes = Defaults.get_lvm_overhead_mbytes() log.info( '--> LVM overhead adding %s MB', lvm_overhead_mbytes ) calculated_disk_mbytes += lvm_overhead_mbytes volume_mbytes = self._accumulate_volume_size( root_filesystem_mbytes ) if volume_mbytes: calculated_disk_mbytes += volume_mbytes log.info( '--> volume(s) size setup adding %s MB', volume_mbytes ) legacy_bios_mbytes = self.firmware.get_legacy_bios_partition_size() if legacy_bios_mbytes: calculated_disk_mbytes += legacy_bios_mbytes log.info( '--> legacy bios boot partition adding %s MB', legacy_bios_mbytes ) boot_mbytes = self.boot_partition_size() if boot_mbytes: calculated_disk_mbytes += boot_mbytes log.info( '--> boot partition adding %s MB', boot_mbytes ) if self.spare_part_mbytes: calculated_disk_mbytes += self.spare_part_mbytes log.info( '--> spare partition adding %s MB', self.spare_part_mbytes ) efi_mbytes = self.firmware.get_efi_partition_size() if efi_mbytes: calculated_disk_mbytes += efi_mbytes log.info( '--> EFI partition adding %s MB', efi_mbytes ) prep_mbytes = self.firmware.get_prep_partition_size() if prep_mbytes: calculated_disk_mbytes += prep_mbytes log.info( '--> PReP partition adding %s MB', prep_mbytes ) recovery_mbytes = self._inplace_recovery_partition_size() if recovery_mbytes: calculated_disk_mbytes += recovery_mbytes log.info( '--> In-place recovery partition adding: %s MB', recovery_mbytes ) if not self.configured_size: log.info( 'Using calculated disk size: %d MB', calculated_disk_mbytes ) return calculated_disk_mbytes elif self.configured_size.additive: result_disk_mbytes = \ self.configured_size.mbytes + calculated_disk_mbytes log.info( 'Using configured disk size: %d MB + %d MB calculated = %d MB', self.configured_size.mbytes, calculated_disk_mbytes, result_disk_mbytes ) return result_disk_mbytes else: log.info( 'Using configured disk size: %d MB', self.configured_size.mbytes ) if self.configured_size.mbytes < calculated_disk_mbytes: log.warning( '--> Configured size smaller than calculated size: %d MB', calculated_disk_mbytes ) return self.configured_size.mbytes def need_boot_partition(self): """ Decide if an extra boot partition is needed. This is done with the bootpartition attribute from the type, however if it is not set it depends on some other type configuration parameters if we need a boot partition or not :return: True or False :rtype: bool """ if self.bootpart_requested is True: return True if self.bootpart_requested is False: return False if self.mdraid: return True if self.volume_manager: return True if self.filesystem == 'btrfs': return True if self.filesystem == 'xfs': return True if self.root_filesystem_is_overlay: return True if self.bootloader == 'grub2_s390x_emu': return True if self.luks: return True def get_boot_label(self): """ Filesystem Label to use for the boot partition :return: label name :rtype: str """ label = 'BOOT' if self.bootloader == 'grub2_s390x_emu': label = 'ZIPL' return label def get_root_label(self): """ Filesystem Label to use for the root partition If not specified in the XML configuration the default root label is set to 'ROOT' :return: label name :rtype: str """ root_label = self.xml_state.build_type.get_rootfs_label() if not root_label: root_label = 'ROOT' return root_label def get_efi_label(self): """ Filesystem Label to use for the EFI partition :return: label name :rtype: str """ return 'EFI' def boot_partition_size(self): """ Size of the boot partition in mbytes :return: boot size mbytes :rtype: int """ if self.need_boot_partition(): if self.bootpart_mbytes: return self.bootpart_mbytes else: return Defaults.get_default_boot_mbytes() def _inplace_recovery_partition_size(self): """ In inplace recovery mode the recovery archive is created at install time. This requires free space on the disk. The amount of free space is specified with the oem-recovery-part-size attribute. If specified we add the given size to the disk. If not specified an inplace setup at install time will be moved to the first boot of an oem image when the recovery partition has been created """ if self.oemconfig and self.oemconfig.get_oem_inplace_recovery(): recovery_mbytes = self.oemconfig.get_oem_recovery_part_size() if recovery_mbytes: return int(recovery_mbytes[0] * 1.7) def _accumulate_volume_size(self, root_mbytes): """ Calculate number of mbytes to add to the disk to allow the creaton of the volumes with their configured size """ disk_volume_mbytes = 0 data_volume_mbytes = self._calculate_volume_mbytes() root_volume = self._get_root_volume_configuration() # For oem types we only add the default min volume size # because their target size request is handled on first boot # of the disk image in oemboot/repart if self.build_type_name == 'oem': for volume in self.volumes: disk_volume_mbytes += Defaults.get_min_volume_mbytes() return disk_volume_mbytes # For vmx types we need to add the configured volume # sizes because the image is used directly as it is without # being deployed and resized on a target disk for volume in self.volumes: if volume.realpath and not volume.realpath == '/' and volume.size: [size_type, req_size] = volume.size.split(':') disk_add_mbytes = 0 if size_type == 'freespace': disk_add_mbytes += int(req_size) else: disk_add_mbytes += int(req_size) - \ data_volume_mbytes.volume[volume.realpath] if disk_add_mbytes > 0: disk_volume_mbytes += disk_add_mbytes + \ Defaults.get_min_volume_mbytes() else: log.warning( 'volume size of %s MB for %s is too small, skipped', int(req_size), volume.realpath ) if root_volume: if root_volume.size_type == 'freespace': disk_add_mbytes = root_volume.req_size else: disk_add_mbytes = root_volume.req_size - \ root_mbytes + data_volume_mbytes.total if disk_add_mbytes > 0: disk_volume_mbytes += disk_add_mbytes + \ Defaults.get_min_volume_mbytes() else: log.warning( 'root volume size of %s MB is too small, skipped', root_volume.req_size ) return disk_volume_mbytes def _get_root_volume_configuration(self): """ Provide LVRoot volume configuration if present and in use according to the selected volume management. So far this only affects the LVM volume manager """ root_volume_type = namedtuple( 'root_volume_type', ['size_type', 'req_size'] ) for volume in self.volumes: if volume.name == 'LVRoot': if volume.size: [size_type, req_size] = volume.size.split(':') return root_volume_type( size_type=size_type, req_size=int(req_size) ) def _calculate_volume_mbytes(self): """ Calculate the number of mbytes each volume path currently consumes and also provide a total number of these values """ volume_mbytes_type = namedtuple( 'volume_mbytes_type', ['volume', 'total'] ) volume_mbytes = {} volume_total = 0 for volume in self.volumes: if volume.realpath and not volume.realpath == '/': path_to_volume = self.root_dir + '/' + volume.realpath if os.path.exists(path_to_volume): volume_size = SystemSize(path_to_volume) volume_mbytes[volume.realpath] = volume_size.customize( volume_size.accumulate_mbyte_file_sizes(), self.filesystem ) volume_total += volume_mbytes[volume.realpath] return volume_mbytes_type( volume=volume_mbytes, total=volume_total )
class FileSystemSetup(object): """ Implement filesystem setup methods providing information from the root directory required before building a filesystem image Attributes * :attr:`configured_size` Configured size section from the build type section * :attr:`size` Instance of Size * :attr:`requested_image_type` Configured image type name * :attr:`requested_filesystem` Configured filesystem name """ def __init__(self, xml_state, root_dir): self.configured_size = xml_state.get_build_type_size() self.size = SystemSize(root_dir) self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type in Defaults.get_filesystem_image_types(): self.requested_filesystem = self.requested_image_type else: self.requested_filesystem = xml_state.build_type.get_filesystem() def get_size_mbytes(self, filesystem=None): """ Precalculate the requires size in mbytes to store all data from the root directory in the requested filesystem. Return the configured value if present, if not return the calculated result :return: mbytes :rtype: int """ root_dir_mbytes = self.size.accumulate_mbyte_file_sizes() filesystem_mbytes = self.size.customize( root_dir_mbytes, self.requested_filesystem or filesystem ) if not self.configured_size: log.info( 'Using calculated size: %d MB', filesystem_mbytes ) return filesystem_mbytes elif self.configured_size.additive: result_filesystem_mbytes = \ self.configured_size.mbytes + filesystem_mbytes log.info( 'Using configured size: %d MB + %d MB calculated = %d MB', self.configured_size.mbytes, filesystem_mbytes, result_filesystem_mbytes ) return result_filesystem_mbytes else: log.info( 'Using configured size: %d MB', self.configured_size.mbytes ) if self.configured_size.mbytes < filesystem_mbytes: log.warning( '--> Configured size smaller than calculated size: %d MB', filesystem_mbytes ) return self.configured_size.mbytes
def _get_container_filesystem_size_mbytes(self): size = SystemSize(self.root_dir) root_dir_mbytes = size.accumulate_mbyte_file_sizes() return size.customize(root_dir_mbytes, 'ext4')