def test_get_disksize_mbytes_with_spare_partition(self): configured_spare_part_size = 42 assert self.setup_arm.get_disksize_mbytes() == \ configured_spare_part_size + \ Defaults.get_default_efi_boot_mbytes() + \ Defaults.get_default_boot_mbytes() + \ self.size.accumulate_mbyte_file_sizes.return_value
def test_get_disksize_mbytes_empty_volumes(self): assert self.setup_empty_volumes.get_disksize_mbytes() == \ Defaults.get_lvm_overhead_mbytes() + \ Defaults.get_default_legacy_bios_mbytes() + \ Defaults.get_default_efi_boot_mbytes() + \ Defaults.get_default_boot_mbytes() + \ self.size.accumulate_mbyte_file_sizes.return_value
def test_get_live_dracut_module_from_flag(self): assert Defaults.get_live_dracut_module_from_flag('foo') == \ 'kiwi-live' assert Defaults.get_live_dracut_module_from_flag('overlay') == \ 'kiwi-live' assert Defaults.get_live_dracut_module_from_flag('dmsquash') == \ 'dmsquash-live'
def _setup_secure_boot_efi_image(self, lookup_path): """ Provide the shim loader and the shim signed grub2 loader in the required boot path. Normally this task is done by the shim-install tool. However, shim-install does not exist on all distributions and the script does not operate well in e.g CD environments from which we generate live and/or install media. Thus shim-install is used if possible at install time of the bootloader because it requires access to the target block device. In any other case this setup code should act as the fallback solution """ log.warning( '--> Running fallback setup for shim secure boot efi image' ) if not lookup_path: lookup_path = self.root_dir shim_image = Defaults.get_shim_loader(lookup_path) if not shim_image: raise KiwiBootLoaderGrubSecureBootError( 'Microsoft signed shim loader not found' ) grub_image = Defaults.get_signed_grub_loader(lookup_path) if not grub_image: raise KiwiBootLoaderGrubSecureBootError( 'Shim signed grub2 efi loader not found' ) Command.run( ['cp', shim_image, self._get_efi_image_name()] ) Command.run( ['cp', grub_image, self.efi_boot_path] )
def get_gfxmode(self, target): """ Graphics mode according to bootloader target Bootloaders which support a graphics mode can be configured to run graphics in a specific resolution and colors. There is no standard for this setup which causes kiwi to create a mapping from the kernel vesa mode number to the corresponding bootloader graphics mode setup :param string target: bootloader name :return: boot graphics mode :rtype: str """ gfxmode_map = Defaults.get_video_mode_map() default_mode = Defaults.get_default_video_mode() requested_gfxmode = self.xml_state.build_type.get_vga() if requested_gfxmode in gfxmode_map: gfxmode = requested_gfxmode else: gfxmode = default_mode if target == 'grub2': return gfxmode_map[gfxmode].grub2 elif target == 'isolinux': return gfxmode_map[gfxmode].isolinux else: return gfxmode
def translate(self, check_build_environment=True): """ Translate repository location according to their URI type Depending on the URI type the provided location needs to be adapted e.g loop mounted in case of an ISO or updated by the service URL in case of an open buildservice project name :raises KiwiUriStyleUnknown: if the uri scheme can't be detected, is unknown or it is inconsistent with the build environment :param bool check_build_environment: specify if the uri translation should depend on the environment the build is called in. As of today this only effects the translation result if the image build happens inside of the Open Build Service :rtype: str """ uri = urlparse(self.uri) if not uri.scheme: raise KiwiUriStyleUnknown( 'URI scheme not detected {uri}'.format(uri=self.uri) ) elif uri.scheme == 'obs': if check_build_environment and Defaults.is_buildservice_worker(): return self._buildservice_path( name=''.join([uri.netloc, uri.path]).replace(':/', ':'), fragment=uri.fragment, urischeme=uri.scheme ) else: return self._obs_project_download_link( ''.join([uri.netloc, uri.path]).replace(':/', ':') ) elif uri.scheme == 'obsrepositories': if not Defaults.is_buildservice_worker(): raise KiwiUriStyleUnknown( 'Only the buildservice can use the {0} schema'.format( uri.scheme ) ) return self._buildservice_path( name=''.join([uri.netloc, uri.path]).replace(':/', ':'), fragment=uri.fragment, urischeme=uri.scheme ) elif uri.scheme == 'dir': return self._local_path(uri.path) elif uri.scheme == 'file': return self._local_path(uri.path) elif uri.scheme == 'iso': return self._iso_mount_path(uri.path) elif uri.scheme.startswith('http') or uri.scheme == 'ftp': return ''.join([uri.scheme, '://', uri.netloc, uri.path]) else: raise KiwiUriStyleUnknown( 'URI schema %s not supported' % self.uri )
def test_get_grub_boot_directory_name(self, mock_which): mock_which.return_value = 'grub2-install-was-found' assert Defaults.get_grub_boot_directory_name( lookup_path='lookup_path' ) == 'grub2' mock_which.return_value = None assert Defaults.get_grub_boot_directory_name( lookup_path='lookup_path' ) == 'grub'
def test_get_disksize_mbytes_root_volume(self, mock_log_warn, mock_exists): mock_exists.return_value = True root_size = self.size.accumulate_mbyte_file_sizes.return_value assert self.setup_root_volume.get_disksize_mbytes() == \ Defaults.get_default_legacy_bios_mbytes() + \ Defaults.get_default_efi_boot_mbytes() + \ Defaults.get_default_boot_mbytes() + \ root_size assert mock_log_warn.called
def _accumulate_volume_size(self, root_mbytes): """ Calculate number of mbytes to add to the disk to allow the creaton of the volumes with their configured size """ disk_volume_mbytes = 0 data_volume_mbytes = self._calculate_volume_mbytes() root_volume = self._get_root_volume_configuration() # For oem types we only add the default min volume size # because their target size request is handled on first boot # of the disk image in oemboot/repart if self.build_type_name == 'oem': for volume in self.volumes: disk_volume_mbytes += Defaults.get_min_volume_mbytes() return disk_volume_mbytes # For vmx types we need to add the configured volume # sizes because the image is used directly as it is without # being deployed and resized on a target disk for volume in self.volumes: if volume.realpath and not volume.realpath == '/' and volume.size: [size_type, req_size] = volume.size.split(':') disk_add_mbytes = 0 if size_type == 'freespace': disk_add_mbytes += int(req_size) else: disk_add_mbytes += int(req_size) - \ data_volume_mbytes.volume[volume.realpath] if disk_add_mbytes > 0: disk_volume_mbytes += disk_add_mbytes + \ Defaults.get_min_volume_mbytes() else: log.warning( 'volume size of %s MB for %s is too small, skipped', int(req_size), volume.realpath ) if root_volume: if root_volume.size_type == 'freespace': disk_add_mbytes = root_volume.req_size else: disk_add_mbytes = root_volume.req_size - \ root_mbytes + data_volume_mbytes.total if disk_add_mbytes > 0: disk_volume_mbytes += disk_add_mbytes + \ Defaults.get_min_volume_mbytes() else: log.warning( 'root volume size of %s MB is too small, skipped', root_volume.req_size ) return disk_volume_mbytes
def test_get_disksize_mbytes_oem_volumes(self, mock_exists): mock_exists.return_value = True root_size = self.size.accumulate_mbyte_file_sizes.return_value assert self.setup_oem_volumes.get_disksize_mbytes() == \ Defaults.get_lvm_overhead_mbytes() + \ Defaults.get_default_legacy_bios_mbytes() + \ Defaults.get_default_efi_boot_mbytes() + \ Defaults.get_default_boot_mbytes() + \ root_size + \ 5 * Defaults.get_min_volume_mbytes()
class TestDefaults(object): def setup(self): self.defaults = Defaults() def test_get(self): assert self.defaults.get('kiwi_align') == 1048576 assert self.defaults.get('kiwi_startsector') == 2048 assert self.defaults.get('kiwi_sectorsize') == 512 assert self.defaults.get('kiwi_inode_size') == 256 assert self.defaults.get('kiwi_inode_ratio') == 16384 assert self.defaults.get('kiwi_min_inodes') == 20000 assert self.defaults.get('kiwi_revision') def test_to_profile(self): profile = mock.MagicMock() self.defaults.to_profile(profile) profile.add.assert_any_call('kiwi_align', 1048576) profile.add.assert_any_call('kiwi_startsector', 2048) profile.add.assert_any_call('kiwi_sectorsize', 512) profile.add.assert_any_call( 'kiwi_revision', self.defaults.get('kiwi_revision') ) def test_get_preparer(self): assert Defaults.get_preparer() == 'KIWI - http://suse.github.com/kiwi' def test_get_publisher(self): assert Defaults.get_publisher() == 'SUSE LINUX GmbH'
def test_get_disksize_mbytes_configured_additive(self): self.setup.configured_size = mock.Mock() self.setup.build_type_name = 'vmx' self.setup.configured_size.additive = True self.setup.configured_size.mbytes = 42 root_size = self.size.accumulate_mbyte_file_sizes.return_value assert self.setup.get_disksize_mbytes() == \ Defaults.get_default_legacy_bios_mbytes() + \ Defaults.get_default_efi_boot_mbytes() + \ Defaults.get_default_boot_mbytes() + \ root_size + 42 + \ 200 * 1.7
def prepare(self): """ Prepare kiwi profile environment to be included in dracut initrd """ profile = Profile(self.xml_state) defaults = Defaults() defaults.to_profile(profile) setup = SystemSetup( self.xml_state, self.boot_root_directory ) setup.import_shell_environment(profile) self.dracut_options.append('--install') self.dracut_options.append('/.profile')
def test_create_volumes( self, mock_mount, mock_mapped_device, mock_fs, mock_command, mock_size, mock_os_exists ): mock_mapped_device.return_value = 'mapped_device' size = mock.Mock() size.customize = mock.Mock( return_value=42 ) mock_size.return_value = size mock_os_exists.return_value = False self.volume_manager.volume_group = 'volume_group' self.volume_manager.create_volumes('ext3') myvol_size = 500 etc_size = 200 + 42 + Defaults.get_min_volume_mbytes() root_size = 100 + 42 + Defaults.get_min_volume_mbytes() assert mock_mount.call_args_list == [ call(device='/dev/volume_group/LVRoot', mountpoint='tmpdir'), call(device='/dev/volume_group/myvol', mountpoint='tmpdir//data'), call(device='/dev/volume_group/LVetc', mountpoint='tmpdir//etc'), call(device='/dev/volume_group/LVhome', mountpoint='tmpdir//home') ] assert mock_command.call_args_list == [ call(['mkdir', '-p', 'root_dir/etc']), call(['mkdir', '-p', 'root_dir/data']), call(['mkdir', '-p', 'root_dir/home']), call([ 'lvcreate', '-L', format(root_size), '-n', 'LVRoot', 'volume_group' ]), call([ 'lvcreate', '-L', format(myvol_size), '-n', 'myvol', 'volume_group' ]), call([ 'lvcreate', '-L', format(etc_size), '-n', 'LVetc', 'volume_group' ]), call([ 'lvcreate', '-l', '+100%FREE', '-n', 'LVhome', 'volume_group' ]) ] assert mock_fs.call_args_list == [ call('ext3', 'mapped_device'), call('ext3', 'mapped_device'), call('ext3', 'mapped_device'), call('ext3', 'mapped_device') ] self.volume_manager.volume_group = None
def create_xz_compressed( self, source_dir, exclude=None, options=None, xz_options=None ): """ Create XZ compressed tar archive :param string source_dir: data source directory :param list exclude: list of excluded items :param list options: custom tar creation options :param list xz_options: custom xz compression options """ if not options: options = [] if not xz_options: xz_options = Defaults.get_xz_compression_options() bash_command = [ 'tar', '-C', source_dir ] + options + self.xattrs_options + [ '-c', '--to-stdout' ] + self._get_archive_items(source_dir, exclude) + [ '|', 'xz', '-f' ] + xz_options + [ '>', self.filename + '.xz' ] Command.run(['bash', '-c', ' '.join(bash_command)]) return self.filename + '.xz'
def _systemdisk_to_profile(self): # kiwi_lvmgroup # kiwi_lvm # kiwi_Volume_X systemdisk = self.xml_state.get_build_type_system_disk_section() if systemdisk: volume_manager_name = self.xml_state.get_volume_management() if volume_manager_name == 'lvm': self.dot_profile['kiwi_lvm'] = 'true' self.dot_profile['kiwi_lvmgroup'] = systemdisk.get_name() if not self.dot_profile['kiwi_lvmgroup']: self.dot_profile['kiwi_lvmgroup'] = \ Defaults.get_default_volume_group_name() volume_count = 1 for volume in self.xml_state.get_volumes(): volume_id_name = 'kiwi_Volume_{id}'.format(id=volume_count) self.dot_profile[volume_id_name] = '|'.join( [ volume.name, 'size:all' if volume.fullsize else volume.size, volume.mountpoint or '' ] ) volume_count += 1
def _obs_project_download_link(self, name): name_parts = name.split(os.sep) repository = name_parts.pop() project = os.sep.join(name_parts) try: download_link = os.sep.join( [ self.runtime_config.get_obs_download_server_url(), project.replace(':', ':/'), repository ] ) if not Defaults.is_buildservice_worker(): request = requests.get(download_link) request.raise_for_status() return request.url else: log.warning( 'Using {0} without location verification due to build ' 'in isolated environment'.format(download_link) ) return download_link except Exception as e: raise KiwiUriOpenError( '{0}: {1}'.format(type(e).__name__, format(e)) )
def test_install_ppc_ieee1275( self, mock_grub_path, mock_mount_manager, mock_command ): mock_grub_path.return_value = \ self.root_mount.mountpoint + '/usr/lib/grub2' self.bootloader.arch = 'ppc64' def side_effect(device, mountpoint=None): return self.mount_managers.pop() mock_mount_manager.side_effect = side_effect self.bootloader.install() self.bootloader.root_mount.mount.assert_called_once_with() self.bootloader.boot_mount.mount.assert_called_once_with() mock_command.assert_called_once_with( [ 'chroot', 'tmp_root', 'grub2-install', '--skip-fs-probe', '--no-nvram', '--directory', '/usr/lib/grub2/powerpc-ieee1275', '--boot-directory', '/boot', '--target', 'powerpc-ieee1275', '--modules', ' '.join( Defaults.get_grub_ofw_modules() ), self.custom_args['prep_device'] ] )
def test_install(self, mock_grub_path, mock_mount_manager, mock_command): mock_grub_path.return_value = \ self.root_mount.mountpoint + '/usr/lib/grub2' self.boot_mount.device = self.root_mount.device def side_effect(device, mountpoint=None): return self.mount_managers.pop() mock_mount_manager.side_effect = side_effect self.bootloader.install() self.root_mount.mount.assert_called_once_with() self.volume_mount.mount.assert_called_once_with( options=['subvol=@/boot/grub2'] ) mock_command.assert_called_once_with( [ 'chroot', 'tmp_root', 'grub2-install', '--skip-fs-probe', '--directory', '/usr/lib/grub2/i386-pc', '--boot-directory', '/boot', '--target', 'i386-pc', '--modules', ' '.join( Defaults.get_grub_bios_modules(multiboot=True) ), '/dev/some-device' ])
def sync_data(self): """ Synchronize data from the given base image to the target root directory. """ self.extract_oci_image() Command.run([ 'umoci', 'unpack', '--image', '{0}:base_layer'.format(self.oci_layout_dir), self.oci_unpack_dir ]) synchronizer = DataSync( os.sep.join([self.oci_unpack_dir, 'rootfs', '']), ''.join([self.root_dir, os.sep]) ) synchronizer.sync_data(options=['-a', '-H', '-X', '-A']) # A copy of the uncompressed image and its checksum are # kept inside the root_dir in order to ensure the later steps # i.e. system create are atomic and don't need any third # party archive. image_copy = Defaults.get_imported_root_image(self.root_dir) Path.create(os.path.dirname(image_copy)) image_tar = ArchiveTar(image_copy) image_tar.create(self.oci_layout_dir) self._make_checksum(image_copy)
def export_package_verification(self, target_dir): """ Export package verification result as metadata reference used by the open buildservice :param string target_dir: path name """ filename = ''.join( [ target_dir, '/', self.xml_state.xml_data.get_name(), '.' + self.arch, '-' + self.xml_state.get_image_version(), '.verified' ] ) packager = Defaults.get_default_packager_tool( self.xml_state.get_package_manager() ) if packager == 'rpm': self._export_rpm_package_verification(filename) return filename elif packager == 'dpkg': self._export_deb_package_verification(filename) return filename
def test_import_description( self, mock_iglob, mock_path, mock_open, mock_command ): mock_iglob.return_value = ['config-cdroot.tar.xz'] mock_path.return_value = True self.setup_with_real_xml.import_description() mock_iglob.assert_called_once_with( '../data/config-cdroot.tar*' ) assert mock_command.call_args_list == [ call(['mkdir', '-p', 'root_dir/image']), call(['cp', '../data/config.sh', 'root_dir/image/config.sh']), call([ 'cp', '../data/my_edit_boot_script', 'root_dir/image/edit_boot_config.sh' ]), call([ 'cp', '/absolute/path/to/my_edit_boot_install', 'root_dir/image/edit_boot_install.sh' ]), call(['cp', '../data/images.sh', 'root_dir/image/images.sh']), call([ 'cp', Defaults.project_file('config/functions.sh'), 'root_dir/.kconfig' ]), call(['cp', '/absolute/path/to/image.tgz', 'root_dir/image/']), call(['cp', '../data/bootstrap.tgz', 'root_dir/image/']), call(['cp', 'config-cdroot.tar.xz', 'root_dir/image/']) ]
def customize(self, size, requested_filesystem): """ Increase the sum of all file sizes by an empiric factor Each filesystem has some overhead it needs to manage itself. Thus the plain data size is always smaller as the size of the container which embeds it. This method increases the given size by a filesystem specific empiric factor to ensure the given data size can be stored in a filesystem of the customized size :param int size: mbsize to update :param str requested_filesystem: filesystem name :return: mbytes :rtype: int """ if requested_filesystem: if requested_filesystem.startswith('ext'): size *= 1.5 file_count = self.accumulate_files() inode_mbytes = \ file_count * Defaults.get_default_inode_size() / 1048576 size += 2 * inode_mbytes elif requested_filesystem == 'btrfs': size *= 1.5 elif requested_filesystem == 'xfs': size *= 1.5 return int(size)
def test_import_description_archive_from_derived( self, mock_path, mock_open, mock_command ): path_return_values = [ True, False, True, True, True, True, True ] def side_effect(arg): return path_return_values.pop() mock_path.side_effect = side_effect self.setup_with_real_xml.import_description() assert mock_command.call_args_list == [ call(['mkdir', '-p', 'root_dir/image']), call(['cp', '../data/config.sh', 'root_dir/image/config.sh']), call([ 'cp', '../data/my_edit_boot_script', 'root_dir/image/edit_boot_config.sh' ]), call([ 'cp', '/absolute/path/to/my_edit_boot_install', 'root_dir/image/edit_boot_install.sh' ]), call(['cp', '../data/images.sh', 'root_dir/image/images.sh']), call([ 'cp', Defaults.project_file('config/functions.sh'), 'root_dir/.kconfig' ]), call(['cp', '/absolute/path/to/image.tgz', 'root_dir/image/']), call([ 'cp', 'derived/description/bootstrap.tgz', 'root_dir/image/' ]) ]
def __init__( self, xml_state, root_dir, target_dir, boot_image_task, custom_args=None ): self.arch = platform.machine() if self.arch == 'i686' or self.arch == 'i586': self.arch = 'ix86' self.root_dir = root_dir self.target_dir = target_dir self.boot_image_task = boot_image_task self.xml_state = xml_state self.initrd_system = xml_state.get_initrd_system() self.firmware = FirmWare(xml_state) self.diskname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version(), '.raw' ] ) self.isoname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version(), '.install.iso' ] ) self.pxename = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version(), '.install.tar.xz' ] ) self.dracut_config_file = ''.join( [self.root_dir, Defaults.get_dracut_conf_name()] ) self.squashed_diskname = ''.join( [xml_state.xml_data.get_name(), '.raw'] ) self.md5name = ''.join( [xml_state.xml_data.get_name(), '.md5'] ) self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.mbrid = SystemIdentifier() self.mbrid.calculate_id() self.media_dir = None self.pxe_dir = None self.squashed_contents = None self.custom_iso_args = None
def _find_grub_data(self, lookup_path): grub_path = Defaults.get_grub_path(lookup_path) if grub_path: return grub_path raise KiwiBootLoaderGrubDataError( 'No grub2 installation found in %s' % lookup_path )
def post_init(self, custom_args): """ zipl post initialization method :param dict custom_args: Contains zipl config arguments .. code:: python {'targetbase': 'device_name'} """ self.custom_args = custom_args arch = platform.machine() if 's390' in arch: self.arch = arch else: raise KiwiBootLoaderZiplPlatformError( 'host architecture %s not supported for zipl setup' % arch ) if not custom_args or 'targetbase' not in custom_args: raise KiwiBootLoaderZiplSetupError( 'targetbase device name is required for zipl setup' ) self.bootpath = '.' self.timeout = self.get_boot_timeout_seconds() self.cmdline = self.get_boot_cmdline() self.cmdline_failsafe = ' '.join( [self.cmdline, Defaults.get_failsafe_kernel_options()] ) self.target_blocksize = self.xml_state.build_type.get_target_blocksize() if not self.target_blocksize: self.target_blocksize = Defaults.get_s390_disk_block_size() self.target_type = self.xml_state.build_type.get_zipl_targettype() if not self.target_type: self.target_type = Defaults.get_s390_disk_type() self.failsafe_boot = self.failsafe_boot_entry_requested() self.target_device = custom_args['targetbase'] self.firmware = FirmWare(self.xml_state) self.target_table_type = self.firmware.get_partition_table_type() self.zipl = BootLoaderTemplateZipl() self.config = None
def test_get_custom_shared_cache_location(self): sys.argv = [ sys.argv[0], '--shared-cache-dir', '/my/cachedir', 'system', 'prepare', '--description', 'description', '--root', 'directory' ] assert Defaults.get_shared_cache_location() == 'my/cachedir'
def __init__(self, xml_state, root_dir): self.configured_size = xml_state.get_build_type_size() self.size = SystemSize(root_dir) self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type in Defaults.get_filesystem_image_types(): self.requested_filesystem = self.requested_image_type else: self.requested_filesystem = xml_state.build_type.get_filesystem()
def create(self): """ Build a mountable filesystem image Image types which triggers this builder are: * image="ext2" * image="ext3" * image="ext4" * image="btrfs" * image="xfs" :return: result :rtype: instance of :class:`Result` """ log.info( 'Creating %s filesystem', self.requested_filesystem ) supported_filesystems = Defaults.get_filesystem_image_types() if self.requested_filesystem not in supported_filesystems: raise KiwiFileSystemSetupError( 'Unknown filesystem: %s' % self.requested_filesystem ) if self.requested_filesystem not in self.filesystems_no_device_node: self._operate_on_loop() else: self._operate_on_file() self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.filename ) self.result.add( key='filesystem_image', filename=self.filename, use_for_bundle=True, compress=True, shasum=True ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result
def test_create_image_format_with_and_without_guest_additions( self, mock_create_box_img, mock_rand, mock_mkdtemp, mock_command): mock_mkdtemp.return_value = 'tmpdir' mock_create_box_img.return_value = ['arbitrary'] # without guest additions self.xml_state.get_vagrant_config_virtualbox_guest_additions \ .return_value = \ Defaults.get_vagrant_config_virtualbox_guest_additions() m_open = mock_open() with patch('builtins.open', m_open, create=True): self.disk_format.create_image_format() vagrantfile = dedent(''' Vagrant.configure("2") do |config| config.vm.base_mac = "00163E010101" config.vm.synced_folder ".", "/vagrant", type: "rsync" end ''').strip() assert m_open.return_value.write.call_args_list[1] == call(vagrantfile) # without guest additions self.xml_state.get_vagrant_config_virtualbox_guest_additions \ .return_value = True vagrantfile = dedent(''' Vagrant.configure("2") do |config| config.vm.base_mac = "00163E010101" end ''').strip() m_open.reset_mock() with patch('builtins.open', m_open, create=True): self.disk_format.create_image_format() assert m_open.return_value.write.call_args_list[1] == call(vagrantfile)
def create_xz_compressed(self, source_dir, exclude=None, options=None, xz_options=None): """ Create XZ compressed tar archive :param string source_dir: data source directory :param list exclude: list of excluded items :param list options: custom tar creation options :param list xz_options: custom xz compression options """ if not options: options = [] if not xz_options: xz_options = Defaults.get_xz_compression_options() bash_command = ['tar', '-C', source_dir ] + options + self.xattrs_options + [ '-c', '--to-stdout' ] + self._get_archive_items(source_dir, exclude) + [ '|', 'xz', '-f' ] + xz_options + ['>', self.filename + '.xz'] Command.run(['bash', '-c', ' '.join(bash_command)])
def test_import_description(self, mock_path, mock_open, mock_command): mock_path.return_value = True self.setup_with_real_xml.import_description() assert mock_command.call_args_list == [ call(['mkdir', '-p', 'root_dir/image']), call(['cp', '../data/config.sh', 'root_dir/image/config.sh']), call([ 'cp', '../data/my_edit_boot_script', 'root_dir/image/edit_boot_config.sh' ]), call([ 'cp', '/absolute/path/to/my_edit_boot_install', 'root_dir/image/edit_boot_install.sh' ]), call(['cp', '../data/images.sh', 'root_dir/image/images.sh']), call([ 'cp', Defaults.project_file('config/functions.sh'), 'root_dir/.kconfig' ]), call(['cp', '/absolute/path/to/image.tgz', 'root_dir/image/']), call(['cp', '../data/bootstrap.tgz', 'root_dir/image/']) ]
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.target_dir = target_dir self.compressed = xml_state.build_type.get_compressed() self.xen_server = xml_state.is_xen_server() self.custom_cmdline = xml_state.build_type.get_kernelcmdline() self.filesystem = xml_state.build_type.get_filesystem() if self.filesystem: self.filesystem = FileSystemBuilder(xml_state, target_dir, root_dir + '/') self.system_setup = SystemSetup(xml_state=xml_state, root_dir=root_dir) self.initrd_system = xml_state.get_initrd_system() self.boot_signing_keys = custom_args['signing_keys'] if custom_args \ and 'signing_keys' in custom_args else None self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.boot_image_task = BootImage.new( xml_state, target_dir, root_dir, signing_keys=self.boot_signing_keys) self.image_name = ''.join([ target_dir, '/', xml_state.xml_data.get_name(), '.' + Defaults.get_platform_name(), '-' + xml_state.get_image_version() ]) self.image = None self.append_file = ''.join([self.image_name, '.append']) self.archive_name = ''.join([self.image_name, '.tar']) self.checksum_name = ''.join([self.image_name, '.md5']) self.kernel_filename = None self.hypervisor_filename = None self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def _systemdisk_to_profile(self): # kiwi_lvmgroup # kiwi_lvm # kiwi_Volume_X systemdisk = self.xml_state.get_build_type_system_disk_section() if systemdisk: self.dot_profile['kiwi_lvmgroup'] = systemdisk.get_name() if not self.dot_profile['kiwi_lvmgroup']: self.dot_profile['kiwi_lvmgroup'] = \ Defaults.get_default_volume_group_name() if self.xml_state.get_volume_management(): self.dot_profile['kiwi_lvm'] = 'true' volume_count = 1 for volume in self.xml_state.get_volumes(): volume_id_name = 'kiwi_Volume_{id}'.format(id=volume_count) self.dot_profile[volume_id_name] = '|'.join( [ volume.name, 'size:all' if volume.fullsize else volume.size, volume.mountpoint or '' ] ) volume_count += 1
def test_install_ppc_ieee1275(self, mock_glob, mock_grub_path, mock_mount_manager, mock_command, mock_which, mock_wipe): mock_glob.return_value = ['tmp_root/boot/grub2/grubenv'] mock_grub_path.return_value = \ self.root_mount.mountpoint + '/usr/lib/grub2/powerpc-ieee1275' self.bootloader.arch = 'ppc64' def side_effect(device, mountpoint=None): return self.mount_managers.pop() mock_mount_manager.side_effect = side_effect self.bootloader.install() self.bootloader.root_mount.mount.assert_called_once_with() self.bootloader.boot_mount.mount.assert_called_once_with() mock_wipe.assert_called_once_with('tmp_root/boot/grub2/grubenv') mock_command.assert_called_once_with([ 'chroot', 'tmp_root', 'grub2-install', '--skip-fs-probe', '--no-nvram', '--directory', '/usr/lib/grub2/powerpc-ieee1275', '--boot-directory', '/boot', '--target', 'powerpc-ieee1275', '--modules', ' '.join(Defaults.get_grub_ofw_modules()), self.custom_args['prep_device'] ])
def get_target_file_path_for_format(self, format_name): """ Create target file path name for specified format :param string format_name: disk format name :return: file path name :rtype: str """ if format_name != 'raw': if format_name not in Defaults.get_disk_format_types(): raise KiwiFormatSetupError( 'unsupported disk format %s' % format_name ) return ''.join( [ self.target_dir, '/', self.xml_state.xml_data.get_name(), '.' + self.arch, '-' + self.xml_state.get_image_version(), '.' + format_name ] )
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.root_dir = root_dir self.target_dir = target_dir self.container_config = xml_state.get_container_config() self.requested_container_type = xml_state.get_build_type_name() self.base_image = None self.base_image_md5 = None self.container_config['xz_options'] = custom_args['xz_options'] \ if custom_args and 'xz_options' in custom_args else None if xml_state.get_derived_from_image_uri(): # The base image is expected to be unpacked by the kiwi # prepare step and stored inside of the root_dir/image directory. # In addition a md5 file of the image is expected too self.base_image = Defaults.get_imported_root_image(self.root_dir) self.base_image_md5 = ''.join([self.base_image, '.md5']) if not os.path.exists(self.base_image): raise KiwiContainerBuilderError( 'Unpacked Base image {0} not found'.format( self.base_image)) if not os.path.exists(self.base_image_md5): raise KiwiContainerBuilderError( 'Base image MD5 sum {0} not found at'.format( self.base_image_md5)) self.system_setup = SystemSetup(xml_state=xml_state, root_dir=self.root_dir) self.filename = ''.join([ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.', self.requested_container_type, '.tar.xz' ]) self.result = Result(xml_state)
def create_repository_solvable( self, target_dir=Defaults.get_solvable_location() ): """ Create SAT solvable for this repository from previously created intermediate solvables by merge and store the result solvable in the specified target_dir :param str target_dir: path name :return: file path to solvable :rtype: str """ Path.create(target_dir) solvable = os.sep.join( [target_dir, self.uri.alias()] ) if not self.is_uptodate(target_dir): self._setup_repository_metadata() solvable = self._merge_solvables(target_dir) self._cleanup() return solvable
def test_install( self, mock_glob, mock_grub_path, mock_mount_manager, mock_command, mock_which, mock_wipe ): mock_which.return_value = None mock_glob.return_value = ['tmp_root/boot/grub2/grubenv'] mock_grub_path.return_value = \ self.root_mount.mountpoint + '/usr/lib/grub2/i386-pc' self.boot_mount.device = self.root_mount.device def side_effect(device, mountpoint=None): return self.mount_managers.pop() mock_mount_manager.side_effect = side_effect self.bootloader.target_removable = True self.bootloader.install() self.root_mount.mount.assert_called_once_with() self.volume_mount.mount.assert_called_once_with( options=['subvol=@/boot/grub2'] ) mock_wipe.assert_called_once_with( 'tmp_root/boot/grub2/grubenv' ) mock_command.assert_called_once_with( [ 'chroot', 'tmp_root', 'grub2-install', '--removable', '--skip-fs-probe', '--directory', '/usr/lib/grub2/i386-pc', '--boot-directory', '/boot', '--target', 'i386-pc', '--modules', ' '.join( Defaults.get_grub_bios_modules(multiboot=True) ), '/dev/some-device' ])
def apply_xslt_stylesheets(self, description: str) -> str: """ Apply XSLT style sheet rules to an xml file The result of the XSLT processing is stored in a named temporary file and returned to the caller :param str description: path to an XML description file """ # Parse the provided description, raising the appropriate # exception if parsing fails. try: parsed_description = etree.parse(description) except etree.XMLSyntaxError: raise KiwiConfigFileFormatNotSupported( 'Support for non-XML formatted config files requires ' 'the Python anymarkup module.') xslt_transform = etree.XSLT( etree.parse(Defaults.get_xsl_stylesheet_file())) self.description_xslt_processed = NamedTemporaryFile(prefix='xslt-') with open(self.description_xslt_processed.name, "wb") as xsltout: xsltout.write(etree.tostring(xslt_transform(parsed_description))) return self.description_xslt_processed.name
def __new__(self, table_type, storage_provider, start_sector=None): host_architecture = Defaults.get_platform_name() if host_architecture == 'x86_64': if table_type == 'gpt': return PartitionerGpt(storage_provider, start_sector) elif table_type == 'msdos': return PartitionerMsDos(storage_provider, start_sector) elif host_architecture == 'ix86': if table_type == 'msdos': return PartitionerMsDos(storage_provider, start_sector) elif 'ppc64' in host_architecture: if table_type == 'gpt': return PartitionerGpt(storage_provider, start_sector) elif table_type == 'msdos': return PartitionerMsDos(storage_provider, start_sector) elif 's390' in host_architecture: if table_type == 'dasd': if start_sector: log.warning('disk_start_sector value is ignored ' 'for dasd partitions') return PartitionerDasd(storage_provider) elif table_type == 'msdos': return PartitionerMsDos(storage_provider, start_sector) elif 'arm' in host_architecture or host_architecture == 'aarch64': if table_type == 'gpt': return PartitionerGpt(storage_provider, start_sector) elif table_type == 'msdos': return PartitionerMsDos(storage_provider, start_sector) raise KiwiPartitionerSetupError( 'Support for partitioner on %s architecture not implemented' % host_architecture)
def check_target_directory_not_in_shared_cache(self, target_dir): """ The target directory must be outside of the kiwi shared cache directory in order to avoid busy mounts because kiwi bind mounts the cache directory into the image root tree to access host caching information :param string target_dir: path name """ message = dedent('''\n Target directory %s conflicts with kiwi's shared cache directory %s. This is going to create a busy loop mount. Please choose another target directory. ''') shared_cache_location = Defaults.get_shared_cache_location() target_dir_stack = os.path.abspath( os.path.normpath(target_dir) ).replace(os.sep + os.sep, os.sep).split(os.sep) if target_dir_stack[1:4] == shared_cache_location.split(os.sep): raise KiwiRuntimeError( message % (target_dir, shared_cache_location) )
def apply_xslt_stylesheets(self, description: str) -> str: """ Apply XSLT style sheet rules to an xml file The result of the XSLT processing is stored in a named temporary file and returned to the caller :param str description: path to an XML description file """ # Parse the provided description, raising the appropriate # exception if parsing fails. try: parsed_description = etree.parse(description) except etree.XMLSyntaxError: raise KiwiConfigFileFormatNotSupported( 'Configuration file could not be parsed. ' 'In case your configuration file is XML it most likely ' 'contains a syntax error. For other formats the ' 'Python anymarkup module is required.') xslt_transform_parser = etree.XMLParser() xslt_transform_parser.resolvers.add(FileResolver()) xslt_transform = etree.XSLT( etree.parse(Defaults.get_xsl_stylesheet_file(), xslt_transform_parser)) self.description_xslt_processed = Temporary( prefix='kiwi_xslt-').new_file() try: with open(self.description_xslt_processed.name, "wb") as xsltout: xsltout.write( etree.tostring(xslt_transform(parsed_description), pretty_print=True)) except etree.XMLSyntaxError as issue: raise KiwiDescriptionInvalid(issue) return self.description_xslt_processed.name
def import_container_image(self, container_image_ref): """ Imports container image reference to the OCI containers storage. :param str container_image_ref: container image reference """ if not self.imported_image: self.imported_image = 'kiwi-image-{0}:{1}'.format( self._random_string_generator(), Defaults.get_container_base_image_tag()) else: raise KiwiBuildahError( "Image already imported, called: '{0}'".format( self.imported_image)) # We are making use of skopeo instead of only calling 'buildah from' # because we want to control the image name loaded into the containers # storage. This way we are certain to not leave any left over after the # build. Command.run([ 'skopeo', 'copy', container_image_ref, 'containers-storage:{0}'.format(self.imported_image) ]) if not self.working_container: self.working_container = 'kiwi-container-{0}'.format( self._random_string_generator()) else: raise KiwiBuildahError( "Container already initated, called: '{0}'".format( self.working_container)) Command.run([ 'buildah', 'from', '--name', self.working_container, 'containers-storage:{0}'.format(self.imported_image) ])
def __new__(self, xml_state, target_dir, root_dir, custom_args=None): requested_image_type = xml_state.get_build_type_name() if requested_image_type in Defaults.get_filesystem_image_types(): return FileSystemBuilder(xml_state, target_dir, root_dir) elif requested_image_type in Defaults.get_disk_image_types(): return DiskBuilder(xml_state, target_dir, root_dir, custom_args) elif requested_image_type in Defaults.get_live_image_types(): return LiveImageBuilder(xml_state, target_dir, root_dir, custom_args) elif requested_image_type in Defaults.get_network_image_types(): return PxeBuilder(xml_state, target_dir, root_dir, custom_args) elif requested_image_type in Defaults.get_archive_image_types(): return ArchiveBuilder(xml_state, target_dir, root_dir, custom_args) elif requested_image_type in Defaults.get_container_image_types(): return ContainerBuilder(xml_state, target_dir, root_dir, custom_args) else: raise KiwiRequestedTypeError( 'requested image type %s not supported' % requested_image_type)
def __init__(self, root_dir, transport, custom_args=None): self.root_dir = root_dir self.archive_transport = transport if custom_args: self.oci_config = custom_args else: self.oci_config = {} self.runtime_config = RuntimeConfig() # for builds inside the buildservice we include a reference to the # specific build. Thus disturl label only exists inside the # buildservice. if Defaults.is_buildservice_worker(): bs_label = 'org.openbuildservice.disturl' # Do not label anything if the build service label is # already present if 'labels' not in self.oci_config or \ bs_label not in self.oci_config['labels']: self._append_buildservice_disturl_label() if 'container_name' not in self.oci_config: log.info('No container configuration provided, ' 'using default container name "kiwi-container:latest"') self.oci_config['container_name'] = \ Defaults.get_default_container_name() self.oci_config['container_tag'] = \ Defaults.get_default_container_tag() if 'container_tag' not in self.oci_config: self.oci_config['container_tag'] = \ Defaults.get_default_container_tag() if 'entry_command' not in self.oci_config and \ 'entry_subcommand' not in self.oci_config: self.oci_config['entry_subcommand'] = \ Defaults.get_default_container_subcommand() if 'history' not in self.oci_config: self.oci_config['history'] = {} if 'created_by' not in self.oci_config['history']: self.oci_config['history']['created_by'] = \ Defaults.get_default_container_created_by()
def new( xml_state: object, target_dir: str, root_dir: str, custom_args: Dict = None # noqa: E252 ): image_type = xml_state.get_build_type_name() name_map = { 'filesystem': 'FileSystemBuilder' if image_type in Defaults.get_filesystem_image_types() else None, 'disk': 'DiskBuilder' if image_type in Defaults.get_disk_image_types() else None, 'live': 'LiveImageBuilder' if image_type in Defaults.get_live_image_types() else None, 'kis': 'KisBuilder' if image_type in Defaults.get_kis_image_types() else None, 'archive': 'ArchiveBuilder' if image_type in Defaults.get_archive_image_types() else None, 'container': 'ContainerBuilder' if image_type in Defaults.get_container_image_types() else None } for builder_namespace, builder_name in list(name_map.items()): if builder_name: break try: builder = importlib.import_module( 'kiwi.builder.{0}'.format(builder_namespace)) return builder.__dict__[builder_name](xml_state, target_dir, root_dir, custom_args) except Exception as issue: raise KiwiRequestedTypeError( 'Requested image type {0} not supported: {1}'.format( image_type, issue))
def setup(self): Defaults.set_platform_name('x86_64') self.size = mock.Mock() self.size.customize = mock.Mock(return_value=42) self.size.accumulate_mbyte_file_sizes = mock.Mock(return_value=42) kiwi.storage.setup.SystemSize = mock.Mock(return_value=self.size) description = XMLDescription('../data/example_disk_size_config.xml') self.setup = DiskSetup(XMLState(description.load()), 'root_dir') description = XMLDescription( '../data/example_disk_size_volume_config.xml') self.setup_volumes = DiskSetup(XMLState(description.load()), 'root_dir') description = XMLDescription( '../data/example_disk_size_oem_volume_config.xml') self.setup_oem_volumes = DiskSetup(XMLState(description.load()), 'root_dir') description = XMLDescription( '../data/example_disk_size_empty_vol_config.xml') self.setup_empty_volumes = DiskSetup(XMLState(description.load()), 'root_dir') description = XMLDescription( '../data/example_disk_size_vol_root_config.xml') self.setup_root_volume = DiskSetup(XMLState(description.load()), 'root_dir') Defaults.set_platform_name('ppc64') description = XMLDescription( '../data/example_ppc_disk_size_config.xml') self.setup_ppc = DiskSetup(XMLState(description.load()), 'root_dir') Defaults.set_platform_name('arm64') description = XMLDescription( '../data/example_arm_disk_size_config.xml') self.setup_arm = DiskSetup(XMLState(description.load()), 'root_dir')
def _build_and_map_disk_partitions(self, disksize_mbytes): # noqa: C901 self.disk.wipe() disksize_used_mbytes = 0 if self.firmware.legacy_bios_mode(): log.info('--> creating EFI CSM(legacy bios) partition') partition_mbsize = self.firmware.get_legacy_bios_partition_size() self.disk.create_efi_csm_partition(partition_mbsize) disksize_used_mbytes += partition_mbsize if self.firmware.efi_mode(): log.info('--> creating EFI partition') partition_mbsize = self.firmware.get_efi_partition_size() self.disk.create_efi_partition(partition_mbsize) disksize_used_mbytes += partition_mbsize if self.firmware.ofw_mode(): log.info('--> creating PReP partition') partition_mbsize = self.firmware.get_prep_partition_size() self.disk.create_prep_partition(partition_mbsize) disksize_used_mbytes += partition_mbsize if self.disk_setup.need_boot_partition(): log.info('--> creating boot partition') partition_mbsize = self.disk_setup.boot_partition_size() self.disk.create_boot_partition(partition_mbsize) disksize_used_mbytes += partition_mbsize if self.spare_part_mbsize and not self.spare_part_is_last: log.info('--> creating spare partition') self.disk.create_spare_partition(self.spare_part_mbsize) if self.root_filesystem_is_overlay: log.info('--> creating readonly root partition') squashed_root_file = NamedTemporaryFile() squashed_root = FileSystemSquashFs(device_provider=None, root_dir=self.root_dir) squashed_root.create_on_file( filename=squashed_root_file.name, exclude=[Defaults.get_shared_cache_location()]) squashed_rootfs_mbsize = int( os.path.getsize(squashed_root_file.name) / 1048576) + Defaults.get_min_partition_mbytes() self.disk.create_root_readonly_partition(squashed_rootfs_mbsize) disksize_used_mbytes += squashed_rootfs_mbsize if self.spare_part_mbsize and self.spare_part_is_last: rootfs_mbsize = disksize_mbytes - disksize_used_mbytes - \ self.spare_part_mbsize - Defaults.get_min_partition_mbytes() else: rootfs_mbsize = 'all_free' if self.volume_manager_name and self.volume_manager_name == 'lvm': log.info('--> creating LVM root partition') self.disk.create_root_lvm_partition(rootfs_mbsize) elif self.mdraid: log.info('--> creating mdraid root partition') self.disk.create_root_raid_partition(rootfs_mbsize) else: log.info('--> creating root partition') self.disk.create_root_partition(rootfs_mbsize) if self.spare_part_mbsize and self.spare_part_is_last: log.info('--> creating spare partition') self.disk.create_spare_partition('all_free') if self.firmware.bios_mode(): log.info('--> setting active flag to primary boot partition') self.disk.activate_boot_partition() if self.firmware.ofw_mode(): log.info('--> setting active flag to primary PReP partition') self.disk.activate_boot_partition() if self.firmware.efi_mode(): if self.force_mbr: log.info('--> converting partition table to MBR') self.disk.create_mbr() elif self.hybrid_mbr: log.info('--> converting partition table to hybrid GPT/MBR') self.disk.create_hybrid_mbr() self.disk.map_partitions() return self.disk.get_device()
def create_disk(self): # noqa: C901 """ Build a bootable raw disk image :raises KiwiInstallMediaError: if install media is required and image type is not oem :raises KiwiVolumeManagerSetupError: root overlay at the same time volumes are defined is not supported :return: result :rtype: instance of :class:`Result` """ if self.install_media and self.build_type_name != 'oem': raise KiwiInstallMediaError( 'Install media requires oem type setup, got %s' % self.build_type_name) if self.root_filesystem_is_overlay and self.volume_manager_name: raise KiwiVolumeManagerSetupError( 'Volume management together with root overlay is not supported' ) # setup recovery archive, cleanup and create archive if requested self.system_setup.create_recovery_archive() # prepare boot(initrd) root system log.info('Preparing boot system') self.boot_image.prepare() # precalculate needed disk size disksize_mbytes = self.disk_setup.get_disksize_mbytes() # create the disk log.info('Creating raw disk image %s', self.diskname) loop_provider = LoopDevice(self.diskname, disksize_mbytes, self.blocksize) loop_provider.create() self.disk = Disk(self.firmware.get_partition_table_type(), loop_provider, self.xml_state.get_disk_start_sector()) # create the bootloader instance self.bootloader_config = BootLoaderConfig( self.bootloader, self.xml_state, root_dir=self.root_dir, boot_dir=self.root_dir, custom_args={ 'targetbase': loop_provider.get_device(), 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir), 'boot_is_crypto': self.boot_is_crypto }) # create disk partitions and instance device map device_map = self._build_and_map_disk_partitions(disksize_mbytes) # create raid on current root device if requested if self.mdraid: self.raid_root = RaidDevice(device_map['root']) self.raid_root.create_degraded_raid(raid_level=self.mdraid) device_map['root'] = self.raid_root.get_device() self.disk.public_partition_id_map['kiwi_RaidPart'] = \ self.disk.public_partition_id_map['kiwi_RootPart'] self.disk.public_partition_id_map['kiwi_RaidDev'] = \ device_map['root'].get_device() # create luks on current root device if requested if self.luks: self.luks_root = LuksDevice(device_map['root']) self.luks_boot_keyname = '/.root.keyfile' self.luks_boot_keyfile = ''.join( [self.root_dir, self.luks_boot_keyname]) self.luks_root.create_crypto_luks(passphrase=self.luks, os=self.luks_os, keyfile=self.luks_boot_keyfile if self.boot_is_crypto else None) if self.boot_is_crypto: self.luks_boot_keyfile_setup = ''.join( [self.root_dir, '/etc/dracut.conf.d/99-luks-boot.conf']) self.boot_image.write_system_config_file( config={'install_items': [self.luks_boot_keyname]}, config_file=self.luks_boot_keyfile_setup) self.boot_image.include_file( os.sep + os.path.basename(self.luks_boot_keyfile)) device_map['luks_root'] = device_map['root'] device_map['root'] = self.luks_root.get_device() # create spare filesystem on spare partition if present self._build_spare_filesystem(device_map) # create filesystems on boot partition(s) if any self._build_boot_filesystems(device_map) # create volumes and filesystems for root system if self.volume_manager_name: volume_manager_custom_parameters = { 'fs_mount_options': self.custom_root_mount_args, 'fs_create_options': self.custom_root_creation_args, 'root_label': self.disk_setup.get_root_label(), 'root_is_snapshot': self.xml_state.build_type.get_btrfs_root_is_snapshot(), 'root_is_readonly_snapshot': self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot( ), 'quota_groups': self.xml_state.build_type.get_btrfs_quota_groups(), 'image_type': self.xml_state.get_build_type_name() } volume_manager = VolumeManager(self.volume_manager_name, device_map['root'], self.root_dir + '/', self.volumes, volume_manager_custom_parameters) volume_manager.setup(self.volume_group_name) volume_manager.create_volumes(self.requested_filesystem) volume_manager.mount_volumes() self.generic_fstab_entries += volume_manager.get_fstab( self.persistency_type, self.requested_filesystem) self.system = volume_manager device_map['root'] = volume_manager.get_device()['root'] else: log.info('Creating root(%s) filesystem on %s', self.requested_filesystem, device_map['root'].get_device()) filesystem_custom_parameters = { 'mount_options': self.custom_root_mount_args, 'create_options': self.custom_root_creation_args } filesystem = FileSystem(self.requested_filesystem, device_map['root'], self.root_dir + '/', filesystem_custom_parameters) filesystem.create_on_device(label=self.disk_setup.get_root_label()) self.system = filesystem # create a random image identifier self.mbrid = SystemIdentifier() self.mbrid.calculate_id() # create first stage metadata to boot image self._write_partition_id_config_to_boot_image() self._write_recovery_metadata_to_boot_image() self._write_raid_config_to_boot_image() self._write_generic_fstab_to_boot_image(device_map) self.system_setup.export_modprobe_setup( self.boot_image.boot_root_directory) # create first stage metadata to system image self._write_image_identifier_to_system_image() self._write_crypttab_to_system_image() self._write_generic_fstab_to_system_image(device_map) if self.initrd_system == 'dracut': if self.root_filesystem_is_multipath is False: self.boot_image.omit_module('multipath') if self.root_filesystem_is_overlay: self.boot_image.include_module('kiwi-overlay') self.boot_image.write_system_config_file( config={'modules': ['kiwi-overlay']}) if self.build_type_name == 'oem': self.boot_image.include_module('kiwi-repart') # create initrd cpio archive self.boot_image.create_initrd(self.mbrid) # create second stage metadata to system image self._copy_first_boot_files_to_system_image() self._write_bootloader_meta_data_to_system_image(device_map) self.mbrid.write_to_disk(self.disk.storage_provider) # set SELinux file security contexts if context exists self._setup_selinux_file_contexts() # syncing system data to disk image log.info('Syncing system to image') if self.system_spare: self.system_spare.sync_data() if self.system_efi: log.info('--> Syncing EFI boot data to EFI partition') self.system_efi.sync_data() if self.system_boot: log.info('--> Syncing boot data at extra partition') self.system_boot.sync_data( self._get_exclude_list_for_boot_data_sync()) log.info('--> Syncing root filesystem data') if self.root_filesystem_is_overlay: squashed_root_file = NamedTemporaryFile() squashed_root = FileSystemSquashFs(device_provider=None, root_dir=self.root_dir) squashed_root.create_on_file( filename=squashed_root_file.name, exclude=self._get_exclude_list_for_root_data_sync(device_map)) Command.run([ 'dd', 'if=%s' % squashed_root_file.name, 'of=%s' % device_map['readonly'].get_device() ]) else: self.system.sync_data( self._get_exclude_list_for_root_data_sync(device_map)) # install boot loader self._install_bootloader(device_map) # set root filesystem properties self._setup_property_root_is_readonly_snapshot() # prepare for install media if requested if self.install_media: log.info('Saving boot image instance to file') self.boot_image.dump(self.target_dir + '/boot_image.pickledump') self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.diskname) # store image file name in result self.result.add( key='disk_image', filename=self.diskname, use_for_bundle=True if not self.image_format else False, compress=self.runtime_config.get_bundle_compression(default=True), shasum=True) # create image root metadata self.result.add(key='image_packages', filename=self.system_setup.export_package_list( self.target_dir), use_for_bundle=True, compress=False, shasum=False) self.result.add(key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir), use_for_bundle=True, compress=False, shasum=False) return self.result
def create_install_iso(self): """ Create an install ISO from the disk_image as hybrid ISO bootable via legacy BIOS, EFI and as disk from Stick Image types which triggers this builder are: * installiso="true|false" * installstick="true|false" """ self.media_dir = mkdtemp(prefix='kiwi_install_media.', dir=self.target_dir) # unpack cdroot user files to media dir self.setup.import_cdroot_files(self.media_dir) # custom iso metadata self.custom_iso_args = { 'meta_data': { 'volume_id': Defaults.get_install_volume_id(), 'mbr_id': self.mbrid.get_id(), 'efi_mode': self.firmware.efi_mode() } } # the system image transfer is checked against a checksum log.info('Creating disk image checksum') self.squashed_contents = mkdtemp(prefix='kiwi_install_squashfs.', dir=self.target_dir) checksum = Checksum(self.diskname) checksum.md5(self.squashed_contents + '/' + self.md5name) # the system image name is stored in a config file self._write_install_image_info_to_iso_image() if self.initrd_system == 'kiwi': self._write_install_image_info_to_boot_image() # the system image is stored as squashfs embedded file log.info('Creating squashfs embedded disk image') Command.run([ 'cp', '-l', self.diskname, self.squashed_contents + '/' + self.squashed_diskname ]) squashed_image_file = ''.join( [self.target_dir, '/', self.squashed_diskname, '.squashfs']) squashed_image = FileSystemSquashFs(device_provider=None, root_dir=self.squashed_contents) squashed_image.create_on_file(squashed_image_file) Command.run(['mv', squashed_image_file, self.media_dir]) # setup bootloader config to boot the ISO via isolinux log.info('Setting up install image bootloader configuration') bootloader_config_isolinux = BootLoaderConfig('isolinux', self.xml_state, self.media_dir) bootloader_config_isolinux.setup_install_boot_images( mbrid=None, lookup_path=self.boot_image_task.boot_root_directory) bootloader_config_isolinux.setup_install_image_config(mbrid=None) bootloader_config_isolinux.write() # setup bootloader config to boot the ISO via EFI bootloader_config_grub = BootLoaderConfig( 'grub2', self.xml_state, self.media_dir, { 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) }) bootloader_config_grub.setup_install_boot_images( mbrid=self.mbrid, lookup_path=self.root_dir) bootloader_config_grub.setup_install_image_config(mbrid=self.mbrid) bootloader_config_grub.write() # create initrd for install image log.info('Creating install image boot image') self._create_iso_install_kernel_and_initrd() # the system image initrd is stored to allow kexec self._copy_system_image_initrd_to_iso_image() # create iso filesystem from media_dir log.info('Creating ISO filesystem') iso_image = FileSystemIsoFs(device_provider=None, root_dir=self.media_dir, custom_args=self.custom_iso_args) iso_image.create_on_file(self.isoname)
def create(self): """ Build a bootable hybrid live ISO image Image types which triggers this builder are: * image="iso" """ # media dir to store CD contents self.media_dir = mkdtemp( prefix='live-media.', dir=self.target_dir ) rootsize = SystemSize(self.media_dir) # custom iso metadata log.info('Using following live ISO metadata:') log.info('--> Application id: {0}'.format(self.mbrid.get_id())) log.info('--> Publisher: {0}'.format(Defaults.get_publisher())) log.info('--> Volume id: {0}'.format(self.volume_id)) custom_iso_args = { 'create_options': [ '-A', self.mbrid.get_id(), '-p', Defaults.get_preparer(), '-publisher', Defaults.get_publisher(), '-V', self.volume_id ] } # pack system into live boot structure as expected by dracut log.info( 'Packing system into dracut live ISO type: {0}'.format( self.live_type ) ) root_filesystem = Defaults.get_default_live_iso_root_filesystem() filesystem_custom_parameters = { 'mount_options': self.xml_state.get_fs_mount_option_list() } filesystem_setup = FileSystemSetup( self.xml_state, self.root_dir ) root_image = NamedTemporaryFile() loop_provider = LoopDevice( root_image.name, filesystem_setup.get_size_mbytes(root_filesystem), self.xml_state.build_type.get_target_blocksize() ) loop_provider.create() live_filesystem = FileSystem( name=root_filesystem, device_provider=loop_provider, root_dir=self.root_dir + os.sep, custom_args=filesystem_custom_parameters ) live_filesystem.create_on_device() log.info( '--> Syncing data to {0} root image'.format(root_filesystem) ) live_filesystem.sync_data( Defaults.get_exclude_list_for_root_data_sync() ) log.info('--> Creating squashfs container for root image') self.live_container_dir = mkdtemp( prefix='live-container.', dir=self.target_dir ) Path.create(self.live_container_dir + '/LiveOS') shutil.copy( root_image.name, self.live_container_dir + '/LiveOS/rootfs.img' ) live_container_image = FileSystem( name='squashfs', device_provider=None, root_dir=self.live_container_dir ) container_image = NamedTemporaryFile() live_container_image.create_on_file( container_image.name ) Path.create(self.media_dir + '/LiveOS') shutil.copy( container_image.name, self.media_dir + '/LiveOS/squashfs.img' ) # setup bootloader config to boot the ISO via isolinux log.info('Setting up isolinux bootloader configuration') bootloader_config_isolinux = BootLoaderConfig( 'isolinux', self.xml_state, self.media_dir ) bootloader_config_isolinux.setup_live_boot_images( mbrid=None, lookup_path=self.boot_image.boot_root_directory ) bootloader_config_isolinux.setup_live_image_config( mbrid=None ) bootloader_config_isolinux.write() # setup bootloader config to boot the ISO via EFI if self.firmware.efi_mode(): log.info('Setting up EFI grub bootloader configuration') bootloader_config_grub = BootLoaderConfig( 'grub2', self.xml_state, self.media_dir, { 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) } ) bootloader_config_grub.setup_live_boot_images( mbrid=self.mbrid, lookup_path=self.root_dir ) bootloader_config_grub.setup_live_image_config( mbrid=self.mbrid ) bootloader_config_grub.write() # call custom editbootconfig script if present self.system_setup.call_edit_boot_config_script( filesystem='iso:{0}'.format(self.media_dir), boot_part_id=1, working_directory=self.root_dir ) # create dracut initrd for live image log.info('Creating live ISO boot image') self._create_dracut_live_iso_config() self.boot_image.create_initrd(self.mbrid) # setup kernel file(s) and initrd in ISO boot layout log.info('Setting up kernel file(s) and boot image in ISO boot layout') self._setup_live_iso_kernel_and_initrd() # calculate size and decide if we need UDF if rootsize.accumulate_mbyte_file_sizes() > 4096: log.info('ISO exceeds 4G size, using UDF filesystem') custom_iso_args['create_options'].append('-iso-level') custom_iso_args['create_options'].append('3') custom_iso_args['create_options'].append('-udf') # create iso filesystem from media_dir log.info('Creating live ISO image') iso_image = FileSystemIsoFs( device_provider=None, root_dir=self.media_dir, custom_args=custom_iso_args ) iso_header_offset = iso_image.create_on_file(self.isoname) # make it hybrid if self.hybrid: Iso.create_hybrid( iso_header_offset, self.mbrid, self.isoname, self.firmware.efi_mode() ) # include metadata for checkmedia tool if self.xml_state.build_type.get_mediacheck() is True: Iso.set_media_tag(self.isoname) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.isoname ) self.result.add( key='live_image', filename=self.isoname, use_for_bundle=True, compress=False, shasum=True ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.arch = Defaults.get_platform_name() self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.spare_part_mbsize = xml_state.get_build_type_spare_part_size() self.spare_part_fs = xml_state.build_type.get_spare_part_fs() self.spare_part_is_last = xml_state.build_type.get_spare_part_is_last() self.spare_part_mountpoint = \ xml_state.build_type.get_spare_part_mountpoint() self.persistency_type = xml_state.build_type.get_devicepersistency() self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot( ) self.custom_root_mount_args = xml_state.get_fs_mount_option_list() self.custom_root_creation_args = xml_state.get_fs_create_option_list() self.build_type_name = xml_state.get_build_type_name() self.image_format = xml_state.build_type.get_format() self.install_iso = xml_state.build_type.get_installiso() self.install_stick = xml_state.build_type.get_installstick() self.install_pxe = xml_state.build_type.get_installpxe() self.blocksize = xml_state.build_type.get_target_blocksize() self.volume_manager_name = xml_state.get_volume_management() self.volumes = xml_state.get_volumes() self.volume_group_name = xml_state.get_volume_group_name() self.mdraid = xml_state.build_type.get_mdraid() self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr() self.force_mbr = xml_state.build_type.get_force_mbr() self.luks = xml_state.build_type.get_luks() self.luks_os = xml_state.build_type.get_luksOS() self.xen_server = xml_state.is_xen_server() self.requested_filesystem = xml_state.build_type.get_filesystem() self.requested_boot_filesystem = \ xml_state.build_type.get_bootfilesystem() self.bootloader = xml_state.get_build_type_bootloader_name() self.initrd_system = xml_state.get_initrd_system() self.target_removable = xml_state.build_type.get_target_removable() self.root_filesystem_is_multipath = \ xml_state.get_oemconfig_oem_multipath_scan() self.swap_mbytes = xml_state.get_oemconfig_swap_mbytes() self.disk_setup = DiskSetup(xml_state, root_dir) self.unpartitioned_bytes = \ xml_state.get_build_type_unpartitioned_bytes() self.custom_args = custom_args self.signing_keys = None if custom_args and 'signing_keys' in custom_args: self.signing_keys = custom_args['signing_keys'] self.boot_image = BootImage(xml_state, target_dir, root_dir, signing_keys=self.signing_keys) self.firmware = FirmWare(xml_state) self.system_setup = SystemSetup(xml_state=xml_state, root_dir=self.root_dir) self.diskname = ''.join([ target_dir, '/', xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version(), '.raw' ]) self.boot_is_crypto = True if self.luks and not \ self.disk_setup.need_boot_partition() else False self.install_media = self._install_image_requested() self.fstab = Fstab() # an instance of a class with the sync_data capability # representing the entire image system except for the boot/ area # which could live on another part of the disk self.system = None # an instance of a class with the sync_data capability # representing the boot/ area of the disk if not part of # self.system self.system_boot = None # an instance of a class with the sync_data capability # representing the boot/efi area of the disk self.system_efi = None # an instance of a class with the sync_data capability # representing the spare_part_mountpoint area of the disk self.system_spare = None # result store self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def test_get_obs_download_server_url_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.get_obs_download_server_url() == \ Defaults.get_obs_download_server_url()
def process(self): # noqa: C901 """ Prepare and install a new system for chroot access """ self.manual = Help() if self._help(): return Privileges.check_for_root_permissions() self.load_xml_description(self.command_args['--description']) abs_root_path = os.path.abspath(self.command_args['--root']) self.runtime_checker.check_efi_mode_for_disk_overlay_correctly_setup() self.runtime_checker.check_grub_efi_installed_for_efi_firmware() self.runtime_checker.check_boot_description_exists() self.runtime_checker.check_consistent_kernel_in_boot_and_system_image() self.runtime_checker.check_docker_tool_chain_installed() self.runtime_checker.check_volume_setup_has_no_root_definition() self.runtime_checker.check_xen_uniquely_setup_as_server_or_guest() self.runtime_checker.check_target_directory_not_in_shared_cache( abs_root_path) self.runtime_checker.check_mediacheck_only_for_x86_arch() self.runtime_checker.check_dracut_module_for_live_iso_in_package_list() self.runtime_checker.check_dracut_module_for_disk_overlay_in_package_list( ) self.runtime_checker.check_dracut_module_for_disk_oem_in_package_list() self.runtime_checker.check_dracut_module_for_oem_install_in_package_list( ) if self.command_args['--ignore-repos']: self.xml_state.delete_repository_sections() elif self.command_args['--ignore-repos-used-for-build']: self.xml_state.delete_repository_sections_used_for_build() if self.command_args['--set-repo']: self.xml_state.set_repository( *self.sextuple_token(self.command_args['--set-repo'])) if self.command_args['--add-repo']: for add_repo in self.command_args['--add-repo']: self.xml_state.add_repository(*self.sextuple_token(add_repo)) if self.command_args['--set-container-tag']: self.xml_state.set_container_config_tag( self.command_args['--set-container-tag']) if self.command_args['--set-container-derived-from']: self.xml_state.set_derived_from_image_uri( self.command_args['--set-container-derived-from']) self.runtime_checker.check_repositories_configured() self.runtime_checker.check_image_include_repos_publicly_resolvable() log.info('Preparing system') system = SystemPrepare(self.xml_state, abs_root_path, self.command_args['--allow-existing-root']) manager = system.setup_repositories(self.command_args['--clear-cache'], self.command_args['--signing-key']) system.install_bootstrap(manager) system.install_system(manager) if self.command_args['--add-package']: system.install_packages(manager, self.command_args['--add-package']) if self.command_args['--delete-package']: system.delete_packages(manager, self.command_args['--delete-package']) profile = Profile(self.xml_state) defaults = Defaults() defaults.to_profile(profile) setup = SystemSetup(self.xml_state, abs_root_path) setup.import_shell_environment(profile) setup.import_description() setup.import_overlay_files() setup.import_image_identifier() setup.setup_groups() setup.setup_users() setup.setup_keyboard_map() setup.setup_locale() setup.setup_plymouth_splash() setup.setup_timezone() # make sure manager instance is cleaned up now del manager # setup permanent image repositories after cleanup setup.import_repositories_marked_as_imageinclude() setup.call_config_script() # handle uninstall package requests, gracefully uninstall # with dependency cleanup system.pinch_system(force=False) # handle delete package requests, forced uninstall without # any dependency resolution system.pinch_system(force=True) # make sure system instance is cleaned up now del system
def load(self) -> Any: """ Read XML description, validate it against the schema and the schematron rules and pass it to the autogenerated(generateDS) parser. :return: instance of XML toplevel domain (image) :rtype: object """ try: schema_doc = etree.parse(Defaults.get_schema_file()) relaxng = etree.RelaxNG(schema_doc) schematron = isoschematron.Schematron( schema_doc, store_report=True ) except Exception as issue: raise KiwiSchemaImportError(issue) try: description = etree.parse(self.description) validation_rng = relaxng.validate(description) validation_schematron = schematron.validate(description) except Exception as issue: raise KiwiValidationError(issue) if not validation_rng: XMLDescription._get_relaxng_validation_details( Defaults.get_schema_file(), self.description, relaxng.error_log ) if not validation_schematron: XMLDescription._get_schematron_validation_details( schematron.validation_report ) if not validation_rng or not validation_schematron: raise KiwiDescriptionInvalid( 'Failed to validate schema and/or schematron rules' ) parse_result = self._parse() if parse_result.get_extension(): extension_namespace_map = \ description.getroot().xpath('extension')[0].nsmap for namespace_name in extension_namespace_map: extensions_for_namespace = description.getroot().xpath( 'extension/{namespace}:*'.format(namespace=namespace_name), namespaces=extension_namespace_map ) if extensions_for_namespace: # one toplevel entry point per extension via xmlns if len(extensions_for_namespace) > 1: raise KiwiExtensionError( 'Multiple toplevel sections for "{0}" found'.format( namespace_name ) ) # store extension xml data parse tree for this namespace self.extension_data[namespace_name] = \ etree.ElementTree(extensions_for_namespace[0]) # validate extension xml data try: xml_catalog = Command.run( [ 'xmlcatalog', '/etc/xml/catalog', extension_namespace_map[namespace_name] ] ) extension_schema = xml_catalog.output.rstrip().replace( 'file://', '' ) extension_relaxng = etree.RelaxNG( etree.parse(extension_schema) ) except Exception as issue: raise KiwiExtensionError( 'Extension schema error: {0}'.format(issue) ) validation_result = extension_relaxng.validate( self.extension_data[namespace_name] ) if not validation_result: xml_data_unformatted = etree.tostring( self.extension_data[namespace_name], encoding='utf-8' ) xml_data_domtree = minidom.parseString( xml_data_unformatted ) extension_file = Temporary().new_file() with open(extension_file.name, 'w') as xml_data: xml_data.write(xml_data_domtree.toprettyxml()) XMLDescription._get_relaxng_validation_details( extension_schema, extension_file.name, extension_relaxng.error_log ) raise KiwiExtensionError( 'Schema validation for extension XML data failed' ) return parse_result
def __init__( self, xml_state, root_dir, target_dir, boot_image_task, custom_args=None ): self.arch = platform.machine() if self.arch == 'i686' or self.arch == 'i586': self.arch = 'ix86' self.root_dir = root_dir self.target_dir = target_dir self.boot_image_task = boot_image_task self.xml_state = xml_state self.root_filesystem_is_multipath = \ xml_state.get_oemconfig_oem_multipath_scan() self.initrd_system = xml_state.get_initrd_system() self.firmware = FirmWare(xml_state) self.setup = SystemSetup( self.xml_state, self.root_dir ) self.iso_volume_id = self.xml_state.build_type.get_volid() or \ Defaults.get_install_volume_id() self.diskname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version(), '.raw' ] ) self.isoname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version(), '.install.iso' ] ) self.pxename = ''.join( [ xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version() ] ) self.pxetarball = ''.join( [ target_dir, '/', self.pxename, '.install.tar' ] ) self.dracut_config_file = ''.join( [self.root_dir, Defaults.get_dracut_conf_name()] ) self.squashed_diskname = ''.join( [xml_state.xml_data.get_name(), '.raw'] ) self.md5name = ''.join( [xml_state.xml_data.get_name(), '.md5'] ) self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.mbrid = SystemIdentifier() self.mbrid.calculate_id() self.media_dir = None self.pxe_dir = None self.squashed_contents = None self.custom_iso_args = None
def test_get_exclude_list_from_custom_exclude_files(self): assert Defaults.get_exclude_list_from_custom_exclude_files( '../data/root-dir') == [ 'usr/bin/qemu-binfmt', 'usr/bin/qemu-x86_64-binfmt', 'usr/bin/qemu-x86_64' ]
def test_get_vendor_grubenv(self, mock_path_exists): mock_path_exists.return_value = True assert Defaults.get_vendor_grubenv('boot/efi') == \ 'boot/efi/EFI/fedora/grubenv'