def setup_intermediate_config(self): """ Create intermediate config files Some config files e.g etc/hosts needs to be temporarly copied from the buildsystem host to the image root system in order to allow e.g DNS resolution in the way as it is configured on the buildsystem host. These config files only exists during the image build process and are not part of the final image :raises KiwiSetupIntermediateConfigError: if the management of intermediate configuration files fails """ try: for config in self.config_files: if os.path.exists(config): self.cleanup_files.append(config + '.kiwi') Command.run( ['cp', config, self.root_dir + config + '.kiwi'] ) link_target = os.path.basename(config) + '.kiwi' Command.run( ['ln', '-s', '-f', link_target, self.root_dir + config] ) checksum = Checksum(config) with open(self.root_dir + config + '.sha', 'w') as shasum: shasum.write(checksum.sha256()) except Exception as e: self.cleanup() raise KiwiSetupIntermediateConfigError( '%s: %s' % (type(e).__name__, format(e)) )
def create(self) -> Result: """ Builds a container image which is usually a data archive including container specific metadata. Image types which triggers this builder are: * image="docker" * image="oci" * image="appx" :return: result :rtype: instance of :class:`Result` """ if not self.base_image: log.info('Setting up %s container', self.requested_container_type) container_setup = ContainerSetup.new(self.requested_container_type, self.root_dir, self.container_config) container_setup.setup() else: checksum = Checksum(self.base_image) if not checksum.matches(checksum.md5(), self.base_image_md5): raise KiwiContainerBuilderError( 'base image file {0} checksum validation failed'.format( self.base_image)) log.info('--> Creating container image') container_image = ContainerImage.new(self.requested_container_type, self.root_dir, self.container_config) self.filename = container_image.create(self.filename, self.base_image) Result.verify_image_size(self.runtime_config.get_max_size_constraint(), self.filename) self.result.add(key='container', filename=self.filename, use_for_bundle=True, compress=False, shasum=True) self.result.add(key='image_packages', filename=self.system_setup.export_package_list( self.target_dir), use_for_bundle=True, compress=False, shasum=False) self.result.add(key='image_changes', filename=self.system_setup.export_package_changes( self.target_dir), use_for_bundle=True, compress=True, shasum=False) self.result.add(key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir), use_for_bundle=True, compress=False, shasum=False) return self.result
def create_install_pxe_archive(self): """ Create an oem install tar archive suitable for installing a disk image via the network using the PXE boot protocol. The archive contains the raw disk image and its checksum as well as an install initrd and kernel plus the required kernel commandline information which needs to be added as append line in the pxelinux config file on the boot server Image types which triggers this builder are: * installpxe="true|false" """ self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.', dir=self.target_dir) # the system image is transfered as xz compressed variant log.info('xz compressing disk image') pxe_image_filename = ''.join( [self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.xz']) compress = Compress(source_filename=self.diskname, keep_source_on_compress=True) compress.xz(self.xz_options) Command.run(['mv', compress.compressed_filename, pxe_image_filename]) # the system image transfer is checked against a checksum log.info('Creating disk image checksum') pxe_md5_filename = ''.join( [self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.md5']) checksum = Checksum(self.diskname) checksum.md5(pxe_md5_filename) # the kiwi initrd code triggers the install by trigger files self._create_pxe_install_trigger_files() # create pxe config append information # this information helps to configure the boot server correctly append_filename = ''.join( [self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.append']) cmdline = 'pxe=1' custom_cmdline = self.xml_state.build_type.get_kernelcmdline() if custom_cmdline: cmdline += ' ' + custom_cmdline with open(append_filename, 'w') as append: append.write('%s\n' % cmdline) # create initrd for pxe install log.info('Creating pxe install boot image') self._create_pxe_install_kernel_and_initrd() # create pxe install tarball log.info('Creating pxe install archive') archive = ArchiveTar(self.pxename.replace('.xz', '')) archive.create_xz_compressed(self.pxe_dir, xz_options=self.xz_options)
def _cleanup_intermediate_config(self): # delete kiwi copied config files config_files_to_delete = [] for config in self.cleanup_files: config_file = self.root_dir + config shasum_file = config_file.replace('.kiwi', '.sha') config_files_to_delete.append(config_file) config_files_to_delete.append(shasum_file) checksum = Checksum(config_file) if not checksum.matches(checksum.sha256(), shasum_file): message = dedent('''\n Modifications to intermediate config file detected The file: {0} is a copy from the host system and symlinked to its origin in the image root during build time. Modifications to this file by e.g script code will not have any effect because the file gets restored by one of the following actions: 1. A package during installation provides it 2. A custom version of the file is setup as overlay file 3. The file is not provided by install or overlay and will be deleted at the end of the build process If you need a custom version of that file please provide it as an overlay file in your image description ''') log.warning(message.format(config_file)) del self.cleanup_files[:] # delete stale symlinks if there are any. normally the package # installation process should have replaced the symlinks with # real files from the packages. On deletion check for the # presence of a config file template and restore it try: for config in self.config_files: config_file = self.root_dir + config if os.path.islink(config_file): Command.run(['rm', '-f', config_file]) self._restore_config_template(config_file) Command.run(['rm', '-f'] + config_files_to_delete) except Exception as issue: log.warning( 'Failed to cleanup intermediate config files: {0}'.format(issue) ) self._restore_intermediate_config_rpmnew_variants()
def setup(self, mock_exists): read_results = [bytes(b''), bytes(b'data'), bytes(b''), bytes(b'data')] def side_effect(arg): print(read_results[0]) return read_results.pop() self.m_open = mock_open() self.m_open.return_value.read.side_effect = side_effect mock_exists.return_value = True self.checksum = Checksum('some-file')
def create(self): """ Create a root archive tarball Build a simple XZ compressed root tarball from the image root tree Image types which triggers this builder are: * image="tbz" :return: result :rtype: instance of :class:`Result` """ supported_archives = Defaults.get_archive_image_types() if self.requested_archive_type not in supported_archives: raise KiwiArchiveSetupError('Unknown archive type: %s' % self.requested_archive_type) if self.requested_archive_type == 'tbz': log.info('Creating XZ compressed tar archive') archive = ArchiveTar(self._target_file_for('tar')) archive.create_xz_compressed( self.root_dir, xz_options=self.xz_options, exclude=Defaults.get_exclude_list_for_root_data_sync()) checksum = Checksum(self.filename) log.info('--> Creating archive checksum') checksum.md5(self.checksum) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.filename) self.result.add(key='root_archive', filename=self.filename, use_for_bundle=True, compress=False, shasum=True) self.result.add(key='root_archive_md5', filename=self.checksum, use_for_bundle=False) self.result.add(key='image_packages', filename=self.system_setup.export_package_list( self.target_dir), use_for_bundle=True, compress=False, shasum=False) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir), use_for_bundle=True, compress=False, shasum=False) return self.result
def setup(self, mock_exists): self.context_manager_mock = mock.Mock() self.file_mock = mock.Mock() self.enter_mock = mock.Mock() self.exit_mock = mock.Mock() self.enter_mock.return_value = self.file_mock setattr(self.context_manager_mock, '__enter__', self.enter_mock) setattr(self.context_manager_mock, '__exit__', self.exit_mock) read_results = [bytes(b''), bytes(b'data')] def side_effect(arg): return read_results.pop() self.file_mock.read.side_effect = side_effect mock_exists.return_value = True self.checksum = Checksum('some-file')
def create(self): """ Builds a container image which is usually a tarball including container specific metadata. Image types which triggers this builder are: * image="docker" """ if not self.base_image: log.info('Setting up %s container', self.requested_container_type) container_setup = ContainerSetup(self.requested_container_type, self.root_dir, self.container_config) container_setup.setup() else: checksum = Checksum(self.base_image) if not checksum.matches(checksum.md5(), self.base_image_md5): raise KiwiContainerBuilderError( 'base image file {0} checksum validation failed'.format( self.base_image)) log.info('--> Creating container image') container_image = ContainerImage(self.requested_container_type, self.root_dir, self.container_config) container_image.create(self.filename, self.base_image) self.result.add(key='container', filename=self.filename, use_for_bundle=True, compress=False, shasum=True) self.result.add(key='image_packages', filename=self.system_setup.export_package_list( self.target_dir), use_for_bundle=True, compress=False, shasum=False) self.result.add(key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir), use_for_bundle=True, compress=False, shasum=False) return self.result
def create(self): """ Create a root archive tarball Build a simple XZ compressed root tarball from the image root tree Image types which triggers this builder are: * image="tbz" """ supported_archives = Defaults.get_archive_image_types() if self.requested_archive_type not in supported_archives: raise KiwiArchiveSetupError('Unknown archive type: %s' % self.requested_archive_type) if self.requested_archive_type == 'tbz': log.info('Creating XZ compressed tar archive') archive = ArchiveTar(self._target_file_for('tar')) archive.create_xz_compressed(self.root_dir) checksum = Checksum(self.filename) log.info('--> Creating archive checksum') checksum.md5(self.checksum) self.result.add(key='root_archive', filename=self.filename, use_for_bundle=True, compress=False, shasum=True) self.result.add(key='root_archive_md5', filename=self.checksum, use_for_bundle=False) self.result.add(key='image_packages', filename=self.system_setup.export_rpm_package_list( self.target_dir), use_for_bundle=True, compress=False, shasum=False) self.result.add( key='image_verified', filename=self.system_setup.export_rpm_package_verification( self.target_dir), use_for_bundle=True, compress=False, shasum=False) return self.result
def _make_checksum(self, image): checksum = Checksum(image) checksum.md5(''.join([image, '.md5']))
def test_checksum_file_not_found(self): Checksum('some-file')
def create_install_iso(self): """ Create an install ISO from the disk_image as hybrid ISO bootable via legacy BIOS, EFI and as disk from Stick Image types which triggers this builder are: * installiso="true|false" * installstick="true|false" """ self.media_dir = mkdtemp( prefix='kiwi_install_media.', dir=self.target_dir ) # custom iso metadata self.custom_iso_args = { 'create_options': [ '-V', Defaults.get_install_volume_id(), '-A', self.mbrid.get_id() ] } # the system image transfer is checked against a checksum log.info('Creating disk image checksum') self.squashed_contents = mkdtemp( prefix='kiwi_install_squashfs.', dir=self.target_dir ) checksum = Checksum(self.diskname) checksum.md5(self.squashed_contents + '/' + self.md5name) # the system image name is stored in a config file self._write_install_image_info_to_iso_image() if self.initrd_system == 'kiwi': self._write_install_image_info_to_boot_image() # the system image is stored as squashfs embedded file log.info('Creating squashfs embedded disk image') Command.run( [ 'cp', '-l', self.diskname, self.squashed_contents + '/' + self.squashed_diskname ] ) squashed_image_file = ''.join( [ self.target_dir, '/', self.squashed_diskname, '.squashfs' ] ) squashed_image = FileSystemSquashFs( device_provider=None, root_dir=self.squashed_contents ) squashed_image.create_on_file(squashed_image_file) Command.run( ['mv', squashed_image_file, self.media_dir] ) # setup bootloader config to boot the ISO via isolinux log.info('Setting up install image bootloader configuration') bootloader_config_isolinux = BootLoaderConfig( 'isolinux', self.xml_state, self.media_dir ) bootloader_config_isolinux.setup_install_boot_images( mbrid=None, lookup_path=self.boot_image_task.boot_root_directory ) bootloader_config_isolinux.setup_install_image_config( mbrid=None ) bootloader_config_isolinux.write() # setup bootloader config to boot the ISO via EFI bootloader_config_grub = BootLoaderConfig( 'grub2', self.xml_state, self.media_dir, { 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) } ) bootloader_config_grub.setup_install_boot_images( mbrid=self.mbrid, lookup_path=self.root_dir ) bootloader_config_grub.setup_install_image_config( mbrid=self.mbrid ) bootloader_config_grub.write() # create initrd for install image log.info('Creating install image boot image') self._create_iso_install_kernel_and_initrd() # the system image initrd is stored to allow kexec self._copy_system_image_initrd_to_iso_image() # create iso filesystem from media_dir log.info('Creating ISO filesystem') iso_image = FileSystemIsoFs( device_provider=None, root_dir=self.media_dir, custom_args=self.custom_iso_args ) iso_header_offset = iso_image.create_on_file(self.isoname) # make it hybrid Iso.create_hybrid( iso_header_offset, self.mbrid, self.isoname, self.firmware.efi_mode() )
def process(self): """ Create result bundle from the image build results in the specified target directory. Each result image will contain the specified bundle identifier as part of its filename. Uncompressed image files will also become xz compressed and a sha sum will be created from every result image """ self.manual = Help() if self._help(): return if self.command_args['--package-as-rpm']: Privileges.check_for_root_permissions() # load serialized result object from target directory result_directory = os.path.abspath(self.command_args['--target-dir']) bundle_directory = os.path.abspath(self.command_args['--bundle-dir']) if result_directory == bundle_directory: raise KiwiBundleError( 'Bundle directory must be different from target directory') log.info('Bundle build results from %s', result_directory) result = Result.load(result_directory + '/kiwi.result') image_version = result.xml_state.get_image_version() image_name = result.xml_state.xml_data.get_name() image_description = result.xml_state.get_description_section() ordered_results = OrderedDict(sorted(result.get_results().items())) # hard link bundle files, compress and build checksum if self.command_args['--package-as-rpm']: Path.wipe(bundle_directory) if not os.path.exists(bundle_directory): Path.create(bundle_directory) bundle_file_format_name = '' if 'bundle_format' in ordered_results: bundle_format = ordered_results['bundle_format'] tags = bundle_format['tags'] bundle_file_format_name = bundle_format['pattern'] # Insert image name bundle_file_format_name = bundle_file_format_name.replace( '%N', tags.N) # Insert Concatenated profile name (_) bundle_file_format_name = bundle_file_format_name.replace( '%P', tags.P) # Insert Architecture name bundle_file_format_name = bundle_file_format_name.replace( '%A', tags.A) # Insert Image build type name bundle_file_format_name = bundle_file_format_name.replace( '%T', tags.T) # Insert Image Major version number bundle_file_format_name = bundle_file_format_name.replace( '%M', format(tags.M)) # Insert Image Minor version number bundle_file_format_name = bundle_file_format_name.replace( '%m', format(tags.m)) # Insert Image Patch version number bundle_file_format_name = bundle_file_format_name.replace( '%p', format(tags.p)) # Insert Bundle ID bundle_file_format_name = bundle_file_format_name.replace( '%I', self.command_args['--id']) del (ordered_results['bundle_format']) for result_file in list(ordered_results.values()): if result_file.use_for_bundle: extension = result_file.filename.split('.').pop() if bundle_file_format_name: bundle_file_basename = '.'.join( [bundle_file_format_name, extension]) else: bundle_file_basename = os.path.basename( result_file.filename) # The bundle id is only taken into account for image results # which contains the image version appended in its file name part_name = list( bundle_file_basename.partition(image_name)) bundle_file_basename = ''.join([ part_name[0], part_name[1], part_name[2].replace( image_version, image_version + '-' + self.command_args['--id']) ]) log.info('Creating %s', bundle_file_basename) bundle_file = ''.join( [bundle_directory, '/', bundle_file_basename]) Command.run(['cp', result_file.filename, bundle_file]) if result_file.compress: log.info('--> XZ compressing') compress = Compress(bundle_file) compress.xz(self.runtime_config.get_xz_options()) bundle_file = compress.compressed_filename if self.command_args['--zsync-source'] and result_file.shasum: # Files with a checksum are considered to be image files # and are therefore eligible to be provided via the # requested Partial/differential file download based on # zsync zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK) if zsyncmake: log.info('--> Creating zsync control file') Command.run([ zsyncmake, '-e', '-u', os.sep.join([ self.command_args['--zsync-source'], os.path.basename(bundle_file) ]), '-o', bundle_file + '.zsync', bundle_file ]) else: log.warning( '--> zsyncmake missing, zsync setup skipped') if result_file.shasum: log.info('--> Creating SHA 256 sum') checksum = Checksum(bundle_file) with open(bundle_file + '.sha256', 'w') as shasum: shasum.write('{0} {1}{2}'.format( checksum.sha256(), os.path.basename(bundle_file), os.linesep)) if self.command_args['--package-as-rpm']: ResultBundleTask._build_rpm_package( bundle_directory, bundle_file_format_name or image_name, image_version, image_description.specification, list(glob.iglob(f'{bundle_directory}/*')))
class TestChecksum: @patch('os.path.exists') def setup(self, mock_exists): self.ascii = encoding.getregentry().name read_results = [bytes(b''), bytes(b'data'), bytes(b''), bytes(b'data')] def side_effect(arg): print(read_results[0]) return read_results.pop() self.m_open = mock_open() self.m_open.return_value.read.side_effect = side_effect mock_exists.return_value = True self.checksum = Checksum('some-file') def test_checksum_file_not_found(self): with raises(KiwiFileNotFound): Checksum('some-file') @patch('os.path.exists') def test_matches_checksum_file_does_not_exist(self, mock_exists): mock_exists.return_value = False assert self.checksum.matches('sum', 'some-file') is False @patch('os.path.exists') def test_matches(self, mock_exists): mock_exists.return_value = True self.m_open.return_value.read.side_effect = None self.m_open.return_value.read.return_value = 'sum' with patch('builtins.open', self.m_open, create=True): assert self.checksum.matches('sum', 'some-file') is True self.m_open.assert_called_once_with( 'some-file', encoding=self.ascii ) with patch('builtins.open', self.m_open, create=True): assert self.checksum.matches('foo', 'some-file') is False @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') def test_md5_xz(self, mock_size, mock_md5, mock_compress, mock_which): checksum = Mock checksum.uncompressed_filename = 'some-file-uncompressed' mock_which.return_value = 'factor' compress = Mock() digest = Mock() digest.block_size = 1024 digest._calculate_hash_hexdigest = Mock( return_value=checksum ) digest.hexdigest = Mock( return_value='sum' ) compress.get_format = Mock( return_value='xz' ) mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress with patch('builtins.open', self.m_open, create=True): self.checksum.md5('outfile') assert self.m_open.call_args_list == [ call('some-file', 'rb'), call('some-file-uncompressed', 'rb'), call('outfile', encoding=self.ascii, mode='w') ] self.m_open.return_value.write.assert_called_once_with( 'sum 163968 8192 163968 8192\n' ) @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') def test_md5( self, mock_size, mock_md5, mock_compress, mock_which ): mock_which.return_value = 'factor' compress = Mock() digest = Mock() digest.block_size = 1024 digest.hexdigest = Mock( return_value='sum' ) compress.get_format = Mock( return_value=None ) mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress with patch('builtins.open', self.m_open, create=True): self.checksum.md5('outfile') assert self.m_open.call_args_list == [ call('some-file', 'rb'), call('outfile', encoding=self.ascii, mode='w') ] self.m_open.return_value.write.assert_called_once_with( 'sum 163968 8192\n' ) @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.sha256') @patch('os.path.getsize') def test_sha256( self, mock_size, mock_sha256, mock_compress, mock_which ): mock_which.return_value = 'factor' compress = Mock() digest = Mock() digest.block_size = 1024 digest.hexdigest = Mock( return_value='sum' ) compress.get_format = Mock( return_value=None ) mock_size.return_value = 1343225856 mock_sha256.return_value = digest mock_compress.return_value = compress with patch('builtins.open', self.m_open, create=True): self.checksum.sha256('outfile') assert self.m_open.call_args_list == [ call('some-file', 'rb'), call('outfile', encoding=self.ascii, mode='w') ] self.m_open.return_value.write.assert_called_once_with( 'sum 163968 8192\n' ) @patch('hashlib.sha256') def test_sha256_plain(self, mock_sha256): digest = Mock() digest.block_size = 1024 digest.hexdigest = Mock( return_value='sum' ) mock_sha256.return_value = digest with patch('builtins.open', self.m_open, create=True): assert self.checksum.sha256() == digest.hexdigest.return_value @patch('hashlib.md5') def test_md5_plain(self, mock_md5): digest = Mock() digest.block_size = 1024 digest.hexdigest = Mock( return_value='sum' ) mock_md5.return_value = digest with patch('builtins.open', self.m_open, create=True): assert self.checksum.md5() == digest.hexdigest.return_value
def create(self): """ Build a pxe image set consisting out of a boot image(initrd) plus its appropriate kernel files and the root filesystem image with a checksum. The result can be used within the kiwi PXE boot infrastructure Image types which triggers this builder are: * image="pxe" :raises KiwiPxeBootImageError: if no kernel or hipervisor is found in boot image tree :return: result :rtype: instance of :class:`Result` """ log.info('Creating PXE root filesystem image') self.filesystem.create() os.rename(self.filesystem.filename, self.image_name) self.image = self.image_name if self.compressed: log.info('xz compressing root filesystem image') compress = Compress(self.image) compress.xz(self.xz_options) self.image = compress.compressed_filename log.info('Creating PXE root filesystem MD5 checksum') checksum = Checksum(self.image) checksum.md5(self.checksum_name) # prepare boot(initrd) root system log.info('Creating PXE boot image') self.boot_image_task.prepare() # export modprobe configuration to boot image self.system_setup.export_modprobe_setup( self.boot_image_task.boot_root_directory) # extract kernel from boot(initrd) root system kernel = Kernel(self.boot_image_task.boot_root_directory) kernel_data = kernel.get_kernel() if kernel_data: self.kernel_filename = ''.join([ os.path.basename(self.image_name), '-', kernel_data.version, '.kernel' ]) kernel.copy_kernel(self.target_dir, self.kernel_filename) else: raise KiwiPxeBootImageError( 'No kernel in boot image tree %s found' % self.boot_image_task.boot_root_directory) # extract hypervisor from boot(initrd) root system if self.xen_server: kernel_data = kernel.get_xen_hypervisor() if kernel_data: self.hypervisor_filename = ''.join( [os.path.basename(self.image_name), '-', kernel_data.name]) kernel.copy_xen_hypervisor(self.target_dir, self.hypervisor_filename) self.result.add(key='xen_hypervisor', filename=self.target_dir + '/' + self.hypervisor_filename, use_for_bundle=True, compress=False, shasum=True) else: raise KiwiPxeBootImageError( 'No hypervisor in boot image tree %s found' % self.boot_image_task.boot_root_directory) # create initrd for pxe boot self.boot_image_task.create_initrd() # put results into a tarball if not self.xz_options: self.xz_options = Defaults.get_xz_compression_options() bash_command = ['tar', '-C', self.target_dir, '-c', '--to-stdout'] + [ self.kernel_filename, os.path.basename(self.boot_image_task.initrd_filename), os.path.basename(self.image), os.path.basename(self.checksum_name) ] + ['|', 'xz', '-f'] + self.xz_options + ['>', self.archive_name] Command.run(['bash', '-c', ' '.join(bash_command)]) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.archive_name) # store results self.result.add( key='pxe_archive', filename=self.archive_name, use_for_bundle=True, compress=self.runtime_config.get_bundle_compression(default=False), shasum=True) # create image root metadata self.result.add(key='image_packages', filename=self.system_setup.export_package_list( self.target_dir), use_for_bundle=True, compress=False, shasum=False) self.result.add(key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir), use_for_bundle=True, compress=False, shasum=False) return self.result
def create(self): """ Builds a container image which is usually a tarball including container specific metadata. Image types which triggers this builder are: * image="docker" :return: result :rtype: instance of :class:`Result` """ if not self.base_image: log.info( 'Setting up %s container', self.requested_container_type ) container_setup = ContainerSetup( self.requested_container_type, self.root_dir, self.container_config ) container_setup.setup() else: checksum = Checksum(self.base_image) if not checksum.matches(checksum.md5(), self.base_image_md5): raise KiwiContainerBuilderError( 'base image file {0} checksum validation failed'.format( self.base_image ) ) log.info( '--> Creating container image' ) container_image = ContainerImage( self.requested_container_type, self.root_dir, self.container_config ) self.filename = container_image.create( self.filename, self.base_image ) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.filename ) self.result.add( key='container', filename=self.filename, use_for_bundle=True, compress=False, shasum=True ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result
def process(self): """ Create result bundle from the image build results in the specified target directory. Each result image will contain the specified bundle identifier as part of its filename. Uncompressed image files will also become xz compressed and a sha sum will be created from every result image """ self.manual = Help() if self._help(): return # load serialized result object from target directory result_directory = os.path.abspath(self.command_args['--target-dir']) bundle_directory = os.path.abspath(self.command_args['--bundle-dir']) if result_directory == bundle_directory: raise KiwiBundleError( 'Bundle directory must be different from target directory') log.info('Bundle build results from %s', result_directory) result = Result.load(result_directory + '/kiwi.result') image_version = result.xml_state.get_image_version() image_name = result.xml_state.xml_data.get_name() ordered_results = OrderedDict(sorted(result.get_results().items())) # hard link bundle files, compress and build checksum if not os.path.exists(bundle_directory): Path.create(bundle_directory) for result_file in list(ordered_results.values()): if result_file.use_for_bundle: bundle_file_basename = os.path.basename(result_file.filename) # The bundle id is only taken into account for image results # which contains the image version appended in its file name part_name = list(bundle_file_basename.partition(image_name)) bundle_file_basename = ''.join([ part_name[0], part_name[1], part_name[2].replace( image_version, image_version + '-' + self.command_args['--id']) ]) log.info('Creating %s', bundle_file_basename) bundle_file = ''.join( [bundle_directory, '/', bundle_file_basename]) Command.run(['cp', result_file.filename, bundle_file]) if result_file.compress: log.info('--> XZ compressing') compress = Compress(bundle_file) compress.xz(self.runtime_config.get_xz_options()) bundle_file = compress.compressed_filename if self.command_args['--zsync-source'] and result_file.shasum: # Files with a checksum are considered to be image files # and are therefore eligible to be provided via the # requested Partial/differential file download based on # zsync zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK) if zsyncmake: log.info('--> Creating zsync control file') Command.run([ zsyncmake, '-e', '-u', os.sep.join([ self.command_args['--zsync-source'], os.path.basename(bundle_file) ]), '-o', bundle_file + '.zsync', bundle_file ]) else: log.warning( '--> zsyncmake missing, zsync setup skipped') if result_file.shasum: log.info('--> Creating SHA 256 sum') checksum = Checksum(bundle_file) with open(bundle_file + '.sha256', 'w') as shasum: shasum.write('{0} {1}'.format( checksum.sha256(), os.path.basename(bundle_file)))
def create_install_iso(self): """ Create an install ISO from the disk_image as hybrid ISO bootable via legacy BIOS, EFI and as disk from Stick Image types which triggers this builder are: * installiso="true|false" * installstick="true|false" """ self.media_dir = mkdtemp( prefix='kiwi_install_media.', dir=self.target_dir ) # custom iso metadata self.custom_iso_args = { 'create_options': [ '-V', '"KIWI Installation System"', '-A', self.mbrid.get_id() ] } # the system image transfer is checked against a checksum log.info('Creating disk image checksum') self.squashed_contents = mkdtemp( prefix='kiwi_install_squashfs.', dir=self.target_dir ) checksum = Checksum(self.diskname) checksum.md5(self.squashed_contents + '/' + self.md5name) # the kiwi initrd code triggers the install by trigger files self._create_iso_install_trigger_files() # the system image is stored as squashfs embedded file log.info('Creating squashfs embedded disk image') Command.run( [ 'cp', '-l', self.diskname, self.squashed_contents + '/' + self.squashed_diskname ] ) squashed_image_file = ''.join( [ self.target_dir, '/', self.squashed_diskname, '.squashfs' ] ) squashed_image = FileSystemSquashFs( device_provider=None, root_dir=self.squashed_contents ) squashed_image.create_on_file(squashed_image_file) Command.run( ['mv', squashed_image_file, self.media_dir] ) # setup bootloader config to boot the ISO via isolinux log.info('Setting up install image bootloader configuration') bootloader_config_isolinux = BootLoaderConfig( 'isolinux', self.xml_state, self.media_dir ) bootloader_config_isolinux.setup_install_boot_images( mbrid=None, lookup_path=self.boot_image_task.boot_root_directory ) bootloader_config_isolinux.setup_install_image_config( mbrid=None ) bootloader_config_isolinux.write() # setup bootloader config to boot the ISO via EFI bootloader_config_grub = BootLoaderConfig( 'grub2', self.xml_state, self.media_dir, { 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) } ) bootloader_config_grub.setup_install_boot_images( mbrid=self.mbrid, lookup_path=self.boot_image_task.boot_root_directory ) bootloader_config_grub.setup_install_image_config( mbrid=self.mbrid ) bootloader_config_grub.write() # create initrd for install image log.info('Creating install image boot image') self._create_iso_install_kernel_and_initrd() # create iso filesystem from media_dir log.info('Creating ISO filesystem') iso_image = FileSystemIsoFs( device_provider=None, root_dir=self.media_dir, custom_args=self.custom_iso_args ) iso_header_offset = iso_image.create_on_file(self.isoname) # make it hybrid Iso.create_hybrid( iso_header_offset, self.mbrid, self.isoname )
def create(self) -> Result: """ Build a component image consisting out of a boot image(initrd) plus its appropriate kernel files and the root filesystem image with a checksum. Image types which triggers this builder are: * image="kis" * image="pxe" :raises KiwiKisBootImageError: if no kernel or hipervisor is found in boot image tree :return: result :rtype: instance of :class:`Result` """ if self.filesystem: log.info('Creating root filesystem image') self.filesystem.create() os.rename( self.filesystem.filename, self.image_name ) self.image = self.image_name if self.compressed: log.info('xz compressing root filesystem image') compress = Compress(self.image) compress.xz(self.xz_options) self.image = compress.compressed_filename log.info('Creating root filesystem MD5 checksum') checksum = Checksum(self.image) checksum.md5(self.checksum_name) # prepare boot(initrd) root system log.info('Creating boot image') self.boot_image_task.prepare() # export modprobe configuration to boot image self.system_setup.export_modprobe_setup( self.boot_image_task.boot_root_directory ) # extract kernel from boot(initrd) root system kernel = Kernel(self.boot_image_task.boot_root_directory) kernel_data = kernel.get_kernel() if kernel_data: self.kernel_filename = ''.join( [ os.path.basename(self.image_name), '-', kernel_data.version, '.kernel' ] ) kernel.copy_kernel( self.target_dir, self.kernel_filename ) else: raise KiwiKisBootImageError( 'No kernel in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # extract hypervisor from boot(initrd) root system if self.xen_server: hypervisor_data = kernel.get_xen_hypervisor() if hypervisor_data: self.hypervisor_filename = ''.join( [ os.path.basename(self.image_name), '-', hypervisor_data.name ] ) kernel.copy_xen_hypervisor( self.target_dir, self.hypervisor_filename ) self.result.add( key='xen_hypervisor', filename=self.target_dir + '/' + self.hypervisor_filename, use_for_bundle=True, compress=False, shasum=True ) else: raise KiwiKisBootImageError( 'No hypervisor in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # create initrd self.boot_image_task.create_initrd() # create append information # this information helps to configure the deployment infrastructure if self.filesystem and self.filesystem.root_uuid \ and self.initrd_system == 'dracut': cmdline = 'root=UUID={}'.format(self.filesystem.root_uuid) if self.custom_cmdline: cmdline += ' {}'.format(self.custom_cmdline) with open(self.append_file, 'w') as append: append.write(cmdline) # put results into a tarball if not self.xz_options: self.xz_options = Defaults.get_xz_compression_options() kis_tarball_files = [ self.kernel_filename, os.path.basename(self.boot_image_task.initrd_filename), os.path.basename(self.checksum_name), ] if self.image: kis_tarball_files.append(os.path.basename(self.image)) if self.filesystem and self.filesystem.root_uuid \ and self.initrd_system == 'dracut': kis_tarball_files.append(os.path.basename(self.append_file)) kis_tarball = ArchiveTar( self.archive_name, create_from_file_list=True, file_list=kis_tarball_files ) if self.compressed: self.archive_name = kis_tarball.create(self.target_dir) else: self.archive_name = kis_tarball.create_xz_compressed( self.target_dir, xz_options=self.xz_options ) Result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.archive_name ) # store results self.result.add( key='kis_archive', filename=self.archive_name, use_for_bundle=True, compress=self.runtime_config.get_bundle_compression( default=False ), shasum=True ) # create image root metadata self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_changes', filename=self.system_setup.export_package_changes( self.target_dir ), use_for_bundle=True, compress=True, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result
def create(self): """ Create a root archive tarball Build a simple XZ compressed root tarball from the image root tree Image types which triggers this builder are: * image="tbz" """ supported_archives = Defaults.get_archive_image_types() if self.requested_archive_type not in supported_archives: raise KiwiArchiveSetupError( 'Unknown archive type: %s' % self.requested_archive_type ) if self.requested_archive_type == 'tbz': log.info('Creating XZ compressed tar archive') archive = ArchiveTar( self._target_file_for('tar') ) archive.create_xz_compressed( self.root_dir, xz_options=self.xz_options, exclude=Defaults.get_exclude_list_for_root_data_sync() ) checksum = Checksum(self.filename) log.info('--> Creating archive checksum') checksum.md5(self.checksum) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.filename ) self.result.add( key='root_archive', filename=self.filename, use_for_bundle=True, compress=False, shasum=True ) self.result.add( key='root_archive_md5', filename=self.checksum, use_for_bundle=False ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result
def test_checksum_file_not_found(self): with raises(KiwiFileNotFound): Checksum('some-file')
class TestChecksum(object): @patch('os.path.exists') def setup(self, mock_exists): self.context_manager_mock = mock.Mock() self.file_mock = mock.Mock() self.enter_mock = mock.Mock() self.exit_mock = mock.Mock() self.enter_mock.return_value = self.file_mock setattr(self.context_manager_mock, '__enter__', self.enter_mock) setattr(self.context_manager_mock, '__exit__', self.exit_mock) read_results = [bytes(b''), bytes(b'data')] def side_effect(arg): return read_results.pop() self.file_mock.read.side_effect = side_effect mock_exists.return_value = True self.checksum = Checksum('some-file') @raises(KiwiFileNotFound) def test_checksum_file_not_found(self): Checksum('some-file') @patch('os.path.exists') def test_matches_checksum_file_does_not_exist(self, mock_exists): mock_exists.return_value = False assert self.checksum.matches('sum', 'some-file') is False @patch('os.path.exists') @patch_open def test_matches(self, mock_open, mock_exists): mock_exists.return_value = True mock_open.return_value = self.context_manager_mock self.file_mock.read.side_effect = None self.file_mock.read.return_value = 'sum' assert self.checksum.matches('sum', 'some-file') is True mock_open.assert_called_once_with('some-file') assert self.checksum.matches('foo', 'some-file') is False @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') @patch_open def test_md5_xz( self, mock_open, mock_size, mock_md5, mock_compress, mock_which ): checksum = mock.Mock checksum.uncompressed_filename = 'some-file-uncompressed' mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest._calculate_hash_hexdigest = mock.Mock( return_value=checksum ) digest.hexdigest = mock.Mock( return_value='sum' ) compress.get_format = mock.Mock( return_value='xz' ) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress self.checksum.md5('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('some-file-uncompressed', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with( 'sum 163968 8192 163968 8192\n' ) @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') @patch_open def test_md5( self, mock_open, mock_size, mock_md5, mock_compress, mock_which ): mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock( return_value='sum' ) compress.get_format = mock.Mock( return_value=None ) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress self.checksum.md5('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with( 'sum 163968 8192\n' ) @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.sha256') @patch('os.path.getsize') @patch_open def test_sha256( self, mock_open, mock_size, mock_sha256, mock_compress, mock_which ): mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock( return_value='sum' ) compress.get_format = mock.Mock( return_value=None ) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_sha256.return_value = digest mock_compress.return_value = compress self.checksum.sha256('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with( 'sum 163968 8192\n' ) @patch('hashlib.sha256') @patch_open def test_sha256_plain(self, mock_open, mock_sha256): digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock( return_value='sum' ) mock_sha256.return_value = digest mock_open.return_value = self.context_manager_mock assert self.checksum.sha256() == digest.hexdigest.return_value @patch('hashlib.md5') @patch_open def test_md5_plain(self, mock_open, mock_md5): digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock( return_value='sum' ) mock_md5.return_value = digest mock_open.return_value = self.context_manager_mock assert self.checksum.md5() == digest.hexdigest.return_value
def create_install_iso(self) -> None: """ Create an install ISO from the disk_image as hybrid ISO bootable via legacy BIOS, EFI and as disk from Stick Image types which triggers this builder are: * installiso="true|false" * installstick="true|false" """ self.media_dir = mkdtemp( prefix='kiwi_install_media.', dir=self.target_dir ) # unpack cdroot user files to media dir self.setup.import_cdroot_files(self.media_dir) # custom iso metadata self.custom_iso_args = { 'meta_data': { 'volume_id': self.iso_volume_id, 'mbr_id': self.mbrid.get_id(), 'efi_mode': self.firmware.efi_mode(), 'ofw_mode': self.firmware.ofw_mode() } } # the system image transfer is checked against a checksum log.info('Creating disk image checksum') self.squashed_contents = mkdtemp( prefix='kiwi_install_squashfs.', dir=self.target_dir ) checksum = Checksum(self.diskname) checksum.md5(self.squashed_contents + '/' + self.md5name) # the system image name is stored in a config file self._write_install_image_info_to_iso_image() if self.initrd_system == 'kiwi': self._write_install_image_info_to_boot_image() # the system image is stored as squashfs embedded file log.info('Creating squashfs embedded disk image') Command.run( [ 'cp', '-l', self.diskname, self.squashed_contents + '/' + self.squashed_diskname ] ) squashed_image_file = ''.join( [ self.target_dir, '/', self.squashed_diskname, '.squashfs' ] ) squashed_image = FileSystemSquashFs( device_provider=DeviceProvider(), root_dir=self.squashed_contents, custom_args={ 'compression': self.xml_state.build_type.get_squashfscompression() } ) squashed_image.create_on_file(squashed_image_file) Command.run( ['mv', squashed_image_file, self.media_dir] ) log.info( 'Setting up install image bootloader configuration' ) if self.firmware.efi_mode(): # setup bootloader config to boot the ISO via EFI # This also embedds an MBR and the respective BIOS modules # for compat boot. The complete bootloader setup will be # based on grub bootloader_config = BootLoaderConfig.new( 'grub2', self.xml_state, root_dir=self.root_dir, boot_dir=self.media_dir, custom_args={ 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) } ) bootloader_config.setup_install_boot_images( mbrid=self.mbrid, lookup_path=self.boot_image_task.boot_root_directory ) else: # setup bootloader config to boot the ISO via isolinux. # This allows for booting on x86 platforms in BIOS mode # only. bootloader_config = BootLoaderConfig.new( 'isolinux', self.xml_state, root_dir=self.root_dir, boot_dir=self.media_dir ) IsoToolsBase.setup_media_loader_directory( self.boot_image_task.boot_root_directory, self.media_dir, bootloader_config.get_boot_theme() ) bootloader_config.write_meta_data() bootloader_config.setup_install_image_config( mbrid=self.mbrid ) bootloader_config.write() # create initrd for install image log.info('Creating install image boot image') self._create_iso_install_kernel_and_initrd() # the system image initrd is stored to allow kexec self._copy_system_image_initrd_to_iso_image() # create iso filesystem from media_dir log.info('Creating ISO filesystem') iso_image = FileSystemIsoFs( device_provider=DeviceProvider(), root_dir=self.media_dir, custom_args=self.custom_iso_args ) iso_image.create_on_file(self.isoname) self.boot_image_task.cleanup()
def process(self): """ Create result bundle from the image build results in the specified target directory. Each result image will contain the specified bundle identifier as part of its filename. Uncompressed image files will also become xz compressed and a sha sum will be created from every result image """ self.manual = Help() if self._help(): return # load serialized result object from target directory result_directory = os.path.abspath(self.command_args['--target-dir']) bundle_directory = os.path.abspath(self.command_args['--bundle-dir']) if result_directory == bundle_directory: raise KiwiBundleError( 'Bundle directory must be different from target directory' ) log.info( 'Bundle build results from %s', result_directory ) result = Result.load( result_directory + '/kiwi.result' ) image_version = result.xml_state.get_image_version() ordered_results = OrderedDict(sorted(result.get_results().items())) # hard link bundle files, compress and build checksum if not os.path.exists(bundle_directory): Path.create(bundle_directory) for result_file in list(ordered_results.values()): if result_file.use_for_bundle: bundle_file_basename = os.path.basename(result_file.filename) # The bundle id is only taken into account for image results # which contains the image version in its nane bundle_file_basename = bundle_file_basename.replace( image_version, image_version + '-' + self.command_args['--id'] ) log.info('Creating %s', bundle_file_basename) bundle_file = ''.join( [bundle_directory, '/', bundle_file_basename] ) checksum_file = ''.join( [bundle_directory, '/', bundle_file_basename, '.sha256'] ) Command.run( [ 'cp', result_file.filename, bundle_file ] ) if result_file.compress: log.info('--> XZ compressing') compress = Compress(bundle_file) compress.xz(self.runtime_config.get_xz_options()) bundle_file = compress.compressed_filename checksum_file = compress.compressed_filename + '.sha256' if self.command_args['--zsync-source']: zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK) if zsyncmake: log.info('--> Creating zsync control file') Command.run( [ zsyncmake, '-e', '-u', os.sep.join( [ self.command_args['--zsync-source'], os.path.basename(bundle_file) ] ), '-o', bundle_file + '.zsync', bundle_file ] ) else: log.warning( '--> zsyncmake missing, zsync setup skipped' ) if result_file.shasum: log.info('--> Creating SHA 256 sum') checksum = Checksum(bundle_file) with open(checksum_file, 'w') as shasum: shasum.write(checksum.sha256())
def create_install_iso(self): """ Create an install ISO from the disk_image as hybrid ISO bootable via legacy BIOS, EFI and as disk from Stick Image types which triggers this builder are: * installiso="true|false" * installstick="true|false" """ self.media_dir = mkdtemp(prefix='kiwi_install_media.', dir=self.target_dir) # unpack cdroot user files to media dir self.setup.import_cdroot_files(self.media_dir) # custom iso metadata self.custom_iso_args = { 'meta_data': { 'volume_id': Defaults.get_install_volume_id(), 'mbr_id': self.mbrid.get_id(), 'efi_mode': self.firmware.efi_mode() } } # the system image transfer is checked against a checksum log.info('Creating disk image checksum') self.squashed_contents = mkdtemp(prefix='kiwi_install_squashfs.', dir=self.target_dir) checksum = Checksum(self.diskname) checksum.md5(self.squashed_contents + '/' + self.md5name) # the system image name is stored in a config file self._write_install_image_info_to_iso_image() if self.initrd_system == 'kiwi': self._write_install_image_info_to_boot_image() # the system image is stored as squashfs embedded file log.info('Creating squashfs embedded disk image') Command.run([ 'cp', '-l', self.diskname, self.squashed_contents + '/' + self.squashed_diskname ]) squashed_image_file = ''.join( [self.target_dir, '/', self.squashed_diskname, '.squashfs']) squashed_image = FileSystemSquashFs(device_provider=None, root_dir=self.squashed_contents) squashed_image.create_on_file(squashed_image_file) Command.run(['mv', squashed_image_file, self.media_dir]) # setup bootloader config to boot the ISO via isolinux log.info('Setting up install image bootloader configuration') bootloader_config_isolinux = BootLoaderConfig('isolinux', self.xml_state, self.media_dir) bootloader_config_isolinux.setup_install_boot_images( mbrid=None, lookup_path=self.boot_image_task.boot_root_directory) bootloader_config_isolinux.setup_install_image_config(mbrid=None) bootloader_config_isolinux.write() # setup bootloader config to boot the ISO via EFI bootloader_config_grub = BootLoaderConfig( 'grub2', self.xml_state, self.media_dir, { 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) }) bootloader_config_grub.setup_install_boot_images( mbrid=self.mbrid, lookup_path=self.root_dir) bootloader_config_grub.setup_install_image_config(mbrid=self.mbrid) bootloader_config_grub.write() # create initrd for install image log.info('Creating install image boot image') self._create_iso_install_kernel_and_initrd() # the system image initrd is stored to allow kexec self._copy_system_image_initrd_to_iso_image() # create iso filesystem from media_dir log.info('Creating ISO filesystem') iso_image = FileSystemIsoFs(device_provider=None, root_dir=self.media_dir, custom_args=self.custom_iso_args) iso_image.create_on_file(self.isoname)
def create(self): """ Build a pxe image set consisting out of a boot image(initrd) plus its appropriate kernel files and the root filesystem image with a checksum. The result can be used within the kiwi PXE boot infrastructure Image types which triggers this builder are: * image="pxe" """ log.info('Creating PXE root filesystem image') self.filesystem.create() os.rename( self.filesystem.filename, self.image_name ) self.image = self.image_name if self.compressed: log.info('xz compressing root filesystem image') compress = Compress(self.image) compress.xz(self.xz_options) self.image = compress.compressed_filename log.info('Creating PXE root filesystem MD5 checksum') self.filesystem_checksum = ''.join([self.image, '.md5']) checksum = Checksum(self.image) checksum.md5(self.filesystem_checksum) # prepare boot(initrd) root system log.info('Creating PXE boot image') self.boot_image_task.prepare() # export modprobe configuration to boot image self.system_setup.export_modprobe_setup( self.boot_image_task.boot_root_directory ) # extract kernel from boot(initrd) root system kernel = Kernel(self.boot_image_task.boot_root_directory) kernel_data = kernel.get_kernel() if kernel_data: self.kernel_filename = ''.join( [ os.path.basename(self.image_name), '-', kernel_data.version, '.kernel' ] ) kernel.copy_kernel( self.target_dir, self.kernel_filename ) else: raise KiwiPxeBootImageError( 'No kernel in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # extract hypervisor from boot(initrd) root system if self.xen_server: kernel_data = kernel.get_xen_hypervisor() if kernel_data: self.hypervisor_filename = ''.join( [os.path.basename(self.image_name), '-', kernel_data.name] ) kernel.copy_xen_hypervisor( self.target_dir, self.hypervisor_filename ) self.result.add( key='xen_hypervisor', filename=self.target_dir + '/' + self.hypervisor_filename, use_for_bundle=True, compress=False, shasum=True ) else: raise KiwiPxeBootImageError( 'No hypervisor in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # create initrd for pxe boot self.boot_image_task.create_initrd() # put results into a tarball Command.run( [ 'tar', '-C', self.target_dir, '-cJf', self.archive_name, self.kernel_filename, os.path.basename(self.boot_image_task.initrd_filename), os.path.basename(self.image), os.path.basename(self.filesystem_checksum) ] ) # store results self.result.add( key='pxe_archive', filename=self.archive_name, use_for_bundle=True, compress=False, shasum=True ) # create image root metadata self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) if self.pxedeploy: log.warning( 'Creation of client config file from pxedeploy not implemented' ) return self.result
def create_install_pxe_archive(self): """ Create an oem install tar archive suitable for installing a disk image via the network using the PXE boot protocol. The archive contains: * The raw system image xz compressed * The raw system image checksum metadata file * The append file template for the boot server * The system image initrd for kexec * The install initrd * The kernel Image types which triggers this builder are: * installpxe="true|false" """ self.pxe_dir = mkdtemp( prefix='kiwi_pxe_install_media.', dir=self.target_dir ) # the system image is transfered as xz compressed variant log.info('xz compressing disk image') pxe_image_filename = ''.join( [ self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.xz' ] ) compress = Compress( source_filename=self.diskname, keep_source_on_compress=True ) compress.xz(self.xz_options) Command.run( ['mv', compress.compressed_filename, pxe_image_filename] ) # the system image transfer is checked against a checksum log.info('Creating disk image checksum') pxe_md5_filename = ''.join( [ self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.md5' ] ) checksum = Checksum(self.diskname) checksum.md5(pxe_md5_filename) # the install image name is stored in a config file if self.initrd_system == 'kiwi': self._write_install_image_info_to_boot_image() # the kexec required system image initrd is stored for dracut kiwi-dump if self.initrd_system == 'dracut': boot_names = self.boot_image_task.get_boot_names() system_image_initrd = os.sep.join( [self.root_dir, 'boot', boot_names.initrd_name] ) target_initrd_name = '{0}/{1}.initrd'.format( self.pxe_dir, self.xml_state.xml_data.get_name() ) shutil.copy( system_image_initrd, target_initrd_name ) os.chmod(target_initrd_name, 420) # create pxe config append information # this information helps to configure the boot server correctly append_filename = ''.join( [ self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.append' ] ) if self.initrd_system == 'kiwi': cmdline = 'pxe=1' else: cmdline = ' '.join( [ 'rd.kiwi.install.pxe', 'rd.kiwi.install.image=http://example.com/image.xz' ] ) custom_cmdline = self.xml_state.build_type.get_kernelcmdline() if custom_cmdline: cmdline += ' ' + custom_cmdline with open(append_filename, 'w') as append: append.write('%s\n' % cmdline) # create initrd for pxe install log.info('Creating pxe install boot image') self._create_pxe_install_kernel_and_initrd() # create pxe install tarball log.info('Creating pxe install archive') archive = ArchiveTar( self.pxename.replace('.xz', '') ) archive.create_xz_compressed( self.pxe_dir, xz_options=self.xz_options )
def create(self): """ Build a pxe image set consisting out of a boot image(initrd) plus its appropriate kernel files and the root filesystem image with a checksum. The result can be used within the kiwi PXE boot infrastructure Image types which triggers this builder are: * image="pxe" :raises KiwiPxeBootImageError: if no kernel or hipervisor is found in boot image tree :return: result :rtype: instance of :class:`Result` """ log.info('Creating PXE root filesystem image') self.filesystem.create() os.rename( self.filesystem.filename, self.image_name ) self.image = self.image_name if self.compressed: log.info('xz compressing root filesystem image') compress = Compress(self.image) compress.xz(self.xz_options) self.image = compress.compressed_filename log.info('Creating PXE root filesystem MD5 checksum') self.filesystem_checksum = ''.join([self.image, '.md5']) checksum = Checksum(self.image) checksum.md5(self.filesystem_checksum) # prepare boot(initrd) root system log.info('Creating PXE boot image') self.boot_image_task.prepare() # export modprobe configuration to boot image self.system_setup.export_modprobe_setup( self.boot_image_task.boot_root_directory ) # extract kernel from boot(initrd) root system kernel = Kernel(self.boot_image_task.boot_root_directory) kernel_data = kernel.get_kernel() if kernel_data: self.kernel_filename = ''.join( [ os.path.basename(self.image_name), '-', kernel_data.version, '.kernel' ] ) kernel.copy_kernel( self.target_dir, self.kernel_filename ) else: raise KiwiPxeBootImageError( 'No kernel in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # extract hypervisor from boot(initrd) root system if self.xen_server: kernel_data = kernel.get_xen_hypervisor() if kernel_data: self.hypervisor_filename = ''.join( [os.path.basename(self.image_name), '-', kernel_data.name] ) kernel.copy_xen_hypervisor( self.target_dir, self.hypervisor_filename ) self.result.add( key='xen_hypervisor', filename=self.target_dir + '/' + self.hypervisor_filename, use_for_bundle=True, compress=False, shasum=True ) else: raise KiwiPxeBootImageError( 'No hypervisor in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # create initrd for pxe boot self.boot_image_task.create_initrd() # put results into a tarball if not self.xz_options: self.xz_options = Defaults.get_xz_compression_options() bash_command = [ 'tar', '-C', self.target_dir, '-c', '--to-stdout' ] + [ self.kernel_filename, os.path.basename(self.boot_image_task.initrd_filename), os.path.basename(self.image), os.path.basename(self.filesystem_checksum) ] + [ '|', 'xz', '-f' ] + self.xz_options + [ '>', self.archive_name ] Command.run(['bash', '-c', ' '.join(bash_command)]) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.archive_name ) # store results self.result.add( key='pxe_archive', filename=self.archive_name, use_for_bundle=True, compress=False, shasum=True ) # create image root metadata self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) if self.pxedeploy: log.warning( 'Creation of client config file from pxedeploy not implemented' ) return self.result
class TestChecksum(object): @patch('os.path.exists') def setup(self, mock_exists): self.context_manager_mock = mock.Mock() self.file_mock = mock.Mock() self.enter_mock = mock.Mock() self.exit_mock = mock.Mock() self.enter_mock.return_value = self.file_mock setattr(self.context_manager_mock, '__enter__', self.enter_mock) setattr(self.context_manager_mock, '__exit__', self.exit_mock) read_results = [bytes(b''), bytes(b'data')] def side_effect(arg): return read_results.pop() self.file_mock.read.side_effect = side_effect mock_exists.return_value = True self.checksum = Checksum('some-file') @raises(KiwiFileNotFound) def test_checksum_file_not_found(self): Checksum('some-file') @patch('os.path.exists') def test_matches_checksum_file_does_not_exist(self, mock_exists): mock_exists.return_value = False assert self.checksum.matches('sum', 'some-file') is False @patch('os.path.exists') @patch_open def test_matches(self, mock_open, mock_exists): mock_exists.return_value = True mock_open.return_value = self.context_manager_mock self.file_mock.read.side_effect = None self.file_mock.read.return_value = 'sum' assert self.checksum.matches('sum', 'some-file') is True mock_open.assert_called_once_with('some-file') assert self.checksum.matches('foo', 'some-file') is False @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') @patch_open def test_md5_xz(self, mock_open, mock_size, mock_md5, mock_compress, mock_which): checksum = mock.Mock checksum.uncompressed_filename = 'some-file-uncompressed' mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest._calculate_hash_hexdigest = mock.Mock(return_value=checksum) digest.hexdigest = mock.Mock(return_value='sum') compress.get_format = mock.Mock(return_value='xz') mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress self.checksum.md5('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('some-file-uncompressed', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with( 'sum 163968 8192 163968 8192\n') @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') @patch_open def test_md5(self, mock_open, mock_size, mock_md5, mock_compress, mock_which): mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock(return_value='sum') compress.get_format = mock.Mock(return_value=None) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress self.checksum.md5('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with('sum 163968 8192\n') @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.sha256') @patch('os.path.getsize') @patch_open def test_sha256(self, mock_open, mock_size, mock_sha256, mock_compress, mock_which): mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock(return_value='sum') compress.get_format = mock.Mock(return_value=None) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_sha256.return_value = digest mock_compress.return_value = compress self.checksum.sha256('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with('sum 163968 8192\n') @patch('hashlib.sha256') @patch_open def test_sha256_plain(self, mock_open, mock_sha256): digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock(return_value='sum') mock_sha256.return_value = digest mock_open.return_value = self.context_manager_mock assert self.checksum.sha256() == digest.hexdigest.return_value @patch('hashlib.md5') @patch_open def test_md5_plain(self, mock_open, mock_md5): digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock(return_value='sum') mock_md5.return_value = digest mock_open.return_value = self.context_manager_mock assert self.checksum.md5() == digest.hexdigest.return_value
def create_install_pxe_archive(self): """ Create an oem install tar archive suitable for installing a disk image via the network using the PXE boot protocol. The archive contains: * The raw system image xz compressed * The raw system image checksum metadata file * The append file template for the boot server * The system image initrd for kexec * The install initrd * The kernel Image types which triggers this builder are: * installpxe="true|false" """ self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.', dir=self.target_dir) # the system image is transfered as xz compressed variant log.info('xz compressing disk image') pxe_image_filename = ''.join( [self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.xz']) compress = Compress(source_filename=self.diskname, keep_source_on_compress=True) compress.xz(self.xz_options) Command.run(['mv', compress.compressed_filename, pxe_image_filename]) # the system image transfer is checked against a checksum log.info('Creating disk image checksum') pxe_md5_filename = ''.join( [self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.md5']) checksum = Checksum(self.diskname) checksum.md5(pxe_md5_filename) # the install image name is stored in a config file if self.initrd_system == 'kiwi': self._write_install_image_info_to_boot_image() # the kexec required system image initrd is stored for dracut kiwi-dump if self.initrd_system == 'dracut': boot_names = self.boot_image_task.get_boot_names() system_image_initrd = os.sep.join( [self.root_dir, 'boot', boot_names.initrd_name]) target_initrd_name = '{0}/{1}.initrd'.format( self.pxe_dir, self.xml_state.xml_data.get_name()) shutil.copy(system_image_initrd, target_initrd_name) os.chmod(target_initrd_name, 420) # create pxe config append information # this information helps to configure the boot server correctly append_filename = ''.join( [self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.append']) if self.initrd_system == 'kiwi': cmdline = 'pxe=1' else: cmdline = ' '.join([ 'rd.kiwi.install.pxe', 'rd.kiwi.install.image=http://example.com/image.xz' ]) custom_cmdline = self.xml_state.build_type.get_kernelcmdline() if custom_cmdline: cmdline += ' ' + custom_cmdline with open(append_filename, 'w') as append: append.write('%s\n' % cmdline) # create initrd for pxe install log.info('Creating pxe install boot image') self._create_pxe_install_kernel_and_initrd() # create pxe install tarball log.info('Creating pxe install archive') archive = ArchiveTar(self.pxename.replace('.xz', '')) archive.create_xz_compressed(self.pxe_dir, xz_options=self.xz_options)
def fetch(self, update_check=True): """ Download box from the open build service :param bool update_check: check for box updates True|False """ download = update_check repo_source = self.box_config.get_box_source() if repo_source: repo = SolverRepository.new(Uri(repo_source, 'rpm-md')) packages_file = self.box_config.get_box_packages_file() packages_shasum_file = \ self.box_config.get_box_packages_shasum_file() if update_check and packages_file and packages_shasum_file: local_packages_file = os.sep.join( [self.box_dir, packages_file]) local_packages_shasum_file = os.sep.join( [self.box_dir, packages_shasum_file]) local_packages_file_tmp = self.box_stage.register( local_packages_file) local_packages_shasum_file_tmp = self.box_stage.register( local_packages_shasum_file) repo.download_from_repository(packages_file, local_packages_file_tmp) checksum = Checksum(local_packages_file_tmp) shasum = checksum.sha256() if checksum.matches(shasum, local_packages_shasum_file): download = False else: self._create_packages_checksum( local_packages_shasum_file_tmp, shasum) for box_file in self.box_config.get_box_files(): local_box_file = os.sep.join([self.box_dir, box_file]) if not os.path.exists(local_box_file): download = True if download: log.info('Downloading {0}'.format(box_file)) local_box_file_tmp = self.box_stage.register( local_box_file) repo.download_from_repository(box_file, local_box_file_tmp) if download: self.box_stage.commit() for box_file in self.box_config.get_box_files(): local_box_file = os.sep.join([self.box_dir, box_file]) if box_file.endswith('.qcow2'): self.system = local_box_file if box_file.endswith('.tar.xz'): self.kernel = self._extract_kernel_from_tarball( local_box_file) if self.box_config.use_initrd(): self.initrd = self._extract_initrd_from_tarball( local_box_file) return self.vm_setup_type( system=self.system, kernel=self.kernel, initrd=self.initrd, append='root={0} console={1} {2}'.format( self.box_config.get_box_root(), self.box_config.get_box_console(), self.box_config.get_box_kernel_cmdline()), ram=self.box_config.get_box_memory_mbytes(), smp=self.box_config.get_box_processors())