Пример #1
0
    def create(self, filename, base_image):
        """
        Create compressed oci system container tar archive

        :param string filename: archive file name
        :param string base_image: archive used as a base image
        """
        exclude_list = Defaults.get_exclude_list_for_root_data_sync()
        exclude_list.append('boot')
        exclude_list.append('dev')
        exclude_list.append('sys')
        exclude_list.append('proc')

        if base_image:
            Path.create(self.oci.container_dir)
            image_tar = ArchiveTar(base_image)
            image_tar.extract(self.oci.container_dir)

        self.oci.init_layout(bool(base_image))

        self.oci.unpack()
        self.oci.sync_rootfs(''.join([self.root_dir, os.sep]), exclude_list)
        self.oci.repack(self.oci_config)

        if 'additional_tags' in self.oci_config:
            for tag in self.oci_config['additional_tags']:
                self.oci.add_tag(tag)

        self.oci.set_config(self.oci_config, bool(base_image))

        self.oci.garbage_collect()

        return self.pack_image_to_file(filename)
Пример #2
0
 def test_create_from_dir_with_excludes(self, mock_command):
     archive = ArchiveTar('foo.tar', False)
     archive.create('source-dir', ['foo', 'bar'])
     mock_command.assert_called_once_with([
         'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
         '-f', 'foo.tar', '.', '--exclude', './foo', '--exclude', './bar'
     ])
Пример #3
0
    def import_overlay_files(self,
                             follow_links=False,
                             preserve_owner_group=False):
        """
        Copy overlay files from the image description to
        the image root tree. Supported are a root/ directory
        or a root.tar.gz tarball. The root/ directory takes
        precedence over the tarball

        :param bool follow_links: follow symlinks true|false
        :param bool preserve_owner_group: preserve permissions true|false
        """
        overlay_directory = self.description_dir + '/root/'
        overlay_archive = self.description_dir + '/root.tar.gz'
        if os.path.exists(overlay_directory):
            log.info('Copying user defined files to image tree')
            sync_options = [
                '-r', '-p', '-t', '-D', '-H', '-X', '-A', '--one-file-system'
            ]
            if follow_links:
                sync_options.append('--copy-links')
            else:
                sync_options.append('--links')
            if preserve_owner_group:
                sync_options.append('-o')
                sync_options.append('-g')
            data = DataSync(overlay_directory, self.root_dir)
            data.sync_data(options=sync_options)
        elif os.path.exists(overlay_archive):
            log.info(
                'Extracting user defined files from archive to image tree')
            archive = ArchiveTar(overlay_archive)
            archive.extract(self.root_dir)
Пример #4
0
    def create_image_format(self) -> None:
        """
        Create GCE disk format and manifest
        """
        gce_tar_ball_file_list = []
        temp_image_dir = Temporary(prefix='kiwi_gce_subformat.',
                                   dir=self.target_dir).new_dir()
        diskname = ''.join([
            self.target_dir, '/',
            self.xml_state.xml_data.get_name(), '.' + self.arch,
            '-' + self.xml_state.get_image_version(), '.raw'
        ])
        if self.tag:
            with open(temp_image_dir.name + '/manifest.json', 'w') as manifest:
                manifest.write('{"licenses": ["%s"]}' % self.tag)
            gce_tar_ball_file_list.append('manifest.json')

        Command.run(['cp', diskname, temp_image_dir.name + '/disk.raw'])
        gce_tar_ball_file_list.append('disk.raw')

        archive_name = os.path.basename(
            self.get_target_file_path_for_format(self.image_format))

        # delete the '.gz' suffix from the name. The suffix is appended by
        # the archive creation method depending on the creation type.
        archive_name = archive_name.replace('.gz', '')

        archive = ArchiveTar(filename=self.target_dir + '/' + archive_name,
                             file_list=gce_tar_ball_file_list)
        archive.create_gnu_gzip_compressed(temp_image_dir.name)
Пример #5
0
    def sync_data(self):
        """
        Synchronize data from the given base image to the target root
        directory.
        """
        self.extract_oci_image()
        Command.run([
            'umoci', 'unpack', '--image',
            '{0}:base_layer'.format(self.oci_layout_dir), self.oci_unpack_dir
        ])

        synchronizer = DataSync(
            os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
            ''.join([self.root_dir, os.sep])
        )
        synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])

        # A copy of the uncompressed image and its checksum are
        # kept inside the root_dir in order to ensure the later steps
        # i.e. system create are atomic and don't need any third
        # party archive.
        image_copy = Defaults.get_imported_root_image(self.root_dir)
        Path.create(os.path.dirname(image_copy))
        image_tar = ArchiveTar(image_copy)
        image_tar.create(self.oci_layout_dir)
        self._make_checksum(image_copy)
Пример #6
0
    def sync_data(self):
        """
        Synchronize data from the given base image to the target root
        directory.
        """
        self.extract_oci_image()
        Command.run([
            'umoci', 'unpack', '--image', self.oci_layout_dir,
            self.oci_unpack_dir
        ])

        synchronizer = DataSync(
            os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
            ''.join([self.root_dir, os.sep]))
        synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])

        # A copy of the uncompressed image and its checksum are
        # kept inside the root_dir in order to ensure the later steps
        # i.e. system create are atomic and don't need any third
        # party archive.
        image_copy = Defaults.get_imported_root_image(self.root_dir)
        Path.create(os.path.dirname(image_copy))
        image_tar = ArchiveTar(image_copy)
        image_tar.create(self.oci_layout_dir)
        self._make_checksum(image_copy)
Пример #7
0
class TestArchiveTar(object):
    def setup(self):
        self.archive = ArchiveTar('foo.tar')

    @patch('kiwi.archive.tar.Command.run')
    def test_extract(self, mock_command):
        self.archive.extract('destination')
        mock_command.assert_called_once_with(
            ['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar'])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        self.archive.create('source-dir')
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
            '-f', 'foo.tar', 'foo', 'bar'
        ])

    @patch('kiwi.archive.tar.Command.run')
    def test_create_from_dir_with_excludes(self, mock_command):
        archive = ArchiveTar('foo.tar', False)
        archive.create('source-dir', ['foo', 'bar'])
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
            '-f', 'foo.tar', '.', '--exclude', './foo', '--exclude', './bar'
        ])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_xz_compressed(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        self.archive.create_xz_compressed('source-dir')
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
            '-J', '-f', 'foo.tar.xz', 'foo', 'bar'
        ])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        self.archive.create_gnu_gzip_compressed('source-dir')
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--format=gnu', '-cSz', '-f',
            'foo.tar.gz', 'foo', 'bar'
        ])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_exclude(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        self.archive.create('source-dir', ['foo'])
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
            '-f', 'foo.tar', 'bar'
        ])
Пример #8
0
Файл: oci.py Проект: agraf/kiwi
    def pack_image_to_file(self, filename):
        """
        Packs the oci image into the given filename.

        :param string filename: file name of the resulting packed image
        """
        image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
        oci_tarfile = ArchiveTar(filename.replace('.xz', ''))
        oci_tarfile.create_xz_compressed(image_dir, xz_options=self.xz_options)
Пример #9
0
    def create_install_pxe_archive(self):
        """
        Create an oem install tar archive suitable for installing a
        disk image via the network using the PXE boot protocol.
        The archive contains the raw disk image and its checksum
        as well as an install initrd and kernel plus the required
        kernel commandline information which needs to be added
        as append line in the pxelinux config file on the boot
        server

        Image types which triggers this builder are:

        * installpxe="true|false"
        """
        self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.',
                               dir=self.target_dir)
        # the system image is transfered as xz compressed variant
        log.info('xz compressing disk image')
        pxe_image_filename = ''.join(
            [self.pxe_dir, '/',
             self.xml_state.xml_data.get_name(), '.xz'])
        compress = Compress(source_filename=self.diskname,
                            keep_source_on_compress=True)
        compress.xz(self.xz_options)
        Command.run(['mv', compress.compressed_filename, pxe_image_filename])

        # the system image transfer is checked against a checksum
        log.info('Creating disk image checksum')
        pxe_md5_filename = ''.join(
            [self.pxe_dir, '/',
             self.xml_state.xml_data.get_name(), '.md5'])
        checksum = Checksum(self.diskname)
        checksum.md5(pxe_md5_filename)

        # the kiwi initrd code triggers the install by trigger files
        self._create_pxe_install_trigger_files()

        # create pxe config append information
        # this information helps to configure the boot server correctly
        append_filename = ''.join(
            [self.pxe_dir, '/',
             self.xml_state.xml_data.get_name(), '.append'])
        cmdline = 'pxe=1'
        custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
        if custom_cmdline:
            cmdline += ' ' + custom_cmdline
        with open(append_filename, 'w') as append:
            append.write('%s\n' % cmdline)

        # create initrd for pxe install
        log.info('Creating pxe install boot image')
        self._create_pxe_install_kernel_and_initrd()

        # create pxe install tarball
        log.info('Creating pxe install archive')
        archive = ArchiveTar(self.pxename.replace('.xz', ''))
        archive.create_xz_compressed(self.pxe_dir, xz_options=self.xz_options)
Пример #10
0
    def create(self) -> Result:
        """
        Create a root archive tarball

        Build a simple XZ compressed root tarball from the image root tree

        Image types which triggers this builder are:

        * image="tbz"

        :return: result

        :rtype: instance of :class:`Result`
        """
        supported_archives = Defaults.get_archive_image_types()
        if self.requested_archive_type not in supported_archives:
            raise KiwiArchiveSetupError('Unknown archive type: %s' %
                                        self.requested_archive_type)

        if self.requested_archive_type == 'tbz':
            log.info('Creating XZ compressed tar archive')
            archive = ArchiveTar(self._target_file_for('tar'))
            archive.create_xz_compressed(
                self.root_dir,
                xz_options=self.xz_options,
                exclude=Defaults.get_exclude_list_for_root_data_sync() +
                Defaults.get_exclude_list_from_custom_exclude_files(
                    self.root_dir))
            Result.verify_image_size(
                self.runtime_config.get_max_size_constraint(), self.filename)
            if self.bundle_format:
                self.result.add_bundle_format(self.bundle_format)
            self.result.add(key='root_archive',
                            filename=self.filename,
                            use_for_bundle=True,
                            compress=False,
                            shasum=True)
            self.result.add(key='image_packages',
                            filename=self.system_setup.export_package_list(
                                self.target_dir),
                            use_for_bundle=True,
                            compress=False,
                            shasum=False)
            self.result.add(key='image_changes',
                            filename=self.system_setup.export_package_changes(
                                self.target_dir),
                            use_for_bundle=True,
                            compress=True,
                            shasum=False)
            self.result.add(
                key='image_verified',
                filename=self.system_setup.export_package_verification(
                    self.target_dir),
                use_for_bundle=True,
                compress=False,
                shasum=False)
        return self.result
Пример #11
0
 def test_create_from_dir_with_excludes(self, mock_command):
     archive = ArchiveTar('foo.tar', False)
     archive.create('source-dir', ['foo', 'bar'])
     mock_command.assert_called_once_with(
         [
             'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
             '-c', '-f', 'foo.tar', '.',
             '--exclude', './foo', '--exclude', './bar'
         ]
     )
Пример #12
0
    def pack_image_to_file(self, filename):
        """
        Packs the oci image into the given filename.

        :param string filename: file name of the resulting packed image
        """
        image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
        oci_tarfile = ArchiveTar(filename.replace('.xz', ''))
        oci_tarfile.create_xz_compressed(
            image_dir, xz_options=self.xz_options
        )
Пример #13
0
    def create(self, filename, base_image):
        """
        Create compressed oci system container tar archive

        :param string filename: archive file name
        :param string base_image: archive used as a base image
        """
        exclude_list = Defaults.get_exclude_list_for_root_data_sync()
        exclude_list.append('boot')
        exclude_list.append('dev')
        exclude_list.append('sys')
        exclude_list.append('proc')

        self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
        self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')

        container_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
        container_name = ':'.join([container_dir, self.container_tag])

        if base_image:
            Path.create(container_dir)
            image_tar = ArchiveTar(base_image)
            image_tar.extract(container_dir)
            Command.run([
                'umoci', 'config', '--image',
                '{0}:base_layer'.format(container_dir), '--tag',
                self.container_tag
            ])
        else:
            Command.run(['umoci', 'init', '--layout', container_dir])
            Command.run(['umoci', 'new', '--image', container_name])

        Command.run(
            ['umoci', 'unpack', '--image', container_name, self.oci_root_dir])
        oci_root = DataSync(''.join([self.root_dir, os.sep]),
                            os.sep.join([self.oci_root_dir, 'rootfs']))
        oci_root.sync_data(options=['-a', '-H', '-X', '-A', '--delete'],
                           exclude=exclude_list)
        Command.run(
            ['umoci', 'repack', '--image', container_name, self.oci_root_dir])
        for tag in self.additional_tags:
            Command.run(
                ['umoci', 'config', '--image', container_name, '--tag', tag])
        Command.run(['umoci', 'config'] + self.maintainer + self.user +
                    self.workingdir + self.entry_command +
                    self.entry_subcommand + self.expose_ports + self.volumes +
                    self.environment + self.labels + [
                        '--image', container_name, '--created',
                        datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00')
                    ])
        Command.run(['umoci', 'gc', '--layout', container_dir])

        return self.pack_image_to_file(filename)
Пример #14
0
    def create(self, filename, base_image=None, ensure_empty_tmpdirs=None):
        """
        Create WSL/Appx archive

        :param string filename: archive file name
        :param string base_image: not-supported
        :param string ensure_empty_tmpdirs: not-supported
        """
        exclude_list = Defaults.\
            get_exclude_list_for_root_data_sync() + Defaults.\
            get_exclude_list_from_custom_exclude_files(self.root_dir)
        exclude_list.append('boot')
        exclude_list.append('dev')
        exclude_list.append('sys')
        exclude_list.append('proc')

        # The C code of WSL-DistroLauncher harcodes the name for the
        # root tarball to be install.tar.gz. Thus we have to use this
        # name for the root tarball
        archive_file_name = os.sep.join(
            [self.meta_data_path, 'install.tar']
        )
        archive = ArchiveTar(
            archive_file_name
        )
        archive_file_name = archive.create(
            self.root_dir, exclude=exclude_list
        )
        compressor = Compress(archive_file_name)
        archive_file_name = compressor.gzip()

        filemap_file = Temporary().new_file()
        with open(filemap_file.name, 'w') as filemap:
            filemap.write('[Files]{0}'.format(os.linesep))
            for topdir, dirs, files in sorted(os.walk(self.meta_data_path)):
                for entry in sorted(dirs + files):
                    if entry in files:
                        mapfile = os.sep.join([topdir, entry])
                        mapfile_relative = os.path.relpath(mapfile, start=self.meta_data_path)
                        log.info(
                            'Adding {0} to Appx filemap as relative path {1}'.format(mapfile, mapfile_relative)
                        )
                        filemap.write(
                            '"{0}" "{1}"{2}'.format(
                                mapfile, mapfile_relative, os.linesep
                            )
                        )

        Command.run(
            ['appx', '-o', filename, '-f', filemap_file.name]
        )
        return filename
Пример #15
0
    def pack_image_to_file(self, filename):
        """
        Packs the oci image into the given filename.

        :param string filename: file name of the resulting packed image
        """
        image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
        oci_tarfile = ArchiveTar(filename)
        container_compressor = self.runtime_config.get_container_compression()
        if container_compressor:
            return oci_tarfile.create_xz_compressed(
                image_dir, xz_options=self.runtime_config.get_xz_options())
        else:
            return oci_tarfile.create(image_dir)
Пример #16
0
    def pack_image_to_file(self, filename):
        """
        Packs the oci image into the given filename.

        :param string filename: file name of the resulting packed image
        """
        image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
        oci_tarfile = ArchiveTar(filename)
        container_compressor = self.runtime_config.get_container_compression()
        if container_compressor:
            return oci_tarfile.create_xz_compressed(
                image_dir, xz_options=self.runtime_config.get_xz_options()
            )
        else:
            return oci_tarfile.create(image_dir)
Пример #17
0
    def import_cdroot_files(self, target_dir):
        """
        Copy cdroot files from the image description to the
        specified target directory. Supported is a tar
        archive named config-cdroot.tar[.compression-postfix]

        :param str target_dir: directory to unpack archive to
        """
        glob_match = self.description_dir + '/config-cdroot.tar*'
        for cdroot_archive in sorted(glob.iglob(glob_match)):
            log.info('Extracting ISO user config archive: {0} to: {1}'.format(
                cdroot_archive, target_dir))
            archive = ArchiveTar(cdroot_archive)
            archive.extract(target_dir)
            break
Пример #18
0
 def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
     command = mock.Mock()
     command.output = 'version 1.26.1'
     mock_command.return_value = command
     archive = ArchiveTar('foo.tar')
     mock_os_dir.return_value = ['foo', 'bar']
     assert archive.create('source-dir') == 'foo.tar'
     calls = [
         call(['tar', '--version']),
         call([
             'tar', '-C', 'source-dir', '-c', '-f', 'foo.tar', 'foo', 'bar'
         ])
     ]
     mock_command.assert_has_calls(calls)
     assert mock_command.call_count == 2
Пример #19
0
 def test_create_from_dir_with_excludes(self, mock_command):
     command = mock.Mock()
     command.output = 'version 1.27.0'
     mock_command.return_value = command
     archive = ArchiveTar('foo.tar', False)
     assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
     calls = [
         call(['tar', '--version']),
         call([
             'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
             '-c', '-f', 'foo.tar', '.', '--exclude', './foo', '--exclude',
             './bar'
         ])
     ]
     mock_command.assert_has_calls(calls)
     assert mock_command.call_count == 2
Пример #20
0
    def create(self):
        """
        Create a root archive tarball

        Build a simple XZ compressed root tarball from the image root tree

        Image types which triggers this builder are:

        * image="tbz"
        """
        supported_archives = Defaults.get_archive_image_types()
        if self.requested_archive_type not in supported_archives:
            raise KiwiArchiveSetupError('Unknown archive type: %s' %
                                        self.requested_archive_type)

        if self.requested_archive_type == 'tbz':
            log.info('Creating XZ compressed tar archive')
            archive = ArchiveTar(self._target_file_for('tar'))
            archive.create_xz_compressed(
                self.root_dir,
                xz_options=self.xz_options,
                exclude=Defaults.get_exclude_list_for_root_data_sync())
            checksum = Checksum(self.filename)
            log.info('--> Creating archive checksum')
            checksum.md5(self.checksum)
            self.result.add(key='root_archive',
                            filename=self.filename,
                            use_for_bundle=True,
                            compress=False,
                            shasum=True)
            self.result.add(key='root_archive_md5',
                            filename=self.checksum,
                            use_for_bundle=False)
            self.result.add(key='image_packages',
                            filename=self.system_setup.export_rpm_package_list(
                                self.target_dir),
                            use_for_bundle=True,
                            compress=False,
                            shasum=False)
            self.result.add(
                key='image_verified',
                filename=self.system_setup.export_rpm_package_verification(
                    self.target_dir),
                use_for_bundle=True,
                compress=False,
                shasum=False)
        return self.result
Пример #21
0
 def _install_archives(self, archive_list):
     log.info("Installing archives")
     for archive in archive_list:
         log.info("--> archive: %s", archive)
         description_dir = \
             self.xml_state.xml_data.description_dir
         derived_description_dir = \
             self.xml_state.xml_data.derived_description_dir
         archive_is_absolute = archive.startswith('/')
         if archive_is_absolute:
             archive_file = archive
         else:
             archive_file = '/'.join([description_dir, archive])
         archive_exists = os.path.exists(archive_file)
         if not archive_is_absolute and not archive_exists and derived_description_dir:
             archive_file = '/'.join([derived_description_dir, archive])
         tar = ArchiveTar(archive_file)
         tar.extract(self.root_bind.root_dir)
Пример #22
0
 def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
     command = mock.Mock()
     command.output = 'version 1.26.1'
     mock_command.return_value = command
     archive = ArchiveTar('foo.tar')
     mock_os_dir.return_value = ['foo', 'bar']
     assert archive.create('source-dir') == 'foo.tar'
     calls = [
         call(['tar', '--version']),
         call(
             [
                 'tar', '-C', 'source-dir',
                 '-c', '-f', 'foo.tar', 'foo', 'bar'
             ]
         )
     ]
     mock_command.assert_has_calls(calls)
     assert mock_command.call_count == 2
Пример #23
0
 def test_create_from_dir_with_excludes(self, mock_command):
     command = mock.Mock()
     command.output = 'version 1.27.0'
     mock_command.return_value = command
     archive = ArchiveTar('foo.tar', False)
     assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
     calls = [
         call(['tar', '--version']),
         call(
             [
                 'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
                 '-c', '-f', 'foo.tar', '.',
                 '--exclude', './foo', '--exclude', './bar'
             ]
         )
     ]
     mock_command.assert_has_calls(calls)
     assert mock_command.call_count == 2
Пример #24
0
    def import_overlay_files(
        self, follow_links: bool = False, preserve_owner_group: bool = False
    ) -> None:
        """
        Copy overlay files from the image description to
        the image root tree. Supported are a root/ directory
        or a root.tar.gz tarball. The root/ directory takes
        precedence over the tarball.

        In addition the method also supports profile specific
        overlay files which are searched in a directory of the
        same name as the profile name.

        The overall order for including overlay files is as
        follows:

        1. root/ dir or root.tar.gz
        2. PROFILE_NAME/ dir(s) in the order of the selected
           profiles

        :param bool follow_links: follow symlinks true|false
        :param bool preserve_owner_group: preserve permissions true|false
        """
        overlay_directory = self.description_dir + '/root/'
        overlay_archive = self.description_dir + '/root.tar.gz'
        if os.path.exists(overlay_directory):
            self._sync_overlay_files(
                overlay_directory, follow_links, preserve_owner_group
            )
        elif os.path.exists(overlay_archive):
            log.info('Extracting user defined files from archive to image tree')
            archive = ArchiveTar(overlay_archive)
            archive.extract(self.root_dir)

        for profile in self.xml_state.profiles:
            overlay_directory = os.path.join(
                self.description_dir, profile
            ) + os.sep
            if os.path.exists(overlay_directory):
                self._sync_overlay_files(
                    overlay_directory, follow_links, preserve_owner_group,
                    profile
                )
Пример #25
0
    def create_image_format(self):
        """
        Create GCE disk format and manifest
        """
        gce_tar_ball_file_list = []
        self.temp_image_dir = mkdtemp(
            prefix='kiwi_gce_subformat.', dir=self.target_dir
        )
        diskname = ''.join(
            [
                self.target_dir, '/',
                self.xml_state.xml_data.get_name(),
                '.' + self.arch,
                '-' + self.xml_state.get_image_version(),
                '.raw'
            ]
        )
        if self.tag:
            with open(self.temp_image_dir + '/manifest.json', 'w') as manifest:
                manifest.write('{"licenses": ["%s"]}' % self.tag)
            gce_tar_ball_file_list.append('manifest.json')

        Command.run(
            ['cp', diskname, self.temp_image_dir + '/disk.raw']
        )
        gce_tar_ball_file_list.append('disk.raw')

        archive_name = os.path.basename(
            self.get_target_file_path_for_format(self.image_format)
        )

        # delete the '.gz' suffix from the name. The suffix is appended by
        # the archive creation method depending on the creation type.
        archive_name = archive_name.replace('.gz', '')

        archive = ArchiveTar(
            filename=self.target_dir + '/' + archive_name,
            file_list=gce_tar_ball_file_list
        )
        archive.create_gnu_gzip_compressed(
            self.temp_image_dir
        )
Пример #26
0
    def sync_data(self):
        """
        Synchronize data from the given base image to the target root
        directory.
        """
        self.extract_oci_image()

        oci = OCI('base_layer', self.oci_layout_dir)
        oci.unpack()
        oci.import_rootfs(self.root_dir)

        # A copy of the uncompressed image and its checksum are
        # kept inside the root_dir in order to ensure the later steps
        # i.e. system create are atomic and don't need any third
        # party archive.
        image_copy = Defaults.get_imported_root_image(self.root_dir)
        Path.create(os.path.dirname(image_copy))
        image_tar = ArchiveTar(image_copy)
        image_tar.create(self.oci_layout_dir)
        self._make_checksum(image_copy)
Пример #27
0
    def extract_oci_image(self):
        """
        Extract the image from the provided image file to a temporary
        location to KIWI can work with it.
        """
        if not self.unknown_uri:
            tar = ArchiveTar(self.image_file)
            self.uncompressed_image = mkdtemp(prefix='kiwi_uncompressed.')
            tar.extract(self.uncompressed_image)
            if self.tag:
                skopeo_uri = 'oci:{0}:{1}'.format(self.uncompressed_image,
                                                  self.tag)
            else:
                skopeo_uri = 'oci:{0}'.format(self.uncompressed_image)
        else:
            log.warning('Bypassing base image URI to skopeo tool')
            skopeo_uri = self.unknown_uri

        Command.run([
            'skopeo', 'copy', skopeo_uri, 'oci:{0}'.format(self.oci_layout_dir)
        ])
Пример #28
0
 def _install_archives(self, archive_list):
     log.info("Installing archives")
     for archive in archive_list:
         log.info("--> archive: %s", archive)
         description_dir = \
             self.xml_state.xml_data.description_dir
         derived_description_dir = \
             self.xml_state.xml_data.derived_description_dir
         archive_is_absolute = archive.startswith('/')
         if archive_is_absolute:
             archive_file = archive
         else:
             archive_file = '/'.join(
                 [description_dir, archive]
             )
         archive_exists = os.path.exists(archive_file)
         if not archive_is_absolute and not archive_exists and derived_description_dir:
             archive_file = '/'.join(
                 [derived_description_dir, archive]
             )
         tar = ArchiveTar(archive_file)
         tar.extract(self.root_bind.root_dir)
Пример #29
0
    def extract_oci_image(self):
        """
        Extract the contents from the provided image file to a temporary
        location KIWI can work with.
        """
        if not self.unknown_uri:
            tar = ArchiveTar(self.image_file)
            self.uncompressed_image = mkdtemp(prefix='kiwi_uncompressed.')
            tar.extract(self.uncompressed_image)
            if self.tag:
                skopeo_uri = 'oci:{0}:{1}'.format(
                    self.uncompressed_image, self.tag
                )
            else:
                skopeo_uri = 'oci:{0}'.format(self.uncompressed_image)
        else:
            log.warning('Bypassing base image URI to skopeo tool')
            skopeo_uri = self.unknown_uri

        Command.run([
            'skopeo', 'copy', skopeo_uri,
            'oci:{0}:base_layer'.format(self.oci_layout_dir)
        ])
Пример #30
0
    def import_overlay_files(
        self, follow_links=False, preserve_owner_group=False
    ):
        """
        Copy overlay files from the image description to
        the image root tree. Supported are a root/ directory
        or a root.tar.gz tarball. The root/ directory takes
        precedence over the tarball

        :param bool follow_links: follow symlinks true|false
        :param bool preserve_owner_group: preserve permissions true|false
        """
        overlay_directory = self.description_dir + '/root/'
        overlay_archive = self.description_dir + '/root.tar.gz'
        if os.path.exists(overlay_directory):
            log.info('Copying user defined files to image tree')
            sync_options = [
                '-r', '-p', '-t', '-D', '-H', '-X', '-A', '--one-file-system'
            ]
            if follow_links:
                sync_options.append('--copy-links')
            else:
                sync_options.append('--links')
            if preserve_owner_group:
                sync_options.append('-o')
                sync_options.append('-g')
            data = DataSync(
                overlay_directory, self.root_dir
            )
            data.sync_data(
                options=sync_options
            )
        elif os.path.exists(overlay_archive):
            log.info('Extracting user defined files from archive to image tree')
            archive = ArchiveTar(overlay_archive)
            archive.extract(self.root_dir)
Пример #31
0
class TestArchiveTar(object):
    def setup(self):
        self.archive = ArchiveTar('foo.tar')

    @patch('kiwi.archive.tar.Command.run')
    def test_extract(self, mock_command):
        self.archive.extract('destination')
        mock_command.assert_called_once_with(
            ['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar']
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        self.archive.create('source-dir')
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir',
                '--xattrs', '--xattrs-include=*',
                '-c', '-f', 'foo.tar', 'foo', 'bar'
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    def test_create_from_dir_with_excludes(self, mock_command):
        archive = ArchiveTar('foo.tar', False)
        archive.create('source-dir', ['foo', 'bar'])
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
                '-c', '-f', 'foo.tar', '.',
                '--exclude', './foo', '--exclude', './bar'
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_xz_compressed(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        self.archive.create_xz_compressed('source-dir')
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir',
                '--xattrs', '--xattrs-include=*',
                '-c', '-J', '-f', 'foo.tar.xz', 'foo', 'bar'
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        self.archive.create_gnu_gzip_compressed('source-dir')
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir',
                '--format=gnu', '-cSz', '-f', 'foo.tar.gz', 'foo', 'bar'
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_exclude(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        self.archive.create('source-dir', ['foo'])
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
                '-c', '-f', 'foo.tar', 'bar'
            ]
        )
Пример #32
0
 def setup(self):
     self.archive = ArchiveTar('foo.tar')
Пример #33
0
 def setup(self):
     self.archive = ArchiveTar('foo.tar')
Пример #34
0
    def create(self, filename, base_image):
        """
        Create compressed oci system container tar archive

        :param string filename: archive file name
        :param string base_image: archive used as a base image
        """
        exclude_list = Defaults.get_exclude_list_for_root_data_sync()
        exclude_list.append('boot')
        exclude_list.append('dev')
        exclude_list.append('sys')
        exclude_list.append('proc')

        self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
        self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')

        container_dir = os.sep.join(
            [self.oci_dir, 'umoci_layout']
        )
        container_name = ':'.join(
            [container_dir, self.container_tag]
        )

        if base_image:
            Path.create(container_dir)
            image_tar = ArchiveTar(base_image)
            image_tar.extract(container_dir)
            Command.run([
                'umoci', 'config', '--image',
                '{0}:base_layer'.format(container_dir),
                '--tag', self.container_tag
            ])
        else:
            Command.run(
                ['umoci', 'init', '--layout', container_dir]
            )
            Command.run(
                ['umoci', 'new', '--image', container_name]
            )

        Command.run(
            ['umoci', 'unpack', '--image', container_name, self.oci_root_dir]
        )
        oci_root = DataSync(
            ''.join([self.root_dir, os.sep]),
            os.sep.join([self.oci_root_dir, 'rootfs'])
        )
        oci_root.sync_data(
            options=['-a', '-H', '-X', '-A', '--delete'], exclude=exclude_list
        )
        Command.run(
            ['umoci', 'repack', '--image', container_name, self.oci_root_dir]
        )
        for tag in self.additional_tags:
            Command.run(
                ['umoci', 'config', '--image', container_name, '--tag', tag]
            )
        Command.run(
            [
                'umoci', 'config'
            ] +
            self.maintainer +
            self.user +
            self.workingdir +
            self.entry_command +
            self.entry_subcommand +
            self.expose_ports +
            self.volumes +
            self.environment +
            self.labels +
            [
                '--image', container_name,
                '--created', datetime.utcnow().strftime(
                    '%Y-%m-%dT%H:%M:%S+00:00'
                )
            ]
        )
        Command.run(
            ['umoci', 'gc', '--layout', container_dir]
        )

        return self.pack_image_to_file(filename)
Пример #35
0
    def create_install_pxe_archive(self):
        """
        Create an oem install tar archive suitable for installing a
        disk image via the network using the PXE boot protocol.
        The archive contains:

        * The raw system image xz compressed
        * The raw system image checksum metadata file
        * The append file template for the boot server
        * The system image initrd for kexec
        * The install initrd
        * The kernel

        Image types which triggers this builder are:

        * installpxe="true|false"
        """
        self.pxe_dir = mkdtemp(
            prefix='kiwi_pxe_install_media.', dir=self.target_dir
        )
        # the system image is transfered as xz compressed variant
        log.info('xz compressing disk image')
        pxe_image_filename = ''.join(
            [
                self.pxe_dir, '/',
                self.xml_state.xml_data.get_name(), '.xz'
            ]
        )
        compress = Compress(
            source_filename=self.diskname,
            keep_source_on_compress=True
        )
        compress.xz(self.xz_options)
        Command.run(
            ['mv', compress.compressed_filename, pxe_image_filename]
        )

        # the system image transfer is checked against a checksum
        log.info('Creating disk image checksum')
        pxe_md5_filename = ''.join(
            [
                self.pxe_dir, '/',
                self.xml_state.xml_data.get_name(), '.md5'
            ]
        )
        checksum = Checksum(self.diskname)
        checksum.md5(pxe_md5_filename)

        # the install image name is stored in a config file
        if self.initrd_system == 'kiwi':
            self._write_install_image_info_to_boot_image()

        # the kexec required system image initrd is stored for dracut kiwi-dump
        if self.initrd_system == 'dracut':
            boot_names = self.boot_image_task.get_boot_names()
            system_image_initrd = os.sep.join(
                [self.root_dir, 'boot', boot_names.initrd_name]
            )
            target_initrd_name = '{0}/{1}.initrd'.format(
                self.pxe_dir, self.xml_state.xml_data.get_name()
            )
            shutil.copy(
                system_image_initrd, target_initrd_name
            )
            os.chmod(target_initrd_name, 420)

        # create pxe config append information
        # this information helps to configure the boot server correctly
        append_filename = ''.join(
            [
                self.pxe_dir, '/',
                self.xml_state.xml_data.get_name(), '.append'
            ]
        )
        if self.initrd_system == 'kiwi':
            cmdline = 'pxe=1'
        else:
            cmdline = ' '.join(
                [
                    'rd.kiwi.install.pxe',
                    'rd.kiwi.install.image=http://example.com/image.xz'
                ]
            )
        custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
        if custom_cmdline:
            cmdline += ' ' + custom_cmdline
        with open(append_filename, 'w') as append:
            append.write('%s\n' % cmdline)

        # create initrd for pxe install
        log.info('Creating pxe install boot image')
        self._create_pxe_install_kernel_and_initrd()

        # create pxe install tarball
        log.info('Creating pxe install archive')
        archive = ArchiveTar(
            self.pxename.replace('.xz', '')
        )
        archive.create_xz_compressed(
            self.pxe_dir, xz_options=self.xz_options
        )
Пример #36
0
class TestArchiveTar(object):
    @patch('kiwi.archive.tar.Command.run')
    def setup(self, mock_command):
        command = mock.Mock()
        command.output = 'version 1.27.0'
        mock_command.return_value = command
        self.archive = ArchiveTar('foo.tar')

    @raises(KiwiCommandCapabilitiesError)
    @patch('kiwi.archive.tar.Command.run')
    def test_invalid_tar_command_version(self, mock_command):
        command = mock.Mock()
        command.output = 'version cannot be parsed'
        mock_command.return_value = command
        self.archive = ArchiveTar('foo.tar')

    @patch('kiwi.archive.tar.Command.run')
    def test_extract(self, mock_command):
        self.archive.extract('destination')
        mock_command.assert_called_once_with(
            ['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar']
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create('source-dir') == 'foo.tar'
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir',
                '--xattrs', '--xattrs-include=*',
                '-c', '-f', 'foo.tar', 'foo', 'bar'
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    def test_append_files(self, mock_command):
        assert self.archive.append_files('source-dir', ['foo', 'bar']) \
            == 'foo.tar'
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir', '-r',
                '--file=' + self.archive.filename,
                '--xattrs', '--xattrs-include=*',
                'foo', 'bar'
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_with_options(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create('source-dir', options=[
            '--fake-option', 'fake_arg'
        ]) == 'foo.tar'
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir',
                '--fake-option', 'fake_arg',
                '--xattrs', '--xattrs-include=*',
                '-c', '-f', 'foo.tar', 'foo', 'bar'
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
        command = mock.Mock()
        command.output = 'version 1.26.1'
        mock_command.return_value = command
        archive = ArchiveTar('foo.tar')
        mock_os_dir.return_value = ['foo', 'bar']
        assert archive.create('source-dir') == 'foo.tar'
        calls = [
            call(['tar', '--version']),
            call(
                [
                    'tar', '-C', 'source-dir',
                    '-c', '-f', 'foo.tar', 'foo', 'bar'
                ]
            )
        ]
        mock_command.assert_has_calls(calls)
        assert mock_command.call_count == 2

    @patch('kiwi.archive.tar.Command.run')
    def test_create_from_dir_with_excludes(self, mock_command):
        command = mock.Mock()
        command.output = 'version 1.27.0'
        mock_command.return_value = command
        archive = ArchiveTar('foo.tar', False)
        assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
        calls = [
            call(['tar', '--version']),
            call(
                [
                    'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
                    '-c', '-f', 'foo.tar', '.',
                    '--exclude', './foo', '--exclude', './bar'
                ]
            )
        ]
        mock_command.assert_has_calls(calls)
        assert mock_command.call_count == 2

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_xz_compressed(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create_xz_compressed('source-dir') == 'foo.tar.xz'
        mock_command.assert_called_once_with(
            [
                'bash', '-c',
                ' '.join([
                    'tar', '-C', 'source-dir', '--xattrs',
                    '--xattrs-include=*', '-c', '--to-stdout',
                    'foo', 'bar', '|', 'xz', '-f', '--threads=0',
                    '>', 'foo.tar.xz'
                ])
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_xz_compressed_with_custom_xz_options(
        self, mock_os_dir, mock_command
    ):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create_xz_compressed(
            'source-dir', xz_options=['-a', '-b']
        ) == 'foo.tar.xz'
        mock_command.assert_called_once_with(
            [
                'bash', '-c',
                ' '.join([
                    'tar', '-C', 'source-dir', '--xattrs',
                    '--xattrs-include=*', '-c', '--to-stdout',
                    'foo', 'bar', '|', 'xz', '-f', '-a', '-b',
                    '>', 'foo.tar.xz'
                ])
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create_gnu_gzip_compressed('source-dir') \
            == 'foo.tar.gz'
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir',
                '--format=gnu', '-cSz', '-f', 'foo.tar.gz', 'foo', 'bar'
            ]
        )

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_exclude(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create('source-dir', ['foo']) == 'foo.tar'
        mock_command.assert_called_once_with(
            [
                'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
                '-c', '-f', 'foo.tar', 'bar'
            ]
        )
Пример #37
0
 def create_recovery_archive(self):
     """
     Create a compressed recovery archive from the root tree
     for use with kiwi's recvoery system. The method creates
     additional data into the image root filesystem which is
     deleted prior to the creation of a new recovery data set
     """
     # cleanup
     bash_comand = [
         'rm', '-f', self.root_dir + '/recovery.*'
     ]
     Command.run(['bash', '-c', ' '.join(bash_comand)])
     if not self.oemconfig['recovery']:
         return
     # recovery.tar
     log.info('Creating recovery tar archive')
     metadata = {
         'archive_name':
             self.root_dir + '/recovery.tar',
         'archive_filecount':
             self.root_dir + '/recovery.tar.files',
         'archive_size':
             self.root_dir + '/recovery.tar.size',
         'partition_size':
             self.root_dir + '/recovery.partition.size',
         'partition_filesystem':
             self.root_dir + '/recovery.tar.filesystem'
     }
     recovery_archive = NamedTemporaryFile(
         delete=False
     )
     archive = ArchiveTar(
         filename=recovery_archive.name,
         create_from_file_list=False
     )
     archive.create(
         source_dir=self.root_dir,
         exclude=['dev', 'proc', 'sys'],
         options=[
             '--numeric-owner',
             '--hard-dereference',
             '--preserve-permissions'
         ]
     )
     Command.run(
         ['mv', recovery_archive.name, metadata['archive_name']]
     )
     # recovery.tar.filesystem
     recovery_filesystem = self.xml_state.build_type.get_filesystem()
     with open(metadata['partition_filesystem'], 'w') as partfs:
         partfs.write('%s' % recovery_filesystem)
     log.info(
         '--> Recovery partition filesystem: %s', recovery_filesystem
     )
     # recovery.tar.files
     bash_comand = [
         'tar', '-tf', metadata['archive_name'], '|', 'wc', '-l'
     ]
     tar_files_call = Command.run(
         ['bash', '-c', ' '.join(bash_comand)]
     )
     tar_files_count = int(tar_files_call.output.rstrip('\n'))
     with open(metadata['archive_filecount'], 'w') as files:
         files.write('%d\n' % tar_files_count)
     log.info(
         '--> Recovery file count: %d files', tar_files_count
     )
     # recovery.tar.size
     recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
     with open(metadata['archive_size'], 'w') as size:
         size.write('%d' % recovery_archive_size_bytes)
     log.info(
         '--> Recovery uncompressed size: %d mbytes',
         int(recovery_archive_size_bytes / 1048576)
     )
     # recovery.tar.gz
     log.info('--> Compressing recovery archive')
     compress = Compress(self.root_dir + '/recovery.tar')
     compress.gzip()
     # recovery.partition.size
     recovery_archive_gz_size_mbytes = int(
         os.path.getsize(metadata['archive_name'] + '.gz') / 1048576
     )
     recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
         + Defaults.get_recovery_spare_mbytes()
     with open(metadata['partition_size'], 'w') as gzsize:
         gzsize.write('%d' % recovery_partition_mbytes)
     log.info(
         '--> Recovery partition size: %d mbytes',
         recovery_partition_mbytes
     )
     # delete recovery archive if inplace recovery is requested
     # In this mode the recovery archive is created at install time
     # and not at image creation time. However the recovery metadata
     # is preserved in order to be able to check if enough space
     # is available on the disk to create the recovery archive.
     if self.oemconfig['recovery_inplace']:
         log.info(
             '--> Inplace recovery requested, deleting archive'
         )
         Path.wipe(metadata['archive_name'] + '.gz')
Пример #38
0
 def setup(self, mock_command):
     command = mock.Mock()
     command.output = 'version 1.27.0'
     mock_command.return_value = command
     self.archive = ArchiveTar('foo.tar')
Пример #39
0
 def test_invalid_tar_command_version(self, mock_command):
     command = mock.Mock()
     command.output = 'version cannot be parsed'
     mock_command.return_value = command
     self.archive = ArchiveTar('foo.tar')
Пример #40
0
Файл: kis.py Проект: jfkw/kiwi
    def create(self) -> Result:
        """
        Build a component image consisting out of a boot image(initrd)
        plus its appropriate kernel files and the root filesystem
        image with a checksum.

        Image types which triggers this builder are:

        * image="kis"
        * image="pxe"

        :raises KiwiKisBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        if self.filesystem:
            log.info('Creating root filesystem image')
            self.filesystem.create()
            os.rename(
                self.filesystem.filename, self.image_name
            )
            self.image = self.image_name
            if self.compressed:
                log.info('xz compressing root filesystem image')
                compress = Compress(self.image)
                compress.xz(self.xz_options)
                self.image = compress.compressed_filename

            log.info('Creating root filesystem MD5 checksum')
            checksum = Checksum(self.image)
            checksum.md5(self.checksum_name)

        # prepare boot(initrd) root system
        log.info('Creating boot image')
        self.boot_image_task.prepare()

        # export modprobe configuration to boot image
        self.system_setup.export_modprobe_setup(
            self.boot_image_task.boot_root_directory
        )

        # extract kernel from boot(initrd) root system
        kernel = Kernel(self.boot_image_task.boot_root_directory)
        kernel_data = kernel.get_kernel()
        if kernel_data:
            self.kernel_filename = ''.join(
                [
                    os.path.basename(self.image_name), '-',
                    kernel_data.version, '.kernel'
                ]
            )
            kernel.copy_kernel(
                self.target_dir, self.kernel_filename
            )
        else:
            raise KiwiKisBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image_task.boot_root_directory
            )

        # extract hypervisor from boot(initrd) root system
        if self.xen_server:
            hypervisor_data = kernel.get_xen_hypervisor()
            if hypervisor_data:
                self.hypervisor_filename = ''.join(
                    [
                        os.path.basename(self.image_name),
                        '-', hypervisor_data.name
                    ]
                )
                kernel.copy_xen_hypervisor(
                    self.target_dir, self.hypervisor_filename
                )
                self.result.add(
                    key='xen_hypervisor',
                    filename=self.target_dir + '/' + self.hypervisor_filename,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )
            else:
                raise KiwiKisBootImageError(
                    'No hypervisor in boot image tree %s found' %
                    self.boot_image_task.boot_root_directory
                )

        # create initrd
        self.boot_image_task.create_initrd()

        # create append information
        # this information helps to configure the deployment infrastructure
        if self.filesystem and self.filesystem.root_uuid \
           and self.initrd_system == 'dracut':
            cmdline = 'root=UUID={}'.format(self.filesystem.root_uuid)
            if self.custom_cmdline:
                cmdline += ' {}'.format(self.custom_cmdline)
            with open(self.append_file, 'w') as append:
                append.write(cmdline)

        # put results into a tarball
        if not self.xz_options:
            self.xz_options = Defaults.get_xz_compression_options()

        kis_tarball_files = [
            self.kernel_filename,
            os.path.basename(self.boot_image_task.initrd_filename),
            os.path.basename(self.checksum_name),
        ]

        if self.image:
            kis_tarball_files.append(os.path.basename(self.image))

        if self.filesystem and self.filesystem.root_uuid \
           and self.initrd_system == 'dracut':
            kis_tarball_files.append(os.path.basename(self.append_file))

        kis_tarball = ArchiveTar(
            self.archive_name,
            create_from_file_list=True,
            file_list=kis_tarball_files
        )

        if self.compressed:
            self.archive_name = kis_tarball.create(self.target_dir)
        else:
            self.archive_name = kis_tarball.create_xz_compressed(
                self.target_dir, xz_options=self.xz_options
            )

        Result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.archive_name
        )
        # store results
        self.result.add(
            key='kis_archive',
            filename=self.archive_name,
            use_for_bundle=True,
            compress=self.runtime_config.get_bundle_compression(
                default=False
            ),
            shasum=True
        )

        # create image root metadata
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_changes',
            filename=self.system_setup.export_package_changes(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=True,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result
Пример #41
0
class TestArchiveTar(object):
    @patch('kiwi.archive.tar.Command.run')
    def setup(self, mock_command):
        command = mock.Mock()
        command.output = 'version 1.27.0'
        mock_command.return_value = command
        self.archive = ArchiveTar('foo.tar')

    @raises(KiwiCommandCapabilitiesError)
    @patch('kiwi.archive.tar.Command.run')
    def test_invalid_tar_command_version(self, mock_command):
        command = mock.Mock()
        command.output = 'version cannot be parsed'
        mock_command.return_value = command
        self.archive = ArchiveTar('foo.tar')

    @patch('kiwi.archive.tar.Command.run')
    def test_extract(self, mock_command):
        self.archive.extract('destination')
        mock_command.assert_called_once_with(
            ['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar'])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create('source-dir') == 'foo.tar'
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
            '-f', 'foo.tar', 'foo', 'bar'
        ])

    @patch('kiwi.archive.tar.Command.run')
    def test_append_files(self, mock_command):
        assert self.archive.append_files('source-dir', ['foo', 'bar']) \
            == 'foo.tar'
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '-r', '--file=' + self.archive.filename,
            '--xattrs', '--xattrs-include=*', 'foo', 'bar'
        ])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_with_options(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create('source-dir',
                                   options=['--fake-option',
                                            'fake_arg']) == 'foo.tar'
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--fake-option', 'fake_arg', '--xattrs',
            '--xattrs-include=*', '-c', '-f', 'foo.tar', 'foo', 'bar'
        ])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
        command = mock.Mock()
        command.output = 'version 1.26.1'
        mock_command.return_value = command
        archive = ArchiveTar('foo.tar')
        mock_os_dir.return_value = ['foo', 'bar']
        assert archive.create('source-dir') == 'foo.tar'
        calls = [
            call(['tar', '--version']),
            call([
                'tar', '-C', 'source-dir', '-c', '-f', 'foo.tar', 'foo', 'bar'
            ])
        ]
        mock_command.assert_has_calls(calls)
        assert mock_command.call_count == 2

    @patch('kiwi.archive.tar.Command.run')
    def test_create_from_dir_with_excludes(self, mock_command):
        command = mock.Mock()
        command.output = 'version 1.27.0'
        mock_command.return_value = command
        archive = ArchiveTar('foo.tar', False)
        assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
        calls = [
            call(['tar', '--version']),
            call([
                'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
                '-c', '-f', 'foo.tar', '.', '--exclude', './foo', '--exclude',
                './bar'
            ])
        ]
        mock_command.assert_has_calls(calls)
        assert mock_command.call_count == 2

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_xz_compressed(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create_xz_compressed('source-dir') == 'foo.tar.xz'
        mock_command.assert_called_once_with([
            'bash', '-c', ' '.join([
                'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
                '-c', '--to-stdout', 'foo', 'bar', '|', 'xz', '-f',
                '--threads=0', '>', 'foo.tar.xz'
            ])
        ])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_xz_compressed_with_custom_xz_options(
            self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create_xz_compressed('source-dir',
                                                 xz_options=['-a', '-b'
                                                             ]) == 'foo.tar.xz'
        mock_command.assert_called_once_with([
            'bash', '-c', ' '.join([
                'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
                '-c', '--to-stdout', 'foo', 'bar', '|', 'xz', '-f', '-a', '-b',
                '>', 'foo.tar.xz'
            ])
        ])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create_gnu_gzip_compressed('source-dir') \
            == 'foo.tar.gz'
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--format=gnu', '-cSz', '-f',
            'foo.tar.gz', 'foo', 'bar'
        ])

    @patch('kiwi.archive.tar.Command.run')
    @patch('os.listdir')
    def test_create_exclude(self, mock_os_dir, mock_command):
        mock_os_dir.return_value = ['foo', 'bar']
        assert self.archive.create('source-dir', ['foo']) == 'foo.tar'
        mock_command.assert_called_once_with([
            'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
            '-f', 'foo.tar', 'bar'
        ])
Пример #42
0
 def test_invalid_tar_command_version(self, mock_command):
     command = mock.Mock()
     command.output = 'version cannot be parsed'
     mock_command.return_value = command
     with raises(KiwiCommandCapabilitiesError):
         self.archive = ArchiveTar('foo.tar')
Пример #43
0
 def test_invalid_tar_command_version(self, mock_command):
     command = mock.Mock()
     command.output = 'version cannot be parsed'
     mock_command.return_value = command
     self.archive = ArchiveTar('foo.tar')
Пример #44
0
 def setup(self, mock_command):
     command = mock.Mock()
     command.output = 'version 1.27.0'
     mock_command.return_value = command
     self.archive = ArchiveTar('foo.tar')
Пример #45
0
 def create_recovery_archive(self):
     """
     Create a compressed recovery archive from the root tree
     for use with kiwi's recvoery system. The method creates
     additional data into the image root filesystem which is
     deleted prior to the creation of a new recovery data set
     """
     # cleanup
     bash_comand = ['rm', '-f', self.root_dir + '/recovery.*']
     Command.run(['bash', '-c', ' '.join(bash_comand)])
     if not self.oemconfig['recovery']:
         return
     # recovery.tar
     log.info('Creating recovery tar archive')
     metadata = {
         'archive_name': self.root_dir + '/recovery.tar',
         'archive_filecount': self.root_dir + '/recovery.tar.files',
         'archive_size': self.root_dir + '/recovery.tar.size',
         'partition_size': self.root_dir + '/recovery.partition.size',
         'partition_filesystem': self.root_dir + '/recovery.tar.filesystem'
     }
     recovery_archive = NamedTemporaryFile(delete=False)
     archive = ArchiveTar(filename=recovery_archive.name,
                          create_from_file_list=False)
     archive.create(source_dir=self.root_dir,
                    exclude=['dev', 'proc', 'sys'],
                    options=[
                        '--numeric-owner', '--hard-dereference',
                        '--preserve-permissions'
                    ])
     Command.run(['mv', recovery_archive.name, metadata['archive_name']])
     # recovery.tar.filesystem
     recovery_filesystem = self.xml_state.build_type.get_filesystem()
     with open(metadata['partition_filesystem'], 'w') as partfs:
         partfs.write('%s' % recovery_filesystem)
     log.info('--> Recovery partition filesystem: %s', recovery_filesystem)
     # recovery.tar.files
     bash_comand = ['tar', '-tf', metadata['archive_name'], '|', 'wc', '-l']
     tar_files_call = Command.run(['bash', '-c', ' '.join(bash_comand)])
     tar_files_count = int(tar_files_call.output.rstrip('\n'))
     with open(metadata['archive_filecount'], 'w') as files:
         files.write('%d\n' % tar_files_count)
     log.info('--> Recovery file count: %d files', tar_files_count)
     # recovery.tar.size
     recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
     with open(metadata['archive_size'], 'w') as size:
         size.write('%d' % recovery_archive_size_bytes)
     log.info('--> Recovery uncompressed size: %d mbytes',
              int(recovery_archive_size_bytes / 1048576))
     # recovery.tar.gz
     log.info('--> Compressing recovery archive')
     compress = Compress(self.root_dir + '/recovery.tar')
     compress.gzip()
     # recovery.partition.size
     recovery_archive_gz_size_mbytes = int(
         os.path.getsize(metadata['archive_name'] + '.gz') / 1048576)
     recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
         + Defaults.get_recovery_spare_mbytes()
     with open(metadata['partition_size'], 'w') as gzsize:
         gzsize.write('%d' % recovery_partition_mbytes)
     log.info('--> Recovery partition size: %d mbytes',
              recovery_partition_mbytes)
     # delete recovery archive if inplace recovery is requested
     # In this mode the recovery archive is created at install time
     # and not at image creation time. However the recovery metadata
     # is preserved in order to be able to check if enough space
     # is available on the disk to create the recovery archive.
     if self.oemconfig['recovery_inplace']:
         log.info('--> Inplace recovery requested, deleting archive')
         Path.wipe(metadata['archive_name'] + '.gz')
Пример #46
0
    def create(self):
        """
        Create a root archive tarball

        Build a simple XZ compressed root tarball from the image root tree

        Image types which triggers this builder are:

        * image="tbz"
        """
        supported_archives = Defaults.get_archive_image_types()
        if self.requested_archive_type not in supported_archives:
            raise KiwiArchiveSetupError(
                'Unknown archive type: %s' % self.requested_archive_type
            )

        if self.requested_archive_type == 'tbz':
            log.info('Creating XZ compressed tar archive')
            archive = ArchiveTar(
                self._target_file_for('tar')
            )
            archive.create_xz_compressed(
                self.root_dir, xz_options=self.xz_options,
                exclude=Defaults.get_exclude_list_for_root_data_sync()
            )
            checksum = Checksum(self.filename)
            log.info('--> Creating archive checksum')
            checksum.md5(self.checksum)
            self.result.verify_image_size(
                self.runtime_config.get_max_size_constraint(),
                self.filename
            )
            self.result.add(
                key='root_archive',
                filename=self.filename,
                use_for_bundle=True,
                compress=False,
                shasum=True
            )
            self.result.add(
                key='root_archive_md5',
                filename=self.checksum,
                use_for_bundle=False
            )
            self.result.add(
                key='image_packages',
                filename=self.system_setup.export_package_list(
                    self.target_dir
                ),
                use_for_bundle=True,
                compress=False,
                shasum=False
            )
            self.result.add(
                key='image_verified',
                filename=self.system_setup.export_package_verification(
                    self.target_dir
                ),
                use_for_bundle=True,
                compress=False,
                shasum=False
            )
        return self.result
Пример #47
0
    def create_install_pxe_archive(self):
        """
        Create an oem install tar archive suitable for installing a
        disk image via the network using the PXE boot protocol.
        The archive contains:

        * The raw system image xz compressed
        * The raw system image checksum metadata file
        * The append file template for the boot server
        * The system image initrd for kexec
        * The install initrd
        * The kernel

        Image types which triggers this builder are:

        * installpxe="true|false"
        """
        self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.',
                               dir=self.target_dir)
        # the system image is transfered as xz compressed variant
        log.info('xz compressing disk image')
        pxe_image_filename = ''.join(
            [self.pxe_dir, '/',
             self.xml_state.xml_data.get_name(), '.xz'])
        compress = Compress(source_filename=self.diskname,
                            keep_source_on_compress=True)
        compress.xz(self.xz_options)
        Command.run(['mv', compress.compressed_filename, pxe_image_filename])

        # the system image transfer is checked against a checksum
        log.info('Creating disk image checksum')
        pxe_md5_filename = ''.join(
            [self.pxe_dir, '/',
             self.xml_state.xml_data.get_name(), '.md5'])
        checksum = Checksum(self.diskname)
        checksum.md5(pxe_md5_filename)

        # the install image name is stored in a config file
        if self.initrd_system == 'kiwi':
            self._write_install_image_info_to_boot_image()

        # the kexec required system image initrd is stored for dracut kiwi-dump
        if self.initrd_system == 'dracut':
            boot_names = self.boot_image_task.get_boot_names()
            system_image_initrd = os.sep.join(
                [self.root_dir, 'boot', boot_names.initrd_name])
            target_initrd_name = '{0}/{1}.initrd'.format(
                self.pxe_dir, self.xml_state.xml_data.get_name())
            shutil.copy(system_image_initrd, target_initrd_name)
            os.chmod(target_initrd_name, 420)

        # create pxe config append information
        # this information helps to configure the boot server correctly
        append_filename = ''.join(
            [self.pxe_dir, '/',
             self.xml_state.xml_data.get_name(), '.append'])
        if self.initrd_system == 'kiwi':
            cmdline = 'pxe=1'
        else:
            cmdline = ' '.join([
                'rd.kiwi.install.pxe',
                'rd.kiwi.install.image=http://example.com/image.xz'
            ])
        custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
        if custom_cmdline:
            cmdline += ' ' + custom_cmdline
        with open(append_filename, 'w') as append:
            append.write('%s\n' % cmdline)

        # create initrd for pxe install
        log.info('Creating pxe install boot image')
        self._create_pxe_install_kernel_and_initrd()

        # create pxe install tarball
        log.info('Creating pxe install archive')
        archive = ArchiveTar(self.pxename.replace('.xz', ''))
        archive.create_xz_compressed(self.pxe_dir, xz_options=self.xz_options)
Пример #48
0
    def create(self):
        """
        Build a pxe image set consisting out of a boot image(initrd)
        plus its appropriate kernel files and the root filesystem
        image with a checksum. The result can be used within the kiwi
        PXE boot infrastructure

        Image types which triggers this builder are:

        * image="pxe"

        :raises KiwiPxeBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        log.info('Creating PXE root filesystem image')
        self.filesystem.create()
        os.rename(
            self.filesystem.filename, self.image_name
        )
        self.image = self.image_name
        if self.compressed:
            log.info('xz compressing root filesystem image')
            compress = Compress(self.image)
            compress.xz(self.xz_options)
            self.image = compress.compressed_filename

        log.info('Creating PXE root filesystem MD5 checksum')
        checksum = Checksum(self.image)
        checksum.md5(self.checksum_name)

        # prepare boot(initrd) root system
        log.info('Creating PXE boot image')
        self.boot_image_task.prepare()

        # export modprobe configuration to boot image
        self.system_setup.export_modprobe_setup(
            self.boot_image_task.boot_root_directory
        )

        # extract kernel from boot(initrd) root system
        kernel = Kernel(self.boot_image_task.boot_root_directory)
        kernel_data = kernel.get_kernel()
        if kernel_data:
            self.kernel_filename = ''.join(
                [
                    os.path.basename(self.image_name), '-',
                    kernel_data.version, '.kernel'
                ]
            )
            kernel.copy_kernel(
                self.target_dir, self.kernel_filename
            )
        else:
            raise KiwiPxeBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image_task.boot_root_directory
            )

        # extract hypervisor from boot(initrd) root system
        if self.xen_server:
            kernel_data = kernel.get_xen_hypervisor()
            if kernel_data:
                self.hypervisor_filename = ''.join(
                    [os.path.basename(self.image_name), '-', kernel_data.name]
                )
                kernel.copy_xen_hypervisor(
                    self.target_dir, self.hypervisor_filename
                )
                self.result.add(
                    key='xen_hypervisor',
                    filename=self.target_dir + '/' + self.hypervisor_filename,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )
            else:
                raise KiwiPxeBootImageError(
                    'No hypervisor in boot image tree %s found' %
                    self.boot_image_task.boot_root_directory
                )

        # create initrd for pxe boot
        self.boot_image_task.create_initrd()

        # put results into a tarball
        if not self.xz_options:
            self.xz_options = Defaults.get_xz_compression_options()

        pxe_tarball_files = [
            self.kernel_filename,
            os.path.basename(self.boot_image_task.initrd_filename),
            os.path.basename(self.image),
            os.path.basename(self.checksum_name)
        ]
        pxe_tarball = ArchiveTar(
            self.archive_name,
            create_from_file_list=True,
            file_list=pxe_tarball_files
        )

        if self.compressed:
            self.archive_name = pxe_tarball.create(self.target_dir)
        else:
            self.archive_name = pxe_tarball.create_xz_compressed(
                self.target_dir, xz_options=self.xz_options
            )

        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.archive_name
        )
        # store results
        self.result.add(
            key='pxe_archive',
            filename=self.archive_name,
            use_for_bundle=True,
            compress=self.runtime_config.get_bundle_compression(
                default=False
            ),
            shasum=True
        )

        # create image root metadata
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result