Esempio n. 1
0
 def test_load(self, mock_open, mock_exists, mock_pickle_load):
     mock_open.return_value = self.context_manager_mock
     mock_exists.return_value = True
     Result.load('kiwi.result')
     mock_open.assert_called_once_with(
         'kiwi.result', 'rb'
     )
     mock_pickle_load.assert_called_once_with(
         self.file_mock
     )
Esempio n. 2
0
    def setup(self):
        sys.argv = [
            sys.argv[0],
            "result",
            "bundle",
            "--target-dir",
            "target_dir",
            "--bundle-dir",
            "bundle_dir",
            "--id",
            "Build_42",
        ]
        self.context_manager_mock = mock.Mock()
        self.file_mock = mock.Mock()
        self.enter_mock = mock.Mock()
        self.exit_mock = mock.Mock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, "__enter__", self.enter_mock)
        setattr(self.context_manager_mock, "__exit__", self.exit_mock)

        self.file_mock.read.return_value = b"data"

        self.xml_state = mock.Mock()
        self.xml_state.get_image_version = mock.Mock(return_value="1.2.3")

        self.result = Result(self.xml_state)
        self.result.add(key="keyname", filename="filename-1.2.3", use_for_bundle=True, compress=True, shasum=True)

        kiwi.tasks.result_bundle.Help = mock.Mock(return_value=mock.Mock())
        self.task = ResultBundleTask()
Esempio n. 3
0
    def setup(self):
        sys.argv = [
            sys.argv[0], 'result', 'bundle', '--target-dir', 'target_dir',
            '--bundle-dir', 'bundle_dir', '--id', 'Build_42'
        ]
        self.context_manager_mock = mock.Mock()
        self.file_mock = mock.Mock()
        self.enter_mock = mock.Mock()
        self.exit_mock = mock.Mock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, '__enter__', self.enter_mock)
        setattr(self.context_manager_mock, '__exit__', self.exit_mock)

        self.file_mock.read.return_value = b'data'

        self.xml_state = mock.Mock()
        self.xml_state.get_image_version = mock.Mock(
            return_value='1.2.3'
        )

        self.result = Result(self.xml_state)
        self.result.add(
            key='keyname', filename='filename-1.2.3',
            use_for_bundle=True, compress=True, shasum=True
        )

        kiwi.tasks.result_bundle.Help = mock.Mock(
            return_value=mock.Mock()
        )
        self.task = ResultBundleTask()
Esempio n. 4
0
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.target_dir = target_dir
        self.compressed = xml_state.build_type.get_compressed()
        self.xen_server = xml_state.is_xen_server()
        self.pxedeploy = xml_state.get_build_type_pxedeploy_section()
        self.filesystem = FileSystemBuilder(
            xml_state, target_dir, root_dir + '/'
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=root_dir
        )

        self.boot_signing_keys = custom_args['signing_keys'] if custom_args \
            and 'signing_keys' in custom_args else None

        self.xz_options = custom_args['xz_options'] if custom_args \
            and 'xz_options' in custom_args else None

        self.boot_image_task = BootImage(
            xml_state, target_dir,
            signing_keys=self.boot_signing_keys, custom_args=custom_args
        )
        self.image_name = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version()
            ]
        )
        self.archive_name = ''.join([self.image_name, '.tar.xz'])
        self.kernel_filename = None
        self.hypervisor_filename = None
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()
Esempio n. 5
0
    def setup(self):
        self.context_manager_mock = mock.MagicMock()
        self.file_mock = mock.MagicMock()
        self.enter_mock = mock.MagicMock()
        self.exit_mock = mock.MagicMock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, '__enter__', self.enter_mock)
        setattr(self.context_manager_mock, '__exit__', self.exit_mock)

        self.xml_state = mock.Mock()

        self.result = Result(self.xml_state)
Esempio n. 6
0
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.requested_archive_type = xml_state.get_build_type_name()
        self.result = Result(xml_state)
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.filename = self._target_file_for('tar.xz')
        self.checksum = self._target_file_for('md5')
        self.xz_options = custom_args['xz_options'] if custom_args \
            and 'xz_options' in custom_args else None

        self.runtime_config = RuntimeConfig()
Esempio n. 7
0
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.container_config = xml_state.get_container_config()
        self.requested_container_type = xml_state.get_build_type_name()
        self.base_image = None
        self.base_image_md5 = None

        self.container_config['xz_options'] = custom_args['xz_options'] \
            if custom_args and 'xz_options' in custom_args else None

        if xml_state.get_derived_from_image_uri():
            # The base image is expected to be unpacked by the kiwi
            # prepare step and stored inside of the root_dir/image directory.
            # In addition a md5 file of the image is expected too
            self.base_image = Defaults.get_imported_root_image(
                self.root_dir
            )
            self.base_image_md5 = ''.join([self.base_image, '.md5'])

            if not os.path.exists(self.base_image):
                raise KiwiContainerBuilderError(
                    'Unpacked Base image {0} not found'.format(
                        self.base_image
                    )
                )
            if not os.path.exists(self.base_image_md5):
                raise KiwiContainerBuilderError(
                    'Base image MD5 sum {0} not found at'.format(
                        self.base_image_md5
                    )
                )

        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.filename = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version(),
                '.', self.requested_container_type, '.tar'
            ]
        )
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()
Esempio n. 8
0
    def process(self):
        """
        List result information from a previous system command
        """
        self.manual = Help()
        if self._help():
            return

        result_directory = os.path.abspath(
            self.command_args['--target-dir']
        )
        log.info(
            'Listing results from %s', result_directory
        )
        result = Result.load(
            result_directory + '/kiwi.result'
        )
        result.print_results()
Esempio n. 9
0
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.media_dir = None
        self.live_container_dir = None
        self.arch = platform.machine()
        if self.arch == 'i686' or self.arch == 'i586':
            self.arch = 'ix86'
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.live_type = xml_state.build_type.get_flags()
        self.volume_id = xml_state.build_type.get_volid() or \
            Defaults.get_volume_id()
        self.mbrid = SystemIdentifier()
        self.mbrid.calculate_id()
        self.filesystem_custom_parameters = {
            'mount_options': xml_state.get_fs_mount_option_list()
        }
        self.publisher = xml_state.build_type.get_publisher() or \
            Defaults.get_publisher()

        if not self.live_type:
            self.live_type = Defaults.get_default_live_iso_type()

        self.boot_image = BootImageDracut(
            xml_state, target_dir, self.root_dir
        )
        self.firmware = FirmWare(
            xml_state
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.isoname = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version(),
                '.iso'
            ]
        )
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()
Esempio n. 10
0
 def __init__(self, xml_state, target_dir, root_dir):
     self.label = None
     self.root_dir = root_dir
     self.target_dir = target_dir
     self.requested_image_type = xml_state.get_build_type_name()
     if self.requested_image_type == 'pxe':
         self.requested_filesystem = xml_state.build_type.get_filesystem()
     else:
         self.requested_filesystem = self.requested_image_type
     if not self.requested_filesystem:
         raise KiwiFileSystemSetupError(
             'No filesystem configured in %s type' %
             self.requested_image_type
         )
     self.filesystem_custom_parameters = {
         'mount_options': xml_state.get_fs_mount_option_list()
     }
     self.system_setup = SystemSetup(
         xml_state=xml_state, root_dir=self.root_dir
     )
     self.filename = ''.join(
         [
             target_dir, '/',
             xml_state.xml_data.get_name(),
             '.' + platform.machine(),
             '-' + xml_state.get_image_version(),
             '.', self.requested_filesystem
         ]
     )
     self.blocksize = xml_state.build_type.get_target_blocksize()
     self.filesystem_setup = FileSystemSetup(xml_state, root_dir)
     self.filesystems_no_device_node = [
         'squashfs'
     ]
     self.result = Result(xml_state)
     self.runtime_config = RuntimeConfig()
Esempio n. 11
0
class TestResult:
    @fixture(autouse=True)
    def inject_fixtures(self, caplog):
        self._caplog = caplog

    def setup(self):
        self.xml_state = Mock()

        self.result = Result(self.xml_state)

    def test_add(self):
        assert self.result.add('foo', 'bar') is None
        result = self.result.get_results()
        assert isinstance(result, dict)
        assert result['foo'].filename == 'bar'
        assert result['foo'].use_for_bundle is True
        assert result['foo'].compress is False

    def test_print_results_no_data(self):
        assert self.result.print_results() is None
        assert not self._caplog.text

    def test_print_results_data(self):
        assert self.result.add('foo', 'bar') is None
        with self._caplog.at_level(logging.INFO):
            self.result.print_results()

    @patch('pickle.dump')
    def test_dump(self, mock_pickle_dump):

        m_open = mock_open()
        with patch('builtins.open', m_open, create=True):
            assert self.result.dump('kiwi.result') is None

        m_open.assert_called_once_with('kiwi.result', 'wb')
        mock_pickle_dump.assert_called_once_with(self.result,
                                                 m_open.return_value)

    @patch('pickle.dump')
    def test_dump_failed(self, mock_pickle_dump):
        mock_pickle_dump.side_effect = Exception
        with patch('builtins.open'):
            with raises(KiwiResultError):
                self.result.dump('kiwi.result')

    @patch('pickle.load')
    @patch('os.path.exists')
    def test_load(self, mock_exists, mock_pickle_load):
        mock_exists.return_value = True

        m_open = mock_open()
        mock_pickle_load.return_value = Result
        with patch('builtins.open', m_open, create=True):
            assert Result.load('kiwi.result') is Result

        m_open.assert_called_once_with('kiwi.result', 'rb')
        mock_pickle_load.assert_called_once_with(m_open.return_value)

    @patch('os.path.exists')
    def test_load_result_not_present(self, mock_exists):
        mock_exists.return_value = False
        with raises(KiwiResultError):
            Result.load('kiwi.result')

    @patch('pickle.load')
    @patch('os.path.exists')
    def test_load_failed(self, mock_exists, mock_pickle_load):
        mock_exists.return_value = True
        mock_pickle_load.side_effect = Exception
        with raises(KiwiResultError):
            Result.load('kiwi.result')

    @patch('os.path.getsize')
    def test_build_constraint(self, mock_getsize):
        mock_getsize.return_value = 524288000
        assert self.result.verify_image_size(524288000, 'foo') is None

    @patch('os.path.getsize')
    def test_build_constraint_failure(self, mock_getsize):
        mock_getsize.return_value = 524288000
        with raises(KiwiResultError):
            self.result.verify_image_size(524287999, 'foo')
Esempio n. 12
0
 def test_load(self, mock_open, mock_exists, mock_pickle_load):
     mock_open.return_value = self.context_manager_mock
     mock_exists.return_value = True
     Result.load('kiwi.result')
     mock_open.assert_called_once_with('kiwi.result', 'rb')
     mock_pickle_load.assert_called_once_with(self.file_mock)
Esempio n. 13
0
class PxeBuilder(object):
    """
    Filesystem based PXE image builder.

    Attributes

    * :attr:`xml_state`
        Instance of XMLState

    * :attr:`target_dir`
        Target directory path name

    * :attr:`root_dir`
        System image root directory

    * :attr:`custom_args`
        Custom processing arguments defined as hash keys:
        * signing_keys: list of package signing keys
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.target_dir = target_dir
        self.compressed = xml_state.build_type.get_compressed()
        self.xen_server = xml_state.is_xen_server()
        self.pxedeploy = xml_state.get_build_type_pxedeploy_section()
        self.filesystem = FileSystemBuilder(
            xml_state, target_dir, root_dir + '/'
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=root_dir
        )

        self.boot_signing_keys = custom_args['signing_keys'] if custom_args \
            and 'signing_keys' in custom_args else None

        self.xz_options = custom_args['xz_options'] if custom_args \
            and 'xz_options' in custom_args else None

        self.boot_image_task = BootImage(
            xml_state, target_dir,
            signing_keys=self.boot_signing_keys, custom_args=custom_args
        )
        self.image_name = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version()
            ]
        )
        self.archive_name = ''.join([self.image_name, '.tar.xz'])
        self.kernel_filename = None
        self.hypervisor_filename = None
        self.result = Result(xml_state)

    def create(self):
        """
        Build a pxe image set consisting out of a boot image(initrd)
        plus its appropriate kernel files and the root filesystem
        image with a checksum. The result can be used within the kiwi
        PXE boot infrastructure

        Image types which triggers this builder are:

        * image="pxe"
        """
        log.info('Creating PXE root filesystem image')
        self.filesystem.create()
        os.rename(
            self.filesystem.filename, self.image_name
        )
        self.image = self.image_name
        if self.compressed:
            log.info('xz compressing root filesystem image')
            compress = Compress(self.image)
            compress.xz(self.xz_options)
            self.image = compress.compressed_filename

        log.info('Creating PXE root filesystem MD5 checksum')
        self.filesystem_checksum = ''.join([self.image, '.md5'])
        checksum = Checksum(self.image)
        checksum.md5(self.filesystem_checksum)

        # prepare boot(initrd) root system
        log.info('Creating PXE boot image')
        self.boot_image_task.prepare()

        # export modprobe configuration to boot image
        self.system_setup.export_modprobe_setup(
            self.boot_image_task.boot_root_directory
        )

        # extract kernel from boot(initrd) root system
        kernel = Kernel(self.boot_image_task.boot_root_directory)
        kernel_data = kernel.get_kernel()
        if kernel_data:
            self.kernel_filename = ''.join(
                [
                    os.path.basename(self.image_name), '-',
                    kernel_data.version, '.kernel'
                ]
            )
            kernel.copy_kernel(
                self.target_dir, self.kernel_filename
            )
        else:
            raise KiwiPxeBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image_task.boot_root_directory
            )

        # extract hypervisor from boot(initrd) root system
        if self.xen_server:
            kernel_data = kernel.get_xen_hypervisor()
            if kernel_data:
                self.hypervisor_filename = ''.join(
                    [os.path.basename(self.image_name), '-', kernel_data.name]
                )
                kernel.copy_xen_hypervisor(
                    self.target_dir, self.hypervisor_filename
                )
                self.result.add(
                    key='xen_hypervisor',
                    filename=self.target_dir + '/' + self.hypervisor_filename,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )
            else:
                raise KiwiPxeBootImageError(
                    'No hypervisor in boot image tree %s found' %
                    self.boot_image_task.boot_root_directory
                )

        # create initrd for pxe boot
        self.boot_image_task.create_initrd()

        # put results into a tarball
        Command.run(
            [
                'tar', '-C', self.target_dir, '-cJf', self.archive_name,
                self.kernel_filename,
                os.path.basename(self.boot_image_task.initrd_filename),
                os.path.basename(self.image),
                os.path.basename(self.filesystem_checksum)
            ]
        )

        # store results
        self.result.add(
            key='pxe_archive',
            filename=self.archive_name,
            use_for_bundle=True,
            compress=False,
            shasum=True
        )

        # create image root metadata
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )

        if self.pxedeploy:
            log.warning(
                'Creation of client config file from pxedeploy not implemented'
            )

        return self.result
Esempio n. 14
0
    def setup(self):
        self.xml_state = Mock()

        self.result = Result(self.xml_state)
Esempio n. 15
0
    def process(self):
        """
        Create result bundle from the image build results in the
        specified target directory. Each result image will contain
        the specified bundle identifier as part of its filename.
        Uncompressed image files will also become xz compressed
        and a sha sum will be created from every result image
        """
        self.manual = Help()
        if self._help():
            return

        # load serialized result object from target directory
        result_directory = os.path.abspath(self.command_args['--target-dir'])
        bundle_directory = os.path.abspath(self.command_args['--bundle-dir'])
        if result_directory == bundle_directory:
            raise KiwiBundleError(
                'Bundle directory must be different from target directory')

        log.info('Bundle build results from %s', result_directory)
        result = Result.load(result_directory + '/kiwi.result')
        image_version = result.xml_state.get_image_version()
        image_name = result.xml_state.xml_data.get_name()
        ordered_results = OrderedDict(sorted(result.get_results().items()))

        # hard link bundle files, compress and build checksum
        if not os.path.exists(bundle_directory):
            Path.create(bundle_directory)
        for result_file in list(ordered_results.values()):
            if result_file.use_for_bundle:
                bundle_file_basename = os.path.basename(result_file.filename)
                # The bundle id is only taken into account for image results
                # which contains the image version appended in its file name
                part_name = list(bundle_file_basename.partition(image_name))
                bundle_file_basename = ''.join([
                    part_name[0], part_name[1], part_name[2].replace(
                        image_version,
                        image_version + '-' + self.command_args['--id'])
                ])
                log.info('Creating %s', bundle_file_basename)
                bundle_file = ''.join(
                    [bundle_directory, '/', bundle_file_basename])
                Command.run(['cp', result_file.filename, bundle_file])
                if result_file.compress:
                    log.info('--> XZ compressing')
                    compress = Compress(bundle_file)
                    compress.xz(self.runtime_config.get_xz_options())
                    bundle_file = compress.compressed_filename

                if self.command_args['--zsync-source'] and result_file.shasum:
                    # Files with a checksum are considered to be image files
                    # and are therefore eligible to be provided via the
                    # requested Partial/differential file download based on
                    # zsync
                    zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK)
                    if zsyncmake:
                        log.info('--> Creating zsync control file')
                        Command.run([
                            zsyncmake, '-e', '-u',
                            os.sep.join([
                                self.command_args['--zsync-source'],
                                os.path.basename(bundle_file)
                            ]), '-o', bundle_file + '.zsync', bundle_file
                        ])
                    else:
                        log.warning(
                            '--> zsyncmake missing, zsync setup skipped')

                if result_file.shasum:
                    log.info('--> Creating SHA 256 sum')
                    checksum = Checksum(bundle_file)
                    with open(bundle_file + '.sha256', 'w') as shasum:
                        shasum.write('{0}  {1}'.format(
                            checksum.sha256(), os.path.basename(bundle_file)))
Esempio n. 16
0
class LiveImageBuilder(object):
    """
    Live image builder

    Attributes

    * :attr:`xml_state`
        Instance of XMLState

    * :attr:`target_dir`
        target directory path name

    * :attr:`root_dir`
        root directory path name

    * :attr:`custom_args`
        Custom processing arguments defined as hash keys:
        * signing_keys: list of package signing keys
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.media_dir = None
        self.arch = platform.machine()
        if self.arch == 'i686' or self.arch == 'i586':
            self.arch = 'ix86'
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.live_type = xml_state.build_type.get_flags()
        self.types = Defaults.get_live_iso_types()
        self.hybrid = xml_state.build_type.get_hybrid()
        self.volume_id = xml_state.build_type.get_volid()
        self.mbrid = SystemIdentifier()
        self.mbrid.calculate_id()
        self.filesystem_custom_parameters = {
            'mount_options': xml_state.get_fs_mount_option_list()
        }

        if not self.live_type:
            self.live_type = Defaults.get_default_live_iso_type()

        boot_signing_keys = None
        if custom_args and 'signing_keys' in custom_args:
            boot_signing_keys = custom_args['signing_keys']

        self.boot_image_task = BootImage(xml_state,
                                         target_dir,
                                         signing_keys=boot_signing_keys,
                                         custom_args=custom_args)
        self.firmware = FirmWare(xml_state)
        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.isoname = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '.' + platform.machine(),
            '-' + xml_state.get_image_version(), '.iso'
        ])
        self.live_image_file = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '-read-only.', self.arch, '-',
            xml_state.get_image_version()
        ])
        self.result = Result(xml_state)

    def create(self):
        """
        Build a bootable hybrid live ISO image

        Image types which triggers this builder are:

        * image="iso"
        """
        # media dir to store CD contents
        self.media_dir = mkdtemp(prefix='live-media.', dir=self.target_dir)
        rootsize = SystemSize(self.media_dir)

        # custom iso metadata
        log.info('Using following live ISO metadata:')
        log.info('--> Application id: %s', self.mbrid.get_id())
        log.info('--> Publisher: %s', Defaults.get_publisher())
        custom_iso_args = {
            'create_options': [
                '-A',
                self.mbrid.get_id(), '-p', '"' + Defaults.get_preparer() + '"',
                '-publisher', '"' + Defaults.get_publisher() + '"'
            ]
        }
        if self.volume_id:
            log.info('--> Volume id: %s', self.volume_id)
            custom_iso_args['create_options'].append('-V')
            custom_iso_args['create_options'].append('"' + self.volume_id +
                                                     '"')

        # prepare boot(initrd) root system
        log.info('Preparing live ISO boot system')
        self.boot_image_task.prepare()

        # export modprobe configuration to boot image
        self.system_setup.export_modprobe_setup(
            self.boot_image_task.boot_root_directory)

        # pack system into live boot structure
        log.info('Packing system into live ISO type: %s', self.live_type)
        if self.live_type in self.types:
            live_type_image = FileSystem(
                name=self.types[self.live_type],
                device_provider=None,
                root_dir=self.root_dir,
                custom_args=self.filesystem_custom_parameters)
            live_type_image.create_on_file(
                self.live_image_file,
                exclude=Defaults.get_exclude_list_for_root_data_sync())
            Command.run(['mv', self.live_image_file, self.media_dir])
            self._create_live_iso_client_config(self.live_type)
        else:
            raise KiwiLiveBootImageError('live ISO type "%s" not supported' %
                                         self.live_type)

        # setup bootloader config to boot the ISO via isolinux
        log.info('Setting up isolinux bootloader configuration')
        bootloader_config_isolinux = BootLoaderConfig('isolinux',
                                                      self.xml_state,
                                                      self.media_dir)
        bootloader_config_isolinux.setup_live_boot_images(
            mbrid=None, lookup_path=self.boot_image_task.boot_root_directory)
        bootloader_config_isolinux.setup_live_image_config(mbrid=None)
        bootloader_config_isolinux.write()
        self.system_setup.call_edit_boot_config_script(
            filesystem=self.types[self.live_type],
            boot_part_id=1,
            working_directory=self.media_dir)

        # setup bootloader config to boot the ISO via EFI
        if self.firmware.efi_mode():
            log.info('Setting up EFI grub bootloader configuration')
            bootloader_config_grub = BootLoaderConfig(
                'grub2', self.xml_state, self.media_dir, {
                    'grub_directory_name':
                    Defaults.get_grub_boot_directory_name(self.root_dir)
                })
            bootloader_config_grub.setup_live_boot_images(
                mbrid=self.mbrid, lookup_path=self.root_dir)
            bootloader_config_grub.setup_live_image_config(mbrid=self.mbrid)
            bootloader_config_grub.write()

        # create initrd for live image
        log.info('Creating live ISO boot image')
        self._create_live_iso_kernel_and_initrd()

        # calculate size and decide if we need UDF
        if rootsize.accumulate_mbyte_file_sizes() > 4096:
            log.info('ISO exceeds 4G size, using UDF filesystem')
            custom_iso_args['create_options'].append('-iso-level')
            custom_iso_args['create_options'].append('3')
            custom_iso_args['create_options'].append('-udf')

        # create iso filesystem from media_dir
        log.info('Creating live ISO image')
        iso_image = FileSystemIsoFs(device_provider=None,
                                    root_dir=self.media_dir,
                                    custom_args=custom_iso_args)
        iso_header_offset = iso_image.create_on_file(self.isoname)

        # make it hybrid
        if self.hybrid:
            Iso.create_hybrid(iso_header_offset, self.mbrid, self.isoname)

        # include metadata for checkmedia tool
        if self.xml_state.build_type.get_mediacheck() is True:
            Iso.set_media_tag(self.isoname)

        self.result.add(key='live_image',
                        filename=self.isoname,
                        use_for_bundle=True,
                        compress=False,
                        shasum=True)
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        return self.result

    def _create_live_iso_kernel_and_initrd(self):
        boot_path = self.media_dir + '/boot/' + self.arch + '/loader'
        Path.create(boot_path)
        kernel = Kernel(self.boot_image_task.boot_root_directory)
        if kernel.get_kernel():
            kernel.copy_kernel(boot_path, '/linux')
        else:
            raise KiwiLiveBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image_task.boot_root_directory)
        if self.xml_state.is_xen_server():
            if kernel.get_xen_hypervisor():
                kernel.copy_xen_hypervisor(boot_path, '/xen.gz')
            else:
                raise KiwiLiveBootImageError(
                    'No hypervisor in boot image tree %s found' %
                    self.boot_image_task.boot_root_directory)
        self.boot_image_task.create_initrd(self.mbrid)
        Command.run([
            'mv', self.boot_image_task.initrd_filename, boot_path + '/initrd'
        ])

    def _create_live_iso_client_config(self, iso_type):
        """
            Setup IMAGE and UNIONFS_CONFIG variables as they are used in
            the kiwi isoboot code. Variable contents:

            + IMAGE=target_device;live_iso_name_definition
            + UNIONFS_CONFIG=rw_device,ro_device,union_type

            If no real block device is used or can be predefined the
            word 'loop' is set as a placeholder or indicator to use a loop
            device. For more details please refer to the kiwi shell boot
            code
        """
        iso_client_config_file = self.media_dir + '/config.isoclient'
        iso_client_params = Defaults.get_live_iso_client_parameters()
        (system_device, union_device, union_type) = iso_client_params[iso_type]

        with open(iso_client_config_file, 'w') as config:
            config.write('IMAGE="%s;%s.%s;%s"\n' %
                         (system_device, self.xml_state.xml_data.get_name(),
                          self.arch, self.xml_state.get_image_version()))
            config.write('UNIONFS_CONFIG="%s,loop,%s"\n' %
                         (union_device, union_type))

    def __del__(self):
        if self.media_dir:
            log.info('Cleaning up %s instance', type(self).__name__)
            Path.wipe(self.media_dir)
Esempio n. 17
0
class DiskBuilder:
    """
    **Disk image builder**

    :param object xml_state: Instance of :class:`XMLState`
    :param str target_dir: Target directory path name
    :param str root_dir: Root directory path name
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * signing_keys: list of package signing keys
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.arch = Defaults.get_platform_name()
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.spare_part_mbsize = xml_state.get_build_type_spare_part_size()
        self.spare_part_fs = xml_state.build_type.get_spare_part_fs()
        self.spare_part_is_last = xml_state.build_type.get_spare_part_is_last()
        self.spare_part_mountpoint = \
            xml_state.build_type.get_spare_part_mountpoint()
        self.persistency_type = xml_state.build_type.get_devicepersistency()
        self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot(
        )
        self.custom_root_mount_args = xml_state.get_fs_mount_option_list()
        self.custom_root_creation_args = xml_state.get_fs_create_option_list()
        self.build_type_name = xml_state.get_build_type_name()
        self.image_format = xml_state.build_type.get_format()
        self.install_iso = xml_state.build_type.get_installiso()
        self.install_stick = xml_state.build_type.get_installstick()
        self.install_pxe = xml_state.build_type.get_installpxe()
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.volume_manager_name = xml_state.get_volume_management()
        self.volumes = xml_state.get_volumes()
        self.volume_group_name = xml_state.get_volume_group_name()
        self.mdraid = xml_state.build_type.get_mdraid()
        self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr()
        self.force_mbr = xml_state.build_type.get_force_mbr()
        self.luks = xml_state.build_type.get_luks()
        self.luks_os = xml_state.build_type.get_luksOS()
        self.xen_server = xml_state.is_xen_server()
        self.requested_filesystem = xml_state.build_type.get_filesystem()
        self.requested_boot_filesystem = \
            xml_state.build_type.get_bootfilesystem()
        self.bootloader = xml_state.get_build_type_bootloader_name()
        self.initrd_system = xml_state.get_initrd_system()
        self.target_removable = xml_state.build_type.get_target_removable()
        self.root_filesystem_is_multipath = \
            xml_state.get_oemconfig_oem_multipath_scan()
        self.swap_mbytes = xml_state.get_oemconfig_swap_mbytes()
        self.disk_setup = DiskSetup(xml_state, root_dir)
        self.unpartitioned_bytes = \
            xml_state.get_build_type_unpartitioned_bytes()
        self.custom_args = custom_args

        self.signing_keys = None
        if custom_args and 'signing_keys' in custom_args:
            self.signing_keys = custom_args['signing_keys']

        self.boot_image = BootImage(xml_state,
                                    target_dir,
                                    root_dir,
                                    signing_keys=self.signing_keys)
        self.firmware = FirmWare(xml_state)
        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.diskname = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '.' + self.arch,
            '-' + xml_state.get_image_version(), '.raw'
        ])
        self.boot_is_crypto = True if self.luks and not \
            self.disk_setup.need_boot_partition() else False
        self.install_media = self._install_image_requested()
        self.fstab = Fstab()

        # an instance of a class with the sync_data capability
        # representing the entire image system except for the boot/ area
        # which could live on another part of the disk
        self.system = None

        # an instance of a class with the sync_data capability
        # representing the boot/ area of the disk if not part of
        # self.system
        self.system_boot = None

        # an instance of a class with the sync_data capability
        # representing the boot/efi area of the disk
        self.system_efi = None

        # an instance of a class with the sync_data capability
        # representing the spare_part_mountpoint area of the disk
        self.system_spare = None

        # result store
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a bootable disk image and optional installation image
        The installation image is a bootable hybrid ISO image which
        embeds the disk image and an image installer

        Image types which triggers this builder are:

        * image="oem"
        * image="vmx"

        :return: result

        :rtype: instance of :class:`Result`
        """
        disk = DiskBuilder(self.xml_state, self.target_dir, self.root_dir,
                           self.custom_args)
        result = disk.create_disk()

        # cleanup disk resources taken prior to next steps
        del disk

        disk_installer = DiskBuilder(self.xml_state, self.target_dir,
                                     self.root_dir)
        result = disk_installer.create_install_media(result)

        disk_format = DiskBuilder(self.xml_state, self.target_dir,
                                  self.root_dir)

        disk_format.append_unpartitioned_space()
        result = disk_format.create_disk_format(result)

        return result

    def create_disk(self):  # noqa: C901
        """
        Build a bootable raw disk image

        :raises KiwiInstallMediaError:
            if install media is required and image type is not oem
        :raises KiwiVolumeManagerSetupError:
            root overlay at the same time volumes are defined is not supported

        :return: result

        :rtype: instance of :class:`Result`
        """
        if self.install_media and self.build_type_name != 'oem':
            raise KiwiInstallMediaError(
                'Install media requires oem type setup, got %s' %
                self.build_type_name)

        if self.root_filesystem_is_overlay and self.volume_manager_name:
            raise KiwiVolumeManagerSetupError(
                'Volume management together with root overlay is not supported'
            )

        # setup recovery archive, cleanup and create archive if requested
        self.system_setup.create_recovery_archive()

        # prepare boot(initrd) root system
        log.info('Preparing boot system')
        self.boot_image.prepare()

        # precalculate needed disk size
        disksize_mbytes = self.disk_setup.get_disksize_mbytes()

        # create the disk
        log.info('Creating raw disk image %s', self.diskname)
        self.loop_provider = LoopDevice(self.diskname, disksize_mbytes,
                                        self.blocksize)
        self.loop_provider.create()

        self.disk = Disk(self.firmware.get_partition_table_type(),
                         self.loop_provider,
                         self.xml_state.get_disk_start_sector())

        # create the bootloader instance
        self.bootloader_config = BootLoaderConfig(
            self.bootloader,
            self.xml_state,
            root_dir=self.root_dir,
            boot_dir=self.root_dir,
            custom_args={
                'targetbase':
                self.loop_provider.get_device(),
                'grub_directory_name':
                Defaults.get_grub_boot_directory_name(self.root_dir),
                'boot_is_crypto':
                self.boot_is_crypto
            })

        # create disk partitions and instance device map
        device_map = self._build_and_map_disk_partitions(disksize_mbytes)

        # create raid on current root device if requested
        if self.mdraid:
            self.raid_root = RaidDevice(device_map['root'])
            self.raid_root.create_degraded_raid(raid_level=self.mdraid)
            device_map['root'] = self.raid_root.get_device()
            self.disk.public_partition_id_map['kiwi_RaidPart'] = \
                self.disk.public_partition_id_map['kiwi_RootPart']
            self.disk.public_partition_id_map['kiwi_RaidDev'] = \
                device_map['root'].get_device()

        # create luks on current root device if requested
        if self.luks:
            self.luks_root = LuksDevice(device_map['root'])
            self.luks_boot_keyname = '/.root.keyfile'
            self.luks_boot_keyfile = ''.join(
                [self.root_dir, self.luks_boot_keyname])
            self.luks_root.create_crypto_luks(passphrase=self.luks,
                                              os=self.luks_os,
                                              keyfile=self.luks_boot_keyfile
                                              if self.boot_is_crypto else None)
            if self.boot_is_crypto:
                self.luks_boot_keyfile_setup = ''.join(
                    [self.root_dir, '/etc/dracut.conf.d/99-luks-boot.conf'])
                self.boot_image.write_system_config_file(
                    config={'install_items': [self.luks_boot_keyname]},
                    config_file=self.luks_boot_keyfile_setup)
                self.boot_image.include_file(
                    os.sep + os.path.basename(self.luks_boot_keyfile))
            device_map['luks_root'] = device_map['root']
            device_map['root'] = self.luks_root.get_device()

        # create spare filesystem on spare partition if present
        self._build_spare_filesystem(device_map)

        # create filesystems on boot partition(s) if any
        self._build_boot_filesystems(device_map)

        # create volumes and filesystems for root system
        if self.volume_manager_name:
            volume_manager_custom_parameters = {
                'fs_mount_options':
                self.custom_root_mount_args,
                'fs_create_options':
                self.custom_root_creation_args,
                'root_label':
                self.disk_setup.get_root_label(),
                'root_is_snapshot':
                self.xml_state.build_type.get_btrfs_root_is_snapshot(),
                'root_is_readonly_snapshot':
                self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot(
                ),
                'quota_groups':
                self.xml_state.build_type.get_btrfs_quota_groups(),
                'image_type':
                self.xml_state.get_build_type_name()
            }
            self.volume_manager = VolumeManager(
                self.volume_manager_name, device_map, self.root_dir + '/',
                self.volumes, volume_manager_custom_parameters)
            self.volume_manager.setup(self.volume_group_name)
            self.volume_manager.create_volumes(self.requested_filesystem)
            self.volume_manager.mount_volumes()
            self.system = self.volume_manager
            device_map['root'] = self.volume_manager.get_device().get('root')
            device_map['swap'] = self.volume_manager.get_device().get('swap')
        else:
            log.info('Creating root(%s) filesystem on %s',
                     self.requested_filesystem,
                     device_map['root'].get_device())
            filesystem_custom_parameters = {
                'mount_options': self.custom_root_mount_args,
                'create_options': self.custom_root_creation_args
            }
            filesystem = FileSystem(self.requested_filesystem,
                                    device_map['root'], self.root_dir + '/',
                                    filesystem_custom_parameters)
            filesystem.create_on_device(label=self.disk_setup.get_root_label())
            self.system = filesystem

        # create swap on current root device if requested
        if self.swap_mbytes:
            swap = FileSystem('swap', device_map['swap'])
            swap.create_on_device(label='SWAP')

        # store root partition/filesystem uuid for profile
        self._preserve_root_partition_uuid(device_map)
        self._preserve_root_filesystem_uuid(device_map)

        # create a random image identifier
        self.mbrid = SystemIdentifier()
        self.mbrid.calculate_id()

        # create first stage metadata to boot image
        self._write_partition_id_config_to_boot_image()

        self._write_recovery_metadata_to_boot_image()

        self._write_raid_config_to_boot_image()

        self._write_generic_fstab_to_boot_image(device_map)

        self.system_setup.export_modprobe_setup(
            self.boot_image.boot_root_directory)

        # create first stage metadata to system image
        self._write_image_identifier_to_system_image()

        self._write_crypttab_to_system_image()

        self._write_generic_fstab_to_system_image(device_map)

        if self.initrd_system == 'dracut':
            if self.root_filesystem_is_multipath is False:
                self.boot_image.omit_module('multipath')
            if self.root_filesystem_is_overlay:
                self.boot_image.include_module('kiwi-overlay')
                self.boot_image.write_system_config_file(
                    config={'modules': ['kiwi-overlay']})
            if self.build_type_name == 'oem':
                self.boot_image.include_module('kiwi-repart')

        # create initrd cpio archive
        self.boot_image.create_initrd(self.mbrid)

        # create second stage metadata to system image
        self._copy_first_boot_files_to_system_image()

        self._write_bootloader_meta_data_to_system_image(device_map)

        self.mbrid.write_to_disk(self.disk.storage_provider)

        # set SELinux file security contexts if context exists
        self._setup_selinux_file_contexts()

        # syncing system data to disk image
        log.info('Syncing system to image')
        if self.system_spare:
            self.system_spare.sync_data()

        if self.system_efi:
            log.info('--> Syncing EFI boot data to EFI partition')
            self.system_efi.sync_data()

        if self.system_boot:
            log.info('--> Syncing boot data at extra partition')
            self.system_boot.sync_data(
                self._get_exclude_list_for_boot_data_sync())

        log.info('--> Syncing root filesystem data')
        if self.root_filesystem_is_overlay:
            squashed_root_file = NamedTemporaryFile()
            squashed_root = FileSystemSquashFs(
                device_provider=None,
                root_dir=self.root_dir,
                custom_args={
                    'compression':
                    self.xml_state.build_type.get_squashfscompression()
                })
            squashed_root.create_on_file(
                filename=squashed_root_file.name,
                exclude=self._get_exclude_list_for_root_data_sync(device_map))
            Command.run([
                'dd',
                'if=%s' % squashed_root_file.name,
                'of=%s' % device_map['readonly'].get_device()
            ])
        else:
            self.system.sync_data(
                self._get_exclude_list_for_root_data_sync(device_map))

        # run post sync script hook
        if self.system_setup.script_exists(defaults.POST_DISK_SYNC_SCRIPT):
            disk_system = SystemSetup(self.xml_state,
                                      self.system.get_mountpoint())
            disk_system.import_description()
            disk_system.call_disk_script()
            disk_system.cleanup()

        # install boot loader
        self._install_bootloader(device_map)

        # set root filesystem properties
        self._setup_property_root_is_readonly_snapshot()

        # prepare for install media if requested
        if self.install_media:
            log.info('Saving boot image instance to file')
            self.boot_image.dump(self.target_dir + '/boot_image.pickledump')

        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(), self.diskname)
        # store image file name in result
        self.result.add(
            key='disk_image',
            filename=self.diskname,
            use_for_bundle=True if not self.image_format else False,
            compress=self.runtime_config.get_bundle_compression(default=True),
            shasum=True)

        # create image root metadata
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)

        return self.result

    def create_disk_format(self, result_instance):
        """
        Create a bootable disk format from a previously
        created raw disk image

        :param object result_instance: instance of :class:`Result`

        :return: updated result_instance

        :rtype: instance of :class:`Result`
        """
        if self.image_format:
            log.info('Creating %s Disk Format', self.image_format)
            disk_format = DiskFormat(self.image_format, self.xml_state,
                                     self.root_dir, self.target_dir)
            disk_format.create_image_format()
            disk_format.store_to_result(result_instance)

        return result_instance

    def append_unpartitioned_space(self):
        """
        Extends the raw disk if an unpartitioned area is specified
        """
        if self.unpartitioned_bytes:
            log.info('Expanding disk with %d bytes of unpartitioned space',
                     self.unpartitioned_bytes)
            disk_format = DiskFormat('raw', self.xml_state, self.root_dir,
                                     self.target_dir)
            disk_format.resize_raw_disk(self.unpartitioned_bytes, append=True)
            firmware = FirmWare(self.xml_state)
            loop_provider = LoopDevice(disk_format.diskname)
            loop_provider.create(overwrite=False)
            partitioner = Partitioner(firmware.get_partition_table_type(),
                                      loop_provider)
            partitioner.resize_table()

    def create_install_media(self, result_instance):
        """
        Build an installation image. The installation image is a
        bootable hybrid ISO image which embeds the raw disk image
        and an image installer

        :param object result_instance: instance of :class:`Result`

        :return: updated result_instance with installation media

        :rtype: instance of :class:`Result`
        """
        if self.install_media:
            install_image = InstallImageBuilder(
                self.xml_state, self.root_dir, self.target_dir,
                self._load_boot_image_instance(), self.custom_args)

            if self.install_iso or self.install_stick:
                log.info('Creating hybrid ISO installation image')
                install_image.create_install_iso()
                result_instance.add(key='installation_image',
                                    filename=install_image.isoname,
                                    use_for_bundle=True,
                                    compress=False,
                                    shasum=True)

            if self.install_pxe:
                log.info('Creating PXE installation archive')
                install_image.create_install_pxe_archive()
                result_instance.add(key='installation_pxe_archive',
                                    filename=install_image.pxetarball,
                                    use_for_bundle=True,
                                    compress=False,
                                    shasum=True)

        return result_instance

    def _load_boot_image_instance(self):
        boot_image_dump_file = self.target_dir + '/boot_image.pickledump'
        if not os.path.exists(boot_image_dump_file):
            raise KiwiInstallMediaError(
                'No boot image instance dump %s found' % boot_image_dump_file)
        try:
            with open(boot_image_dump_file, 'rb') as boot_image_dump:
                boot_image = pickle.load(boot_image_dump)
            Path.wipe(boot_image_dump_file)
        except Exception as e:
            raise KiwiInstallMediaError('Failed to load boot image dump: %s' %
                                        type(e).__name__)
        return boot_image

    def _setup_selinux_file_contexts(self):
        security_context = '/etc/selinux/targeted/contexts/files/file_contexts'
        if os.path.exists(self.root_dir + security_context):
            self.system_setup.set_selinux_file_contexts(security_context)

    def _install_image_requested(self):
        if self.install_iso or self.install_stick or self.install_pxe:
            return True

    def _get_exclude_list_for_root_data_sync(self, device_map):
        exclude_list = Defaults.get_exclude_list_for_root_data_sync()
        if 'spare' in device_map and self.spare_part_mountpoint:
            exclude_list.append('{0}/*'.format(
                self.spare_part_mountpoint.lstrip(os.sep)))
            exclude_list.append('{0}/.*'.format(
                self.spare_part_mountpoint.lstrip(os.sep)))
        if 'boot' in device_map and self.bootloader == 'grub2_s390x_emu':
            exclude_list.append('boot/zipl/*')
            exclude_list.append('boot/zipl/.*')
        elif 'boot' in device_map:
            exclude_list.append('boot/*')
            exclude_list.append('boot/.*')
        if 'efi' in device_map:
            exclude_list.append('boot/efi/*')
            exclude_list.append('boot/efi/.*')
        return exclude_list

    def _get_exclude_list_for_boot_data_sync(self):
        return ['efi/*']

    def _build_spare_filesystem(self, device_map):
        if 'spare' in device_map and self.spare_part_fs:
            spare_part_data_path = None
            spare_part_custom_parameters = {
                'fs_attributes':
                self.xml_state.get_build_type_spare_part_fs_attributes()
            }
            if self.spare_part_mountpoint:
                spare_part_data_path = self.root_dir + '{0}/'.format(
                    self.spare_part_mountpoint)
            filesystem = FileSystem(self.spare_part_fs, device_map['spare'],
                                    spare_part_data_path,
                                    spare_part_custom_parameters)
            filesystem.create_on_device(label='SPARE')
            self.system_spare = filesystem

    def _build_boot_filesystems(self, device_map):
        if 'efi' in device_map:
            log.info('Creating EFI(fat16) filesystem on %s',
                     device_map['efi'].get_device())
            filesystem = FileSystem('fat16', device_map['efi'],
                                    self.root_dir + '/boot/efi/')
            filesystem.create_on_device(label=self.disk_setup.get_efi_label())
            self.system_efi = filesystem

        if 'boot' in device_map:
            boot_filesystem = self.requested_boot_filesystem
            if not boot_filesystem:
                boot_filesystem = self.requested_filesystem
            boot_directory = self.root_dir + '/boot/'
            if self.bootloader == 'grub2_s390x_emu':
                boot_directory = self.root_dir + '/boot/zipl/'
                boot_filesystem = 'ext2'
            log.info('Creating boot(%s) filesystem on %s', boot_filesystem,
                     device_map['boot'].get_device())
            filesystem = FileSystem(boot_filesystem, device_map['boot'],
                                    boot_directory)
            filesystem.create_on_device(label=self.disk_setup.get_boot_label())
            self.system_boot = filesystem

    def _build_and_map_disk_partitions(self, disksize_mbytes):  # noqa: C901
        self.disk.wipe()
        disksize_used_mbytes = 0
        if self.firmware.legacy_bios_mode():
            log.info('--> creating EFI CSM(legacy bios) partition')
            partition_mbsize = self.firmware.get_legacy_bios_partition_size()
            self.disk.create_efi_csm_partition(partition_mbsize)
            disksize_used_mbytes += partition_mbsize

        if self.firmware.efi_mode():
            log.info('--> creating EFI partition')
            partition_mbsize = self.firmware.get_efi_partition_size()
            self.disk.create_efi_partition(partition_mbsize)
            disksize_used_mbytes += partition_mbsize

        if self.firmware.ofw_mode():
            log.info('--> creating PReP partition')
            partition_mbsize = self.firmware.get_prep_partition_size()
            self.disk.create_prep_partition(partition_mbsize)
            disksize_used_mbytes += partition_mbsize

        if self.disk_setup.need_boot_partition():
            log.info('--> creating boot partition')
            partition_mbsize = self.disk_setup.boot_partition_size()
            self.disk.create_boot_partition(partition_mbsize)
            disksize_used_mbytes += partition_mbsize

        if self.swap_mbytes:
            if not self.volume_manager_name or self.volume_manager_name != 'lvm':
                log.info('--> creating SWAP partition')
                self.disk.create_swap_partition(self.swap_mbytes)
                disksize_used_mbytes += self.swap_mbytes

        if self.spare_part_mbsize and not self.spare_part_is_last:
            log.info('--> creating spare partition')
            self.disk.create_spare_partition(self.spare_part_mbsize)

        if self.root_filesystem_is_overlay:
            log.info('--> creating readonly root partition')
            squashed_root_file = NamedTemporaryFile()
            squashed_root = FileSystemSquashFs(
                device_provider=None,
                root_dir=self.root_dir,
                custom_args={
                    'compression':
                    self.xml_state.build_type.get_squashfscompression()
                })
            squashed_root.create_on_file(
                filename=squashed_root_file.name,
                exclude=[Defaults.get_shared_cache_location()])
            squashed_rootfs_mbsize = int(
                os.path.getsize(squashed_root_file.name) /
                1048576) + Defaults.get_min_partition_mbytes()
            self.disk.create_root_readonly_partition(squashed_rootfs_mbsize)
            disksize_used_mbytes += squashed_rootfs_mbsize

        if self.spare_part_mbsize and self.spare_part_is_last:
            rootfs_mbsize = disksize_mbytes - disksize_used_mbytes - \
                self.spare_part_mbsize - Defaults.get_min_partition_mbytes()
        else:
            rootfs_mbsize = 'all_free'

        if self.volume_manager_name and self.volume_manager_name == 'lvm':
            log.info('--> creating LVM root partition')
            self.disk.create_root_lvm_partition(rootfs_mbsize)

        elif self.mdraid:
            log.info('--> creating mdraid root partition')
            self.disk.create_root_raid_partition(rootfs_mbsize)

        else:
            log.info('--> creating root partition')
            self.disk.create_root_partition(rootfs_mbsize)

        if self.spare_part_mbsize and self.spare_part_is_last:
            log.info('--> creating spare partition')
            self.disk.create_spare_partition('all_free')

        if self.firmware.bios_mode():
            log.info('--> setting active flag to primary boot partition')
            self.disk.activate_boot_partition()

        if self.firmware.ofw_mode():
            log.info('--> setting active flag to primary PReP partition')
            self.disk.activate_boot_partition()

        if self.firmware.efi_mode():
            if self.force_mbr:
                log.info('--> converting partition table to MBR')
                self.disk.create_mbr()
            elif self.hybrid_mbr:
                log.info('--> converting partition table to hybrid GPT/MBR')
                self.disk.create_hybrid_mbr()

        self.disk.map_partitions()

        return self.disk.get_device()

    def _write_partition_id_config_to_boot_image(self):
        log.info('Creating config.partids in boot system')
        filename = ''.join(
            [self.boot_image.boot_root_directory, '/config.partids'])
        partition_id_map = self.disk.get_public_partition_id_map()
        with open(filename, 'w') as partids:
            for id_name, id_value in list(partition_id_map.items()):
                partids.write('{0}="{1}"{2}'.format(id_name, id_value,
                                                    os.linesep))
        self.boot_image.include_file(os.sep + os.path.basename(filename))

    def _write_raid_config_to_boot_image(self):
        if self.mdraid:
            log.info('Creating etc/mdadm.conf in boot system')
            filename = ''.join(
                [self.boot_image.boot_root_directory, '/etc/mdadm.conf'])
            self.raid_root.create_raid_config(filename)
            self.boot_image.include_file(
                os.sep + os.sep.join(['etc', os.path.basename(filename)]))

    def _write_crypttab_to_system_image(self):
        if self.luks:
            log.info('Creating etc/crypttab')
            filename = ''.join([self.root_dir, '/etc/crypttab'])
            self.luks_root.create_crypttab(filename)
            self.boot_image.include_file(
                os.sep + os.sep.join(['etc', os.path.basename(filename)]))

    def _write_generic_fstab_to_system_image(self, device_map):
        log.info('Creating generic system etc/fstab')
        self._write_generic_fstab(device_map, self.system_setup)

    def _write_generic_fstab_to_boot_image(self, device_map):
        if self.initrd_system == 'kiwi':
            log.info('Creating generic boot image etc/fstab')
            self._write_generic_fstab(device_map, self.boot_image.setup)

    def _write_generic_fstab(self, device_map, setup):
        root_is_snapshot = \
            self.xml_state.build_type.get_btrfs_root_is_snapshot()
        root_is_readonly_snapshot = \
            self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()

        fs_check_interval = '1 1'
        custom_root_mount_args = list(self.custom_root_mount_args)
        if root_is_snapshot and root_is_readonly_snapshot:
            custom_root_mount_args += ['ro']
            fs_check_interval = '0 0'

        self._add_generic_fstab_entry(device_map['root'].get_device(), '/',
                                      custom_root_mount_args,
                                      fs_check_interval)
        if device_map.get('boot'):
            if self.bootloader == 'grub2_s390x_emu':
                boot_mount_point = '/boot/zipl'
            else:
                boot_mount_point = '/boot'
            self._add_generic_fstab_entry(device_map['boot'].get_device(),
                                          boot_mount_point)
        if device_map.get('efi'):
            self._add_generic_fstab_entry(device_map['efi'].get_device(),
                                          '/boot/efi')
        if self.volume_manager_name:
            volume_fstab_entries = self.system.get_fstab(
                self.persistency_type, self.requested_filesystem)
            for volume_fstab_entry in volume_fstab_entries:
                self.fstab.add_entry(volume_fstab_entry)
        if device_map.get('spare') and \
           self.spare_part_fs and self.spare_part_mountpoint:
            self._add_generic_fstab_entry(device_map['spare'].get_device(),
                                          self.spare_part_mountpoint)
        if device_map.get('swap'):
            if self.volume_manager_name and self.volume_manager_name == 'lvm':
                self._add_simple_fstab_entry(device_map['swap'].get_device(),
                                             'swap', 'swap')
            else:
                self._add_generic_fstab_entry(device_map['swap'].get_device(),
                                              'swap')
        setup.create_fstab(self.fstab)

    def _add_simple_fstab_entry(self,
                                device,
                                mount_point,
                                filesystem,
                                options=None,
                                check='0 0'):
        if not options:
            options = ['defaults']
        fstab_entry = ' '.join(
            [device, mount_point, filesystem, ','.join(options), check])
        self.fstab.add_entry(fstab_entry)

    def _add_generic_fstab_entry(self,
                                 device,
                                 mount_point,
                                 options=None,
                                 check='0 0'):
        if not options:
            options = ['defaults']
        block_operation = BlockID(device)
        blkid_type = 'LABEL' if self.persistency_type == 'by-label' else 'UUID'
        device_id = block_operation.get_blkid(blkid_type)
        fstab_entry = ' '.join([
            blkid_type + '=' + device_id, mount_point,
            block_operation.get_filesystem(), ','.join(options), check
        ])
        self.fstab.add_entry(fstab_entry)

    def _preserve_root_partition_uuid(self, device_map):
        block_operation = BlockID(device_map['root'].get_device())
        partition_uuid = block_operation.get_blkid('PARTUUID')
        if partition_uuid:
            self.xml_state.set_root_partition_uuid(partition_uuid)

    def _preserve_root_filesystem_uuid(self, device_map):
        block_operation = BlockID(device_map['root'].get_device())
        rootfs_uuid = block_operation.get_blkid('UUID')
        if rootfs_uuid:
            self.xml_state.set_root_filesystem_uuid(rootfs_uuid)

    def _write_image_identifier_to_system_image(self):
        log.info('Creating image identifier: %s', self.mbrid.get_id())
        self.mbrid.write(self.root_dir + '/boot/mbrid')

    def _write_recovery_metadata_to_boot_image(self):
        if os.path.exists(self.root_dir + '/recovery.partition.size'):
            log.info('Copying recovery metadata to boot image')
            recovery_metadata = ''.join(
                [self.root_dir, '/recovery.partition.size'])
            Command.run(
                ['cp', recovery_metadata, self.boot_image.boot_root_directory])
            self.boot_image.include_file(os.sep +
                                         os.path.basename(recovery_metadata))

    def _write_bootloader_meta_data_to_system_image(self, device_map):
        if self.bootloader != 'custom':
            log.info('Creating %s bootloader configuration', self.bootloader)
            boot_options = []
            if self.mdraid:
                boot_options.append('rd.auto')
            root_device = device_map['root']
            boot_device = root_device
            if 'boot' in device_map:
                boot_device = device_map['boot']

            root_uuid = self.disk.get_uuid(device_map['root'].get_device())
            boot_uuid = self.disk.get_uuid(boot_device.get_device())
            boot_uuid_unmapped = self.disk.get_uuid(
                device_map['luks_root'].get_device(
                )) if self.luks else boot_uuid
            self.bootloader_config.setup_disk_boot_images(boot_uuid_unmapped)
            self.bootloader_config.write_meta_data(
                root_uuid=root_uuid, boot_options=' '.join(boot_options))

            log.info('Creating config.bootoptions')
            filename = ''.join(
                [self.boot_image.boot_root_directory, '/config.bootoptions'])
            kexec_boot_options = ' '.join(
                [self.bootloader_config.get_boot_cmdline(root_uuid)] +
                boot_options)
            with open(filename, 'w') as boot_options:
                boot_options.write('{0}{1}'.format(kexec_boot_options,
                                                   os.linesep))

        partition_id_map = self.disk.get_public_partition_id_map()
        boot_partition_id = partition_id_map['kiwi_RootPart']
        if 'kiwi_BootPart' in partition_id_map:
            boot_partition_id = partition_id_map['kiwi_BootPart']

        self.system_setup.call_edit_boot_config_script(
            self.requested_filesystem, boot_partition_id)

    def _install_bootloader(self, device_map):
        root_device = device_map['root']
        boot_device = root_device
        if 'boot' in device_map:
            boot_device = device_map['boot']

        if 'readonly' in device_map:
            root_device = device_map['readonly']

        custom_install_arguments = {
            'boot_device': boot_device.get_device(),
            'root_device': root_device.get_device(),
            'firmware': self.firmware,
            'target_removable': self.target_removable
        }

        if 'efi' in device_map:
            efi_device = device_map['efi']
            custom_install_arguments.update(
                {'efi_device': efi_device.get_device()})

        if 'prep' in device_map:
            prep_device = device_map['prep']
            custom_install_arguments.update(
                {'prep_device': prep_device.get_device()})

        if self.volume_manager_name:
            self.system.umount_volumes()
            custom_install_arguments.update(
                {'system_volumes': self.system.get_volumes()})

        # create bootloader config prior bootloader installation
        self.bootloader_config.setup_disk_image_config(
            boot_options=custom_install_arguments)
        if 's390' in self.arch:
            self.bootloader_config.write()

        # cleanup bootloader config resources taken prior to next steps
        del self.bootloader_config

        if self.bootloader != 'custom':
            log.debug("custom arguments for bootloader installation %s",
                      custom_install_arguments)
            bootloader = BootLoaderInstall(self.bootloader, self.root_dir,
                                           self.disk.storage_provider,
                                           custom_install_arguments)
            if bootloader.install_required():
                bootloader.install()

        self.system_setup.call_edit_boot_install_script(
            self.diskname, boot_device.get_device())

    def _setup_property_root_is_readonly_snapshot(self):
        if self.volume_manager_name:
            root_is_snapshot = \
                self.xml_state.build_type.get_btrfs_root_is_snapshot()
            root_is_readonly_snapshot = \
                self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()
            if root_is_snapshot and root_is_readonly_snapshot:
                log.info('Setting root filesystem into read-only mode')
                self.system.mount_volumes()
                self.system.set_property_readonly_root()
                self.system.umount_volumes()

    def _copy_first_boot_files_to_system_image(self):
        boot_names = self.boot_image.get_boot_names()
        if self.initrd_system == 'kiwi':
            log.info('Copy boot files to system image')
            kernel = Kernel(self.boot_image.boot_root_directory)

            log.info('--> boot image kernel as %s', boot_names.kernel_name)
            kernel.copy_kernel(self.root_dir,
                               ''.join(['/boot/', boot_names.kernel_name]))

            if self.xen_server:
                if kernel.get_xen_hypervisor():
                    log.info('--> boot image Xen hypervisor as xen.gz')
                    kernel.copy_xen_hypervisor(self.root_dir, '/boot/xen.gz')
                else:
                    raise KiwiDiskBootImageError(
                        'No hypervisor in boot image tree %s found' %
                        self.boot_image.boot_root_directory)

        log.info('--> initrd archive as %s', boot_names.initrd_name)
        Command.run([
            'mv', self.boot_image.initrd_filename,
            self.root_dir + ''.join(['/boot/', boot_names.initrd_name])
        ])
Esempio n. 18
0
class DiskBuilder(object):
    """
    Disk image builder

    Attributes

    * :attr:`xml_state`
        Instance of XMLState

    * :attr:`target_dir`
        Target directory path name

    * :attr:`root_dir`
        Root directory path name

    * :attr:`custom_args`
        Custom processing arguments defined as hash keys:
        * signing_keys: list of package signing keys
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.arch = platform.machine()
        if self.arch == 'i686' or self.arch == 'i586':
            self.arch = 'ix86'
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.spare_part_mbsize = xml_state.get_build_type_spare_part_size()
        self.persistency_type = xml_state.build_type.get_devicepersistency()
        self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot(
        )
        self.custom_root_mount_args = xml_state.get_fs_mount_option_list()
        self.build_type_name = xml_state.get_build_type_name()
        self.image_format = xml_state.build_type.get_format()
        self.install_iso = xml_state.build_type.get_installiso()
        self.install_stick = xml_state.build_type.get_installstick()
        self.install_pxe = xml_state.build_type.get_installpxe()
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.volume_manager_name = xml_state.get_volume_management()
        self.volumes = xml_state.get_volumes()
        self.volume_group_name = xml_state.get_volume_group_name()
        self.mdraid = xml_state.build_type.get_mdraid()
        self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr()
        self.force_mbr = xml_state.build_type.get_force_mbr()
        self.luks = xml_state.build_type.get_luks()
        self.luks_os = xml_state.build_type.get_luksOS()
        self.xen_server = xml_state.is_xen_server()
        self.requested_filesystem = xml_state.build_type.get_filesystem()
        self.requested_boot_filesystem = \
            xml_state.build_type.get_bootfilesystem()
        self.bootloader = xml_state.build_type.get_bootloader()
        self.initrd_system = xml_state.get_initrd_system()
        self.target_removable = xml_state.build_type.get_target_removable()
        self.disk_setup = DiskSetup(xml_state, root_dir)
        self.custom_args = custom_args

        self.signing_keys = None
        if custom_args and 'signing_keys' in custom_args:
            self.signing_keys = custom_args['signing_keys']

        self.boot_image = BootImage(xml_state,
                                    target_dir,
                                    root_dir,
                                    signing_keys=self.signing_keys,
                                    custom_args=self.custom_args)
        self.firmware = FirmWare(xml_state)
        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.diskname = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '.' + self.arch,
            '-' + xml_state.get_image_version(), '.raw'
        ])
        self.install_media = self._install_image_requested()
        self.generic_fstab_entries = []

        # an instance of a class with the sync_data capability
        # representing the entire image system except for the boot/ area
        # which could live on another part of the disk
        self.system = None

        # an instance of a class with the sync_data capability
        # representing the boot/ area of the disk if not part of
        # self.system
        self.system_boot = None

        # an instance of a class with the sync_data capability
        # representing the boot/efi area of the disk
        self.system_efi = None

        # result store
        self.result = Result(xml_state)

    def create(self):
        """
        Build a bootable disk image and optional installation image
        The installation image is a bootable hybrid ISO image which
        embeds the disk image and an image installer

        Image types which triggers this builder are:

        * image="oem"
        * image="vmx"
        """
        disk = DiskBuilder(self.xml_state, self.target_dir, self.root_dir,
                           self.custom_args)
        result = disk.create_disk()

        # cleanup disk resources taken prior to next steps
        del disk

        disk_installer = DiskBuilder(self.xml_state, self.target_dir,
                                     self.root_dir)
        result = disk_installer.create_install_media(result)

        disk_format = DiskBuilder(self.xml_state, self.target_dir,
                                  self.root_dir)
        result = disk_format.create_disk_format(result)

        return result

    def create_disk(self):
        """
        Build a bootable raw disk image
        """
        if self.install_media and self.build_type_name != 'oem':
            raise KiwiInstallMediaError(
                'Install media requires oem type setup, got %s' %
                self.build_type_name)

        if self.root_filesystem_is_overlay and self.volume_manager_name:
            raise KiwiVolumeManagerSetupError(
                'Volume management together with root overlay is not supported'
            )

        # setup recovery archive, cleanup and create archive if requested
        self.system_setup.create_recovery_archive()

        # prepare boot(initrd) root system
        log.info('Preparing boot system')
        self.boot_image.prepare()

        # precalculate needed disk size
        disksize_mbytes = self.disk_setup.get_disksize_mbytes()

        # create the disk
        log.info('Creating raw disk image %s', self.diskname)
        loop_provider = LoopDevice(self.diskname, disksize_mbytes,
                                   self.blocksize)
        loop_provider.create()

        self.disk = Disk(self.firmware.get_partition_table_type(),
                         loop_provider)

        # create the bootloader instance
        self.bootloader_config = BootLoaderConfig(
            self.bootloader, self.xml_state, self.root_dir, {
                'targetbase':
                loop_provider.get_device(),
                'grub_directory_name':
                Defaults.get_grub_boot_directory_name(self.root_dir)
            })

        # create disk partitions and instance device map
        device_map = self._build_and_map_disk_partitions()

        # create raid on current root device if requested
        if self.mdraid:
            self.raid_root = RaidDevice(device_map['root'])
            self.raid_root.create_degraded_raid(raid_level=self.mdraid)
            device_map['root'] = self.raid_root.get_device()

        # create luks on current root device if requested
        if self.luks:
            self.luks_root = LuksDevice(device_map['root'])
            self.luks_root.create_crypto_luks(passphrase=self.luks,
                                              os=self.luks_os)
            device_map['root'] = self.luks_root.get_device()

        # create filesystems on boot partition(s) if any
        self._build_boot_filesystems(device_map)

        # create volumes and filesystems for root system
        if self.volume_manager_name:
            volume_manager_custom_parameters = {
                'fs_mount_options':
                self.custom_root_mount_args,
                'root_label':
                self.disk_setup.get_root_label(),
                'root_is_snapshot':
                self.xml_state.build_type.get_btrfs_root_is_snapshot(),
                'root_is_readonly_snapshot':
                self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot(
                ),
                'image_type':
                self.xml_state.get_build_type_name()
            }
            volume_manager = VolumeManager(self.volume_manager_name,
                                           device_map['root'],
                                           self.root_dir + '/', self.volumes,
                                           volume_manager_custom_parameters)
            volume_manager.setup(self.volume_group_name)
            volume_manager.create_volumes(self.requested_filesystem)
            volume_manager.mount_volumes()
            self.generic_fstab_entries += volume_manager.get_fstab(
                self.persistency_type, self.requested_filesystem)
            self.system = volume_manager
            device_map['root'] = volume_manager.get_device()['root']
        else:
            log.info('Creating root(%s) filesystem on %s',
                     self.requested_filesystem,
                     device_map['root'].get_device())
            filesystem_custom_parameters = {
                'mount_options': self.custom_root_mount_args
            }
            filesystem = FileSystem(self.requested_filesystem,
                                    device_map['root'], self.root_dir + '/',
                                    filesystem_custom_parameters)
            filesystem.create_on_device(label=self.disk_setup.get_root_label())
            self.system = filesystem

        # create a random image identifier
        self.mbrid = SystemIdentifier()
        self.mbrid.calculate_id()

        # create first stage metadata to boot image
        self._write_partition_id_config_to_boot_image()

        self._write_recovery_metadata_to_boot_image()

        self._write_raid_config_to_boot_image()

        self._write_generic_fstab_to_boot_image(device_map)

        self.system_setup.export_modprobe_setup(
            self.boot_image.boot_root_directory)

        # create first stage metadata to system image
        self._write_image_identifier_to_system_image()

        self._write_crypttab_to_system_image()

        self._write_generic_fstab_to_system_image(device_map)

        if self.root_filesystem_is_overlay:
            self._create_dracut_overlay_config()

        # create initrd cpio archive
        self.boot_image.create_initrd(self.mbrid)

        # create second stage metadata to system image
        self._copy_first_boot_files_to_system_image()

        self._write_bootloader_config_to_system_image(device_map)

        self.mbrid.write_to_disk(self.disk.storage_provider)

        # set SELinux file security contexts if context exists
        self._setup_selinux_file_contexts()

        # syncing system data to disk image
        log.info('Syncing system to image')
        if self.system_efi:
            log.info('--> Syncing EFI boot data to EFI partition')
            self.system_efi.sync_data()

        if self.system_boot:
            log.info('--> Syncing boot data at extra partition')
            self.system_boot.sync_data(
                self._get_exclude_list_for_boot_data_sync())

        log.info('--> Syncing root filesystem data')
        if self.root_filesystem_is_overlay:
            squashed_root_file = NamedTemporaryFile()
            squashed_root = FileSystemSquashFs(device_provider=None,
                                               root_dir=self.root_dir)
            squashed_root.create_on_file(
                filename=squashed_root_file.name,
                exclude=self._get_exclude_list_for_root_data_sync(device_map))
            Command.run([
                'dd',
                'if=%s' % squashed_root_file.name,
                'of=%s' % device_map['readonly'].get_device()
            ])
        else:
            self.system.sync_data(
                self._get_exclude_list_for_root_data_sync(device_map))

        # install boot loader
        self._install_bootloader(device_map)

        # set root filesystem properties
        self._setup_property_root_is_readonly_snapshot()

        # prepare for install media if requested
        if self.install_media:
            if self.initrd_system == 'dracut':
                # for the installation process we need a kiwi initrd
                # Therefore an extra install boot root system needs to
                # be prepared if dracut was set as the initrd system
                # to boot the system image
                log.info('Preparing extra install boot system')

                self.xml_state.build_type.set_initrd_system('kiwi')
                self.initrd_system = self.xml_state.get_initrd_system()

                self.boot_image = BootImageKiwi(self.xml_state,
                                                self.target_dir,
                                                signing_keys=self.signing_keys)

                self.boot_image.prepare()

                # apply disk builder metadata also needed in the install initrd
                self._write_partition_id_config_to_boot_image()
                self._write_recovery_metadata_to_boot_image()
                self._write_raid_config_to_boot_image()
                self.system_setup.export_modprobe_setup(
                    self.boot_image.boot_root_directory)

            log.info('Saving boot image instance to file')
            self.boot_image.dump(self.target_dir + '/boot_image.pickledump')

        # store image file name in result
        self.result.add(
            key='disk_image',
            filename=self.diskname,
            use_for_bundle=True if not self.image_format else False,
            compress=True,
            shasum=True)

        # create image root metadata
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)

        return self.result

    def create_disk_format(self, result_instance):
        """
        Create a bootable disk format from a previously
        created raw disk image
        """
        if self.image_format:
            log.info('Creating %s Disk Format', self.image_format)
            disk_format = DiskFormat(self.image_format, self.xml_state,
                                     self.root_dir, self.target_dir)
            disk_format.create_image_format()
            disk_format.store_to_result(result_instance)

        return result_instance

    def create_install_media(self, result_instance):
        """
        Build an installation image. The installation image is a
        bootable hybrid ISO image which embeds the raw disk image
        and an image installer
        """
        if self.install_media:
            install_image = InstallImageBuilder(
                self.xml_state, self.root_dir, self.target_dir,
                self._load_boot_image_instance(), self.custom_args)

            if self.install_iso or self.install_stick:
                log.info('Creating hybrid ISO installation image')
                install_image.create_install_iso()
                result_instance.add(key='installation_image',
                                    filename=install_image.isoname,
                                    use_for_bundle=True,
                                    compress=False,
                                    shasum=True)

            if self.install_pxe:
                log.info('Creating PXE installation archive')
                install_image.create_install_pxe_archive()
                result_instance.add(key='installation_pxe_archive',
                                    filename=install_image.pxename,
                                    use_for_bundle=True,
                                    compress=False,
                                    shasum=True)

        return result_instance

    def _load_boot_image_instance(self):
        boot_image_dump_file = self.target_dir + '/boot_image.pickledump'
        if not os.path.exists(boot_image_dump_file):
            raise KiwiInstallMediaError(
                'No boot image instance dump %s found' % boot_image_dump_file)
        try:
            with open(boot_image_dump_file, 'rb') as boot_image_dump:
                boot_image = pickle.load(boot_image_dump)
            boot_image.enable_cleanup()
            Path.wipe(boot_image_dump_file)
        except Exception as e:
            raise KiwiInstallMediaError('Failed to load boot image dump: %s' %
                                        type(e).__name__)
        return boot_image

    def _setup_selinux_file_contexts(self):
        security_context = '/etc/selinux/targeted/contexts/files/file_contexts'
        if os.path.exists(self.root_dir + security_context):
            self.system_setup.set_selinux_file_contexts(security_context)

    def _install_image_requested(self):
        if self.install_iso or self.install_stick or self.install_pxe:
            return True

    def _get_exclude_list_for_root_data_sync(self, device_map):
        exclude_list = Defaults.get_exclude_list_for_root_data_sync()
        if 'boot' in device_map and self.bootloader == 'grub2_s390x_emu':
            exclude_list.append('boot/zipl/*')
            exclude_list.append('boot/zipl/.*')
        elif 'boot' in device_map:
            exclude_list.append('boot/*')
            exclude_list.append('boot/.*')
        if 'efi' in device_map:
            exclude_list.append('boot/efi/*')
            exclude_list.append('boot/efi/.*')
        return exclude_list

    def _get_exclude_list_for_boot_data_sync(self):
        return ['efi/*']

    def _build_boot_filesystems(self, device_map):
        if 'efi' in device_map:
            log.info('Creating EFI(fat16) filesystem on %s',
                     device_map['efi'].get_device())
            filesystem = FileSystem('fat16', device_map['efi'],
                                    self.root_dir + '/boot/efi/')
            filesystem.create_on_device(label=self.disk_setup.get_efi_label())
            self.system_efi = filesystem

        if 'boot' in device_map:
            boot_filesystem = self.requested_boot_filesystem
            if not boot_filesystem:
                boot_filesystem = self.requested_filesystem
            boot_directory = self.root_dir + '/boot/'
            if self.bootloader == 'grub2_s390x_emu':
                boot_directory = self.root_dir + '/boot/zipl/'
                boot_filesystem = 'ext2'
            log.info('Creating boot(%s) filesystem on %s', boot_filesystem,
                     device_map['boot'].get_device())
            filesystem = FileSystem(boot_filesystem, device_map['boot'],
                                    boot_directory)
            filesystem.create_on_device(label=self.disk_setup.get_boot_label())
            self.system_boot = filesystem

    def _build_and_map_disk_partitions(self):
        self.disk.wipe()
        if self.firmware.legacy_bios_mode():
            log.info('--> creating EFI CSM(legacy bios) partition')
            self.disk.create_efi_csm_partition(
                self.firmware.get_legacy_bios_partition_size())

        if self.firmware.efi_mode():
            log.info('--> creating EFI partition')
            self.disk.create_efi_partition(
                self.firmware.get_efi_partition_size())

        if self.firmware.ofw_mode():
            log.info('--> creating PReP partition')
            self.disk.create_prep_partition(
                self.firmware.get_prep_partition_size())

        if self.disk_setup.need_boot_partition():
            log.info('--> creating boot partition')
            self.disk.create_boot_partition(
                self.disk_setup.boot_partition_size())

        if self.spare_part_mbsize:
            log.info('--> creating spare partition')
            self.disk.create_spare_partition(self.spare_part_mbsize)

        if self.root_filesystem_is_overlay:
            log.info('--> creating readonly root partition')
            squashed_root_file = NamedTemporaryFile()
            squashed_root = FileSystemSquashFs(device_provider=None,
                                               root_dir=self.root_dir)
            squashed_root.create_on_file(
                filename=squashed_root_file.name,
                exclude=[Defaults.get_shared_cache_location()])
            squashed_rootfs_mbsize = os.path.getsize(
                squashed_root_file.name) / 1048576
            self.disk.create_root_readonly_partition(
                int(squashed_rootfs_mbsize + 50))

        if self.volume_manager_name and self.volume_manager_name == 'lvm':
            log.info('--> creating LVM root partition')
            self.disk.create_root_lvm_partition('all_free')

        elif self.mdraid:
            log.info('--> creating mdraid root partition')
            self.disk.create_root_raid_partition('all_free')

        else:
            log.info('--> creating root partition')
            self.disk.create_root_partition('all_free')

        if self.firmware.bios_mode():
            log.info('--> setting active flag to primary boot partition')
            self.disk.activate_boot_partition()

        if self.firmware.ofw_mode():
            log.info('--> setting active flag to primary PReP partition')
            self.disk.activate_boot_partition()

        if self.firmware.efi_mode():
            if self.force_mbr:
                log.info('--> converting partition table to MBR')
                self.disk.create_mbr()
            elif self.hybrid_mbr:
                log.info('--> converting partition table to hybrid GPT/MBR')
                self.disk.create_hybrid_mbr()

        self.disk.map_partitions()

        return self.disk.get_device()

    def _create_dracut_overlay_config(self):
        overlay_config_file = ''.join(
            [self.root_dir, '/etc/dracut.conf.d/02-overlay.conf'])
        overlay_config = [
            'add_dracutmodules+=" kiwi-overlay "', 'hostonly="no"',
            'dracut_rescue_image="no"'
        ]
        with open(overlay_config_file, 'w') as config:
            for entry in overlay_config:
                config.write(entry + os.linesep)

    def _write_partition_id_config_to_boot_image(self):
        if self.initrd_system == 'kiwi':
            log.info('Creating config.partids in boot system')
            filename = ''.join(
                [self.boot_image.boot_root_directory, '/config.partids'])
            partition_id_map = self.disk.get_public_partition_id_map()
            with open(filename, 'w') as partids:
                for id_name, id_value in list(partition_id_map.items()):
                    partids.write('{0}="{1}"{2}'.format(
                        id_name, id_value, os.linesep))

    def _write_raid_config_to_boot_image(self):
        if self.mdraid:
            log.info('Creating etc/mdadm.conf in boot system')
            self.raid_root.create_raid_config(
                self.boot_image.boot_root_directory + '/etc/mdadm.conf')

    def _write_crypttab_to_system_image(self):
        if self.luks:
            log.info('Creating etc/crypttab')
            self.luks_root.create_crypttab(self.root_dir + '/etc/crypttab')

    def _write_generic_fstab_to_system_image(self, device_map):
        log.info('Creating generic system etc/fstab')
        self._write_generic_fstab(device_map, self.system_setup)

    def _write_generic_fstab_to_boot_image(self, device_map):
        if self.initrd_system == 'kiwi':
            log.info('Creating generic boot image etc/fstab')
            self._write_generic_fstab(device_map, self.boot_image.setup)

    def _write_generic_fstab(self, device_map, setup):
        root_is_snapshot = \
            self.xml_state.build_type.get_btrfs_root_is_snapshot()
        root_is_readonly_snapshot = \
            self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()

        fs_check_interval = '1 1'
        custom_root_mount_args = list(self.custom_root_mount_args)
        if root_is_snapshot and root_is_readonly_snapshot:
            custom_root_mount_args += ['ro']
            fs_check_interval = '0 0'

        self._add_generic_fstab_entry(device_map['root'].get_device(), '/',
                                      custom_root_mount_args,
                                      fs_check_interval)
        if 'boot' in device_map:
            if self.bootloader == 'grub2_s390x_emu':
                boot_mount_point = '/boot/zipl'
            else:
                boot_mount_point = '/boot'
            self._add_generic_fstab_entry(device_map['boot'].get_device(),
                                          boot_mount_point)
        if 'efi' in device_map:
            self._add_generic_fstab_entry(device_map['efi'].get_device(),
                                          '/boot/efi')
        setup.create_fstab(self.generic_fstab_entries)

    def _add_generic_fstab_entry(self,
                                 device,
                                 mount_point,
                                 options=None,
                                 check='0 0'):
        if not options:
            options = ['defaults']
        block_operation = BlockID(device)
        blkid_type = 'LABEL' if self.persistency_type == 'by-label' else 'UUID'
        device_id = block_operation.get_blkid(blkid_type)
        fstab_entry = ' '.join([
            blkid_type + '=' + device_id, mount_point,
            block_operation.get_filesystem(), ','.join(options), check
        ])
        if fstab_entry not in self.generic_fstab_entries:
            self.generic_fstab_entries.append(fstab_entry)

    def _write_image_identifier_to_system_image(self):
        log.info('Creating image identifier: %s', self.mbrid.get_id())
        self.mbrid.write(self.root_dir + '/boot/mbrid')

    def _write_recovery_metadata_to_boot_image(self):
        if os.path.exists(self.root_dir + '/recovery.partition.size'):
            log.info('Copying recovery metadata to boot image')
            Command.run([
                'cp', self.root_dir + '/recovery.partition.size',
                self.boot_image.boot_root_directory
            ])

    def _write_bootloader_config_to_system_image(self, device_map):
        if self.bootloader is not 'custom':
            log.info('Creating %s bootloader configuration', self.bootloader)
            boot_names = self._get_boot_names()
            boot_device = device_map['root']
            if 'boot' in device_map:
                boot_device = device_map['boot']

            root_uuid = self.disk.get_uuid(device_map['root'].get_device())
            boot_uuid = self.disk.get_uuid(boot_device.get_device())
            self.bootloader_config.setup_disk_boot_images(boot_uuid)
            self.bootloader_config.setup_disk_image_config(
                boot_uuid=boot_uuid,
                root_uuid=root_uuid,
                kernel=boot_names.kernel_name,
                initrd=boot_names.initrd_name)
            self.bootloader_config.write()

        partition_id_map = self.disk.get_public_partition_id_map()
        boot_partition_id = partition_id_map['kiwi_RootPart']
        if 'kiwi_BootPart' in partition_id_map:
            boot_partition_id = partition_id_map['kiwi_BootPart']

        self.system_setup.call_edit_boot_config_script(
            self.requested_filesystem, boot_partition_id)

    def _install_bootloader(self, device_map):
        root_device = device_map['root']
        boot_device = root_device
        if 'boot' in device_map:
            boot_device = device_map['boot']

        if 'readonly' in device_map:
            root_device = device_map['readonly']

        custom_install_arguments = {
            'boot_device': boot_device.get_device(),
            'root_device': root_device.get_device(),
            'firmware': self.firmware,
            'target_removable': self.target_removable
        }

        if 'efi' in device_map:
            efi_device = device_map['efi']
            custom_install_arguments.update(
                {'efi_device': efi_device.get_device()})

        if 'prep' in device_map:
            prep_device = device_map['prep']
            custom_install_arguments.update(
                {'prep_device': prep_device.get_device()})

        if self.volume_manager_name:
            self.system.umount_volumes()
            custom_install_arguments.update(
                {'system_volumes': self.system.get_volumes()})

        if self.bootloader is not 'custom':
            log.debug("custom arguments for bootloader installation %s",
                      custom_install_arguments)
            bootloader = BootLoaderInstall(self.bootloader, self.root_dir,
                                           self.disk.storage_provider,
                                           custom_install_arguments)
            if bootloader.install_required():
                bootloader.install()

        self.system_setup.call_edit_boot_install_script(
            self.diskname, boot_device.get_device())

    def _setup_property_root_is_readonly_snapshot(self):
        if self.volume_manager_name:
            root_is_snapshot = \
                self.xml_state.build_type.get_btrfs_root_is_snapshot()
            root_is_readonly_snapshot = \
                self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()
            if root_is_snapshot and root_is_readonly_snapshot:
                log.info('Setting root filesystem into read-only mode')
                self.system.mount_volumes()
                self.system.set_property_readonly_root()
                self.system.umount_volumes()

    def _copy_first_boot_files_to_system_image(self):
        boot_names = self._get_boot_names()
        if self.initrd_system == 'kiwi':
            log.info('Copy boot files to system image')
            kernel = Kernel(self.boot_image.boot_root_directory)

            log.info('--> boot image kernel as %s', boot_names.kernel_name)
            kernel.copy_kernel(self.root_dir,
                               ''.join(['/boot/', boot_names.kernel_name]))

            if self.xen_server:
                if kernel.get_xen_hypervisor():
                    log.info('--> boot image Xen hypervisor as xen.gz')
                    kernel.copy_xen_hypervisor(self.root_dir, '/boot/xen.gz')
                else:
                    raise KiwiDiskBootImageError(
                        'No hypervisor in boot image tree %s found' %
                        self.boot_image.boot_root_directory)

        log.info('--> initrd archive as %s', boot_names.initrd_name)
        Command.run([
            'mv', self.boot_image.initrd_filename,
            self.root_dir + ''.join(['/boot/', boot_names.initrd_name])
        ])

    def _get_boot_names(self):
        boot_names_type = namedtuple('boot_names_type',
                                     ['kernel_name', 'initrd_name'])
        kernel = Kernel(self.boot_image.boot_root_directory)
        kernel_info = kernel.get_kernel()
        if not kernel_info:
            raise KiwiDiskBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image.boot_root_directory)
        if self.initrd_system == 'dracut':
            dracut_output_format = self._get_dracut_output_file_format()
            return boot_names_type(kernel_name=kernel_info.name,
                                   initrd_name=dracut_output_format.format(
                                       kernel_version=kernel_info.version))
        else:
            return boot_names_type(kernel_name='linux.vmx',
                                   initrd_name='initrd.vmx')

    def _get_dracut_output_file_format(self):
        """
        Unfortunately the dracut initrd output file format varies between
        the different Linux distributions. Tools like lsinitrd, and also
        grub2 rely on the initrd output file to be in that format.
        Thus when kiwi uses dracut the same file format should be used
        all over the place in order to stay compatible with what the
        distribution does
        """
        default_outfile_format = 'initramfs-{kernel_version}.img'
        dracut_search_env = {
            'PATH': os.sep.join([self.root_dir, 'usr', 'bin'])
        }
        dracut_tool = Path.which('dracut',
                                 custom_env=dracut_search_env,
                                 access_mode=os.X_OK)
        if dracut_tool:
            outfile_expression = r'outfile="/boot/(init.*\$kernel.*)"'
            with open(dracut_tool) as dracut:
                outfile = re.findall(outfile_expression, dracut.read())[0]
                if outfile:
                    return outfile.replace('$kernel', '{kernel_version}')

        log.warning('Could not detect dracut output file format')
        log.warning('Using default initrd file name format {0}'.format(
            default_outfile_format))
        return default_outfile_format
Esempio n. 19
0
class TestResultBundleTask(object):
    def setup(self):
        sys.argv = [
            sys.argv[0], 'result', 'bundle', '--target-dir', 'target_dir',
            '--bundle-dir', 'bundle_dir', '--id', 'Build_42'
        ]
        self.abs_target_dir = os.path.abspath('target_dir')
        self.abs_bundle_dir = os.path.abspath('bundle_dir')
        self.context_manager_mock = mock.Mock()
        self.file_mock = mock.Mock()
        self.enter_mock = mock.Mock()
        self.exit_mock = mock.Mock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, '__enter__', self.enter_mock)
        setattr(self.context_manager_mock, '__exit__', self.exit_mock)

        self.file_mock.read.return_value = b'data'

        self.xml_state = mock.Mock()
        self.xml_state.get_image_version = mock.Mock(return_value='1.2.3')
        self.xml_state.xml_data.get_name = mock.Mock(return_value='test-image')

        self.result = Result(self.xml_state)
        self.result.add(key='keyname',
                        filename='test-image-1.2.3',
                        use_for_bundle=True,
                        compress=True,
                        shasum=True)

        kiwi.tasks.result_bundle.Help = mock.Mock(return_value=mock.Mock())
        self.task = ResultBundleTask()

        runtime_config = mock.Mock()
        runtime_config.is_bundle_compression_requested = mock.Mock(
            return_value=True)
        self.task.runtime_config = runtime_config

    def teardown(self):
        sys.argv = argv_kiwi_tests

    def _init_command_args(self):
        self.task.command_args = {}
        self.task.command_args['help'] = False
        self.task.command_args['bundle'] = False
        self.task.command_args['--target-dir'] = 'target_dir'
        self.task.command_args['--bundle-dir'] = 'bundle_dir'
        self.task.command_args['--id'] = 'Build_42'
        self.task.command_args['--zsync-source'] = None

    @raises(KiwiBundleError)
    def test_process_invalid_bundle_directory(self):
        self._init_command_args()
        self.task.command_args['--bundle-dir'] = \
            self.task.command_args['--target-dir']
        self.task.command_args['bundle'] = True
        self.task.process()

    @patch('kiwi.tasks.result_bundle.Result.load')
    @patch('kiwi.tasks.result_bundle.Command.run')
    @patch('kiwi.tasks.result_bundle.Path.create')
    @patch('kiwi.tasks.result_bundle.Path.which')
    @patch('kiwi.tasks.result_bundle.Compress')
    @patch('kiwi.tasks.result_bundle.Checksum')
    @patch('os.path.exists')
    @patch_open
    def test_process_result_bundle(self, mock_open, mock_exists, mock_checksum,
                                   mock_compress, mock_path_which,
                                   mock_path_create, mock_command, mock_load):
        # This file won't be copied with build id
        self.result.add(key='noversion',
                        filename='test-image-noversion',
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)

        checksum = mock.Mock()
        compress = mock.Mock()
        mock_path_which.return_value = 'zsyncmake'
        compress.compressed_filename = 'compressed_filename'
        mock_compress.return_value = compress
        mock_checksum.return_value = checksum
        mock_exists.return_value = False
        mock_open.return_value = self.context_manager_mock
        mock_load.return_value = self.result
        self._init_command_args()
        self.task.command_args['bundle'] = True
        self.task.command_args['--zsync-source'] = 'http://example.com/zsync'

        self.task.process()

        mock_load.assert_called_once_with(
            os.sep.join([self.abs_target_dir, 'kiwi.result']))
        mock_path_create.assert_called_once_with(self.abs_bundle_dir)
        assert mock_command.call_args_list == [
            call([
                'cp', 'test-image-1.2.3',
                os.sep.join([self.abs_bundle_dir, 'test-image-1.2.3-Build_42'])
            ]),
            call([
                'zsyncmake', '-e', '-u',
                'http://example.com/zsync/compressed_filename', '-o',
                'compressed_filename.zsync', 'compressed_filename'
            ]),
            call([
                'cp', 'test-image-noversion',
                os.sep.join([self.abs_bundle_dir, 'test-image-noversion'])
            ])
        ]
        mock_compress.assert_called_once_with(
            os.sep.join([self.abs_bundle_dir, 'test-image-1.2.3-Build_42']))
        mock_checksum.assert_called_once_with(compress.compressed_filename)
        checksum.sha256.assert_called_once_with()

    @patch('kiwi.tasks.result_bundle.Result.load')
    @patch('kiwi.tasks.result_bundle.Command.run')
    @patch('kiwi.tasks.result_bundle.Path.create')
    @patch('os.path.exists')
    def test_process_result_bundle_name_includes_version(
            self, mock_exists, mock_path_create, mock_command, mock_load):
        result = Result(self.xml_state)
        result.add(key='nameincludesversion',
                   filename='test-1.2.3-image-1.2.3',
                   use_for_bundle=True,
                   compress=False,
                   shasum=False)

        self.xml_state.xml_data.get_name = mock.Mock(
            return_value='test-1.2.3-image')

        mock_exists.return_value = False
        mock_load.return_value = result
        self._init_command_args()

        self.task.process()

        mock_load.assert_called_once_with(
            os.sep.join([self.abs_target_dir, 'kiwi.result']))
        mock_path_create.assert_called_once_with(self.abs_bundle_dir)
        assert mock_command.call_args_list == [
            call([
                'cp', 'test-1.2.3-image-1.2.3',
                os.sep.join(
                    [self.abs_bundle_dir, 'test-1.2.3-image-1.2.3-Build_42'])
            ])
        ]

    @patch('kiwi.logger.log.warning')
    @patch('kiwi.tasks.result_bundle.Result.load')
    @patch('kiwi.tasks.result_bundle.Command.run')
    @patch('kiwi.tasks.result_bundle.Path.create')
    @patch('kiwi.tasks.result_bundle.Path.which')
    @patch('kiwi.tasks.result_bundle.Compress')
    @patch('kiwi.tasks.result_bundle.Checksum')
    @patch('os.path.exists')
    @patch_open
    def test_process_result_bundle_zsyncmake_missing(
            self, mock_open, mock_exists, mock_checksum, mock_compress,
            mock_path_which, mock_path_create, mock_command, mock_load,
            mock_log):
        checksum = mock.Mock()
        compress = mock.Mock()
        mock_path_which.return_value = None
        compress.compressed_filename = 'compressed_filename'
        mock_compress.return_value = compress
        mock_checksum.return_value = checksum
        mock_exists.return_value = False
        mock_open.return_value = self.context_manager_mock
        mock_load.return_value = self.result
        self._init_command_args()
        self.task.command_args['bundle'] = True
        self.task.command_args['--zsync-source'] = 'http://example.com/zsync'

        self.task.process()
        mock_log.assert_called_once_with(
            '--> zsyncmake missing, zsync setup skipped')

    def test_process_result_bundle_help(self):
        self._init_command_args()
        self.task.command_args['help'] = True
        self.task.command_args['bundle'] = True
        self.task.process()
        self.task.manual.show.assert_called_once_with('kiwi::result::bundle')
Esempio n. 20
0
class ArchiveBuilder(object):
    """
    Root archive image builder

    Attributes

    * :attr:`xml_state`
        Instance of XMLState

    * :attr:`target_dir`
        target directory path name

    * :attr:`root_dir`
        root directory path name

    * :attr:`custom_args`
        Custom processing arguments defined as hash keys:
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.requested_archive_type = xml_state.get_build_type_name()
        self.result = Result(xml_state)
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.filename = self._target_file_for('tar.xz')
        self.checksum = self._target_file_for('md5')
        self.xz_options = custom_args['xz_options'] if custom_args \
            and 'xz_options' in custom_args else None

        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Create a root archive tarball

        Build a simple XZ compressed root tarball from the image root tree

        Image types which triggers this builder are:

        * image="tbz"
        """
        supported_archives = Defaults.get_archive_image_types()
        if self.requested_archive_type not in supported_archives:
            raise KiwiArchiveSetupError(
                'Unknown archive type: %s' % self.requested_archive_type
            )

        if self.requested_archive_type == 'tbz':
            log.info('Creating XZ compressed tar archive')
            archive = ArchiveTar(
                self._target_file_for('tar')
            )
            archive.create_xz_compressed(
                self.root_dir, xz_options=self.xz_options,
                exclude=Defaults.get_exclude_list_for_root_data_sync()
            )
            checksum = Checksum(self.filename)
            log.info('--> Creating archive checksum')
            checksum.md5(self.checksum)
            self.result.verify_image_size(
                self.runtime_config.get_max_size_constraint(),
                self.filename
            )
            self.result.add(
                key='root_archive',
                filename=self.filename,
                use_for_bundle=True,
                compress=False,
                shasum=True
            )
            self.result.add(
                key='root_archive_md5',
                filename=self.checksum,
                use_for_bundle=False
            )
            self.result.add(
                key='image_packages',
                filename=self.system_setup.export_package_list(
                    self.target_dir
                ),
                use_for_bundle=True,
                compress=False,
                shasum=False
            )
            self.result.add(
                key='image_verified',
                filename=self.system_setup.export_package_verification(
                    self.target_dir
                ),
                use_for_bundle=True,
                compress=False,
                shasum=False
            )
        return self.result

    def _target_file_for(self, suffix):
        return ''.join(
            [
                self.target_dir, '/',
                self.xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + self.xml_state.get_image_version(),
                '.', suffix
            ]
        )
Esempio n. 21
0
File: live.py Progetto: jfkw/kiwi
    def create(self) -> Result:
        """
        Build a bootable hybrid live ISO image

        Image types which triggers this builder are:

        * image="iso"

        :raises KiwiLiveBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        # media dir to store CD contents
        self.media_dir = mkdtemp(prefix='live-media.', dir=self.target_dir)

        # unpack cdroot user files to media dir
        self.system_setup.import_cdroot_files(self.media_dir)

        rootsize = SystemSize(self.media_dir)

        # custom iso metadata
        log.info('Using following live ISO metadata:')
        log.info('--> Application id: {0}'.format(self.mbrid.get_id()))
        log.info('--> Publisher: {0}'.format(Defaults.get_publisher()))
        log.info('--> Volume id: {0}'.format(self.volume_id))
        custom_iso_args = {
            'meta_data': {
                'publisher': self.publisher,
                'preparer': Defaults.get_preparer(),
                'volume_id': self.volume_id,
                'mbr_id': self.mbrid.get_id(),
                'efi_mode': self.firmware.efi_mode()
            }
        }

        log.info('Setting up live image bootloader configuration')
        if self.firmware.efi_mode():
            # setup bootloader config to boot the ISO via EFI
            # This also embedds an MBR and the respective BIOS modules
            # for compat boot. The complete bootloader setup will be
            # based on grub
            bootloader_config = BootLoaderConfig.new(
                'grub2',
                self.xml_state,
                root_dir=self.root_dir,
                boot_dir=self.media_dir,
                custom_args={
                    'grub_directory_name':
                    Defaults.get_grub_boot_directory_name(self.root_dir)
                })
            bootloader_config.setup_live_boot_images(mbrid=self.mbrid,
                                                     lookup_path=self.root_dir)
        else:
            # setup bootloader config to boot the ISO via isolinux.
            # This allows for booting on x86 platforms in BIOS mode
            # only.
            bootloader_config = BootLoaderConfig.new('isolinux',
                                                     self.xml_state,
                                                     root_dir=self.root_dir,
                                                     boot_dir=self.media_dir)
        IsoToolsBase.setup_media_loader_directory(
            self.boot_image.boot_root_directory, self.media_dir,
            bootloader_config.get_boot_theme())
        bootloader_config.write_meta_data()
        bootloader_config.setup_live_image_config(mbrid=self.mbrid)
        bootloader_config.write()

        # call custom editbootconfig script if present
        self.system_setup.call_edit_boot_config_script(
            filesystem='iso:{0}'.format(self.media_dir),
            boot_part_id=1,
            working_directory=self.root_dir)

        # prepare dracut initrd call
        self.boot_image.prepare()

        # create dracut initrd for live image
        log.info('Creating live ISO boot image')
        live_dracut_modules = Defaults.get_live_dracut_modules_from_flag(
            self.live_type)
        live_dracut_modules.append('pollcdrom')
        for dracut_module in live_dracut_modules:
            self.boot_image.include_module(dracut_module)
        self.boot_image.omit_module('multipath')
        self.boot_image.write_system_config_file(
            config={
                'modules': live_dracut_modules,
                'omit_modules': ['multipath']
            },
            config_file=self.root_dir + '/etc/dracut.conf.d/02-livecd.conf')
        self.boot_image.create_initrd(self.mbrid)

        # setup kernel file(s) and initrd in ISO boot layout
        log.info('Setting up kernel file(s) and boot image in ISO boot layout')
        self._setup_live_iso_kernel_and_initrd()

        # calculate size and decide if we need UDF
        if rootsize.accumulate_mbyte_file_sizes() > 4096:
            log.info('ISO exceeds 4G size, using UDF filesystem')
            custom_iso_args['meta_data']['udf'] = True

        # pack system into live boot structure as expected by dracut
        log.info('Packing system into dracut live ISO type: {0}'.format(
            self.live_type))
        root_filesystem = Defaults.get_default_live_iso_root_filesystem()
        filesystem_custom_parameters = {
            'mount_options': self.xml_state.get_fs_mount_option_list(),
            'create_options': self.xml_state.get_fs_create_option_list()
        }
        filesystem_setup = FileSystemSetup(self.xml_state, self.root_dir)
        root_image = NamedTemporaryFile()
        loop_provider = LoopDevice(
            root_image.name, filesystem_setup.get_size_mbytes(root_filesystem),
            self.xml_state.build_type.get_target_blocksize())
        loop_provider.create()
        live_filesystem = FileSystem.new(
            name=root_filesystem,
            device_provider=loop_provider,
            root_dir=self.root_dir + os.sep,
            custom_args=filesystem_custom_parameters)
        live_filesystem.create_on_device()
        log.info('--> Syncing data to {0} root image'.format(root_filesystem))
        live_filesystem.sync_data(
            Defaults.get_exclude_list_for_root_data_sync() +
            Defaults.get_exclude_list_from_custom_exclude_files(self.root_dir))
        live_filesystem.umount()

        log.info('--> Creating squashfs container for root image')
        self.live_container_dir = mkdtemp(prefix='live-container.',
                                          dir=self.target_dir)
        Path.create(self.live_container_dir + '/LiveOS')
        shutil.copy(root_image.name,
                    self.live_container_dir + '/LiveOS/rootfs.img')
        live_container_image = FileSystem.new(
            name='squashfs',
            device_provider=DeviceProvider(),
            root_dir=self.live_container_dir,
            custom_args={
                'compression':
                self.xml_state.build_type.get_squashfscompression()
            })
        container_image = NamedTemporaryFile()
        live_container_image.create_on_file(container_image.name)
        Path.create(self.media_dir + '/LiveOS')
        shutil.copy(container_image.name,
                    self.media_dir + '/LiveOS/squashfs.img')

        # create iso filesystem from media_dir
        log.info('Creating live ISO image')
        iso_image = FileSystemIsoFs(device_provider=DeviceProvider(),
                                    root_dir=self.media_dir,
                                    custom_args=custom_iso_args)
        iso_image.create_on_file(self.isoname)

        # include metadata for checkmedia tool
        if self.xml_state.build_type.get_mediacheck() is True:
            Iso.set_media_tag(self.isoname)

        Result.verify_image_size(self.runtime_config.get_max_size_constraint(),
                                 self.isoname)
        self.result.add(key='live_image',
                        filename=self.isoname,
                        use_for_bundle=True,
                        compress=False,
                        shasum=True)
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_changes',
                        filename=self.system_setup.export_package_changes(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=True,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        return self.result
Esempio n. 22
0
    def __init__(self,
                 xml_state: XMLState,
                 target_dir: str,
                 root_dir: str,
                 custom_args: Dict = None):
        self.arch = Defaults.get_platform_name()
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.spare_part_mbsize = xml_state.get_build_type_spare_part_size()
        self.spare_part_fs = xml_state.build_type.get_spare_part_fs()
        self.spare_part_is_last = xml_state.build_type.get_spare_part_is_last()
        self.spare_part_mountpoint = \
            xml_state.build_type.get_spare_part_mountpoint()
        self.persistency_type = xml_state.build_type.get_devicepersistency()
        self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot(
        )
        self.custom_root_mount_args = xml_state.get_fs_mount_option_list()
        self.custom_root_creation_args = xml_state.get_fs_create_option_list()
        self.build_type_name = xml_state.get_build_type_name()
        self.image_format = xml_state.build_type.get_format()
        self.install_iso = xml_state.build_type.get_installiso()
        self.install_stick = xml_state.build_type.get_installstick()
        self.install_pxe = xml_state.build_type.get_installpxe()
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.volume_manager_name = xml_state.get_volume_management()
        self.volumes = xml_state.get_volumes()
        self.custom_partitions = xml_state.get_partitions()
        self.volume_group_name = xml_state.get_volume_group_name()
        self.mdraid = xml_state.build_type.get_mdraid()
        self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr()
        self.force_mbr = xml_state.build_type.get_force_mbr()
        self.luks = xml_state.build_type.get_luks()
        self.luks_os = xml_state.build_type.get_luksOS()
        self.xen_server = xml_state.is_xen_server()
        self.requested_filesystem = xml_state.build_type.get_filesystem()
        self.requested_boot_filesystem = \
            xml_state.build_type.get_bootfilesystem()
        self.bootloader = xml_state.get_build_type_bootloader_name()
        self.initrd_system = xml_state.get_initrd_system()
        self.target_removable = xml_state.build_type.get_target_removable()
        self.root_filesystem_is_multipath = \
            xml_state.get_oemconfig_oem_multipath_scan()
        self.disk_resize_requested = \
            xml_state.get_oemconfig_oem_resize()
        self.swap_mbytes = xml_state.get_oemconfig_swap_mbytes()
        self.disk_setup = DiskSetup(xml_state, root_dir)
        self.unpartitioned_bytes = \
            xml_state.get_build_type_unpartitioned_bytes()
        self.custom_args = custom_args

        self.signing_keys = None
        if custom_args and 'signing_keys' in custom_args:
            self.signing_keys = custom_args['signing_keys']

        self.boot_image = BootImage.new(xml_state,
                                        target_dir,
                                        root_dir,
                                        signing_keys=self.signing_keys)
        self.firmware = FirmWare(xml_state)
        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.bundle_format = xml_state.get_build_type_bundle_format()
        self.diskname = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '.' + self.arch,
            '-' + xml_state.get_image_version(), '.raw'
        ])
        self.boot_is_crypto = True if self.luks and not \
            self.disk_setup.need_boot_partition() else False
        self.install_media = self._install_image_requested()
        self.fstab = Fstab()

        # result store
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

        if not self.boot_image.has_initrd_support():
            log.warning('Building without initrd support !')
Esempio n. 23
0
    def create_disk(self) -> Result:
        """
        Build a bootable raw disk image

        :raises KiwiInstallMediaError:
            if install media is required and image type is not oem
        :raises KiwiVolumeManagerSetupError:
            root overlay at the same time volumes are defined is not supported

        :return: result

        :rtype: instance of :class:`Result`
        """
        # an instance of a class with the sync_data capability
        # representing the entire image system except for the boot/ area
        # which could live on another part of the disk
        system: Any = None

        # an instance of a class with the sync_data capability
        # representing the boot/ area of the disk if not part of
        # self.system
        system_boot: Optional[FileSystemBase] = None

        # an instance of a class with the sync_data capability
        # representing the boot/efi area of the disk
        system_efi: Optional[FileSystemBase] = None

        # an instance of a class with the sync_data capability
        # representing the spare_part_mountpoint area of the disk
        system_spare: Optional[FileSystemBase] = None

        # a list of instances with the sync_data capability
        # representing the custom partitions area of the disk
        system_custom_parts: List[FileSystemBase] = []

        if self.install_media and self.build_type_name != 'oem':
            raise KiwiInstallMediaError(
                'Install media requires oem type setup, got {0}'.format(
                    self.build_type_name))

        if self.root_filesystem_is_overlay and self.volume_manager_name:
            raise KiwiVolumeManagerSetupError(
                'Volume management together with root overlay is not supported'
            )

        # setup recovery archive, cleanup and create archive if requested
        self.system_setup.create_recovery_archive()

        # prepare initrd
        if self.boot_image.has_initrd_support():
            log.info('Preparing boot system')
            self.boot_image.prepare()

        # precalculate needed disk size
        disksize_mbytes = self.disk_setup.get_disksize_mbytes()

        # create the disk
        log.info('Creating raw disk image %s', self.diskname)
        loop_provider = LoopDevice(self.diskname, disksize_mbytes,
                                   self.blocksize)
        loop_provider.create()

        disk = Disk(self.firmware.get_partition_table_type(), loop_provider,
                    self.xml_state.get_disk_start_sector())

        # create the bootloader instance
        if self.bootloader != 'custom':
            self.bootloader_config = BootLoaderConfig.new(
                self.bootloader,
                self.xml_state,
                root_dir=self.root_dir,
                boot_dir=self.root_dir,
                custom_args={
                    'targetbase':
                    loop_provider.get_device(),
                    'grub_directory_name':
                    Defaults.get_grub_boot_directory_name(self.root_dir),
                    'crypto_disk':
                    True if self.luks is not None else False,
                    'boot_is_crypto':
                    self.boot_is_crypto
                })

        # create disk partitions and instance device map
        device_map = self._build_and_map_disk_partitions(disk, disksize_mbytes)

        # create raid on current root device if requested
        raid_root = None
        if self.mdraid:
            raid_root = RaidDevice(device_map['root'])
            raid_root.create_degraded_raid(raid_level=self.mdraid)
            device_map['root'] = raid_root.get_device()
            disk.public_partition_id_map['kiwi_RaidPart'] = \
                disk.public_partition_id_map['kiwi_RootPart']
            disk.public_partition_id_map['kiwi_RaidDev'] = \
                device_map['root'].get_device()

        # create luks on current root device if requested
        luks_root = None
        if self.luks is not None:
            luks_root = LuksDevice(device_map['root'])
            self.luks_boot_keyname = '/.root.keyfile'
            self.luks_boot_keyfile = ''.join(
                [self.root_dir, self.luks_boot_keyname])
            # use LUKS key file for the following conditions:
            # 1. /boot is encrypted
            #    In this case grub needs to read from LUKS via the
            #    cryptodisk module which at the moment always asks
            #    for the passphrase even when empty. The keyfile
            #    setup makes sure only one interaction on the grub
            #    stage is needed
            # 2. LUKS passphrase is configured as empty string
            #    In this case the keyfile allows to open the
            #    LUKS pool without asking
            #
            luks_need_keyfile = \
                True if self.boot_is_crypto or self.luks == '' else False
            luks_root.create_crypto_luks(
                passphrase=self.luks,
                os=self.luks_os,
                keyfile=self.luks_boot_keyfile if luks_need_keyfile else '')
            if luks_need_keyfile:
                self.luks_boot_keyfile_setup = ''.join(
                    [self.root_dir, '/etc/dracut.conf.d/99-luks-boot.conf'])
                self.boot_image.write_system_config_file(
                    config={'install_items': [self.luks_boot_keyname]},
                    config_file=self.luks_boot_keyfile_setup)
                self.boot_image.include_file(
                    os.sep + os.path.basename(self.luks_boot_keyfile))
            device_map['luks_root'] = device_map['root']
            device_map['root'] = luks_root.get_device()

        # create spare filesystem on spare partition if present
        system_spare = self._build_spare_filesystem(device_map)

        system_custom_parts = self._build_custom_parts_filesystem(
            device_map, self.custom_partitions)

        # create filesystems on boot partition(s) if any
        system_boot, system_efi = self._build_boot_filesystems(device_map)

        # create volumes and filesystems for root system
        if self.volume_manager_name:
            volume_manager_custom_parameters = {
                'fs_mount_options':
                self.custom_root_mount_args,
                'fs_create_options':
                self.custom_root_creation_args,
                'root_label':
                self.disk_setup.get_root_label(),
                'root_is_snapshot':
                self.xml_state.build_type.get_btrfs_root_is_snapshot(),
                'root_is_readonly_snapshot':
                self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot(
                ),
                'quota_groups':
                self.xml_state.build_type.get_btrfs_quota_groups(),
                'resize_on_boot':
                self.disk_resize_requested
            }
            volume_manager = VolumeManager.new(
                self.volume_manager_name, device_map, self.root_dir + '/',
                self.volumes, volume_manager_custom_parameters)
            volume_manager.setup(self.volume_group_name)
            volume_manager.create_volumes(self.requested_filesystem)
            volume_manager.mount_volumes()
            system = volume_manager
            device_map['root'] = volume_manager.get_device().get('root')
            device_map['swap'] = volume_manager.get_device().get('swap')
        else:
            log.info('Creating root(%s) filesystem on %s',
                     self.requested_filesystem,
                     device_map['root'].get_device())
            filesystem_custom_parameters = {
                'mount_options': self.custom_root_mount_args,
                'create_options': self.custom_root_creation_args
            }
            filesystem = FileSystem.new(self.requested_filesystem,
                                        device_map['root'],
                                        self.root_dir + '/',
                                        filesystem_custom_parameters)
            filesystem.create_on_device(label=self.disk_setup.get_root_label())
            system = filesystem

        # create swap on current root device if requested
        if self.swap_mbytes:
            swap = FileSystem.new('swap', device_map['swap'])
            swap.create_on_device(label='SWAP')

        # store root partition/filesystem uuid for profile
        self._preserve_root_partition_uuid(device_map)
        self._preserve_root_filesystem_uuid(device_map)

        # create a random image identifier
        self.mbrid = SystemIdentifier()
        self.mbrid.calculate_id()

        # create first stage metadata to boot image
        self._write_partition_id_config_to_boot_image(disk)

        self._write_recovery_metadata_to_boot_image()

        self._write_raid_config_to_boot_image(raid_root)

        self._write_generic_fstab_to_boot_image(device_map, system)

        self.system_setup.export_modprobe_setup(
            self.boot_image.boot_root_directory)

        # create first stage metadata to system image
        self._write_image_identifier_to_system_image()

        self._write_crypttab_to_system_image(luks_root)

        self._write_generic_fstab_to_system_image(device_map, system)

        if self.initrd_system == 'dracut':
            if self.root_filesystem_is_multipath is False:
                self.boot_image.omit_module('multipath')
            if self.root_filesystem_is_overlay:
                self.boot_image.include_module('kiwi-overlay')
                self.boot_image.write_system_config_file(
                    config={'modules': ['kiwi-overlay']})
            if self.disk_resize_requested:
                self.boot_image.include_module('kiwi-repart')

        # create initrd
        if self.boot_image.has_initrd_support():
            self.boot_image.create_initrd(self.mbrid)

        # create second stage metadata to system image
        self._copy_first_boot_files_to_system_image()

        self._write_bootloader_meta_data_to_system_image(device_map, disk)

        self.mbrid.write_to_disk(disk.storage_provider)

        # set SELinux file security contexts if context exists
        self._setup_selinux_file_contexts()

        # syncing system data to disk image
        self._sync_system_to_image(device_map, system, system_boot, system_efi,
                                   system_spare, system_custom_parts)

        # run post sync script hook
        if self.system_setup.script_exists(defaults.POST_DISK_SYNC_SCRIPT):
            disk_system = SystemSetup(self.xml_state, system.get_mountpoint())
            disk_system.import_description()
            disk_system.call_disk_script()
            disk_system.cleanup()

        # install boot loader
        self._install_bootloader(device_map, disk, system)

        # set root filesystem properties
        self._setup_property_root_is_readonly_snapshot(system)

        Result.verify_image_size(self.runtime_config.get_max_size_constraint(),
                                 self.diskname)
        # store image bundle_format in result
        if self.bundle_format:
            self.result.add_bundle_format(self.bundle_format)

        # store image file name in result
        compression = self.runtime_config.get_bundle_compression(default=True)
        if self.luks is not None:
            compression = False
        self.result.add(
            key='disk_image',
            filename=self.diskname,
            use_for_bundle=True if not self.image_format else False,
            compress=compression,
            shasum=True)

        # create image root metadata
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_changes',
                        filename=self.system_setup.export_package_changes(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=True,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)

        return self.result
Esempio n. 24
0
 def test_load_failed(self, mock_exists, mock_pickle_load):
     mock_exists.return_value = True
     mock_pickle_load.side_effect = Exception
     Result.load('kiwi.result')
Esempio n. 25
0
class PxeBuilder(object):
    """
    **Filesystem based PXE image builder.**

    :param object xml_state: instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: system image root directory
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * signing_keys: list of package signing keys
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.target_dir = target_dir
        self.compressed = xml_state.build_type.get_compressed()
        self.xen_server = xml_state.is_xen_server()
        self.pxedeploy = xml_state.get_build_type_pxedeploy_section()
        self.filesystem = FileSystemBuilder(
            xml_state, target_dir, root_dir + '/'
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=root_dir
        )

        self.boot_signing_keys = custom_args['signing_keys'] if custom_args \
            and 'signing_keys' in custom_args else None

        self.xz_options = custom_args['xz_options'] if custom_args \
            and 'xz_options' in custom_args else None

        self.boot_image_task = BootImage(
            xml_state, target_dir,
            signing_keys=self.boot_signing_keys, custom_args=custom_args
        )
        self.image_name = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version()
            ]
        )
        self.archive_name = ''.join([self.image_name, '.tar.xz'])
        self.kernel_filename = None
        self.hypervisor_filename = None
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a pxe image set consisting out of a boot image(initrd)
        plus its appropriate kernel files and the root filesystem
        image with a checksum. The result can be used within the kiwi
        PXE boot infrastructure

        Image types which triggers this builder are:

        * image="pxe"

        :raises KiwiPxeBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        log.info('Creating PXE root filesystem image')
        self.filesystem.create()
        os.rename(
            self.filesystem.filename, self.image_name
        )
        self.image = self.image_name
        if self.compressed:
            log.info('xz compressing root filesystem image')
            compress = Compress(self.image)
            compress.xz(self.xz_options)
            self.image = compress.compressed_filename

        log.info('Creating PXE root filesystem MD5 checksum')
        self.filesystem_checksum = ''.join([self.image, '.md5'])
        checksum = Checksum(self.image)
        checksum.md5(self.filesystem_checksum)

        # prepare boot(initrd) root system
        log.info('Creating PXE boot image')
        self.boot_image_task.prepare()

        # export modprobe configuration to boot image
        self.system_setup.export_modprobe_setup(
            self.boot_image_task.boot_root_directory
        )

        # extract kernel from boot(initrd) root system
        kernel = Kernel(self.boot_image_task.boot_root_directory)
        kernel_data = kernel.get_kernel()
        if kernel_data:
            self.kernel_filename = ''.join(
                [
                    os.path.basename(self.image_name), '-',
                    kernel_data.version, '.kernel'
                ]
            )
            kernel.copy_kernel(
                self.target_dir, self.kernel_filename
            )
        else:
            raise KiwiPxeBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image_task.boot_root_directory
            )

        # extract hypervisor from boot(initrd) root system
        if self.xen_server:
            kernel_data = kernel.get_xen_hypervisor()
            if kernel_data:
                self.hypervisor_filename = ''.join(
                    [os.path.basename(self.image_name), '-', kernel_data.name]
                )
                kernel.copy_xen_hypervisor(
                    self.target_dir, self.hypervisor_filename
                )
                self.result.add(
                    key='xen_hypervisor',
                    filename=self.target_dir + '/' + self.hypervisor_filename,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )
            else:
                raise KiwiPxeBootImageError(
                    'No hypervisor in boot image tree %s found' %
                    self.boot_image_task.boot_root_directory
                )

        # create initrd for pxe boot
        self.boot_image_task.create_initrd()

        # put results into a tarball
        if not self.xz_options:
            self.xz_options = Defaults.get_xz_compression_options()
        bash_command = [
            'tar', '-C', self.target_dir, '-c', '--to-stdout'
        ] + [
            self.kernel_filename,
            os.path.basename(self.boot_image_task.initrd_filename),
            os.path.basename(self.image),
            os.path.basename(self.filesystem_checksum)
        ] + [
            '|', 'xz', '-f'
        ] + self.xz_options + [
            '>', self.archive_name
        ]
        Command.run(['bash', '-c', ' '.join(bash_command)])

        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.archive_name
        )
        # store results
        self.result.add(
            key='pxe_archive',
            filename=self.archive_name,
            use_for_bundle=True,
            compress=False,
            shasum=True
        )

        # create image root metadata
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )

        if self.pxedeploy:
            log.warning(
                'Creation of client config file from pxedeploy not implemented'
            )

        return self.result
Esempio n. 26
0
File: live.py Progetto: isbm/kiwi
class LiveImageBuilder:
    """
    **Live image builder**

    :param object xml_state: instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: root directory path name
    :param dict custom_args: Custom processing arguments
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.media_dir = None
        self.live_container_dir = None
        self.arch = Defaults.get_platform_name()
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.live_type = xml_state.build_type.get_flags()
        self.volume_id = xml_state.build_type.get_volid() or \
            Defaults.get_volume_id()
        self.mbrid = SystemIdentifier()
        self.mbrid.calculate_id()
        self.publisher = xml_state.build_type.get_publisher() or \
            Defaults.get_publisher()
        self.custom_args = custom_args

        if not self.live_type:
            self.live_type = Defaults.get_default_live_iso_type()

        self.boot_image = BootImageDracut(xml_state, target_dir, self.root_dir)
        self.firmware = FirmWare(xml_state)
        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.isoname = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '.' + Defaults.get_platform_name(),
            '-' + xml_state.get_image_version(), '.iso'
        ])
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a bootable hybrid live ISO image

        Image types which triggers this builder are:

        * image="iso"

        :raises KiwiLiveBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        # media dir to store CD contents
        self.media_dir = mkdtemp(prefix='live-media.', dir=self.target_dir)

        # unpack cdroot user files to media dir
        self.system_setup.import_cdroot_files(self.media_dir)

        rootsize = SystemSize(self.media_dir)

        # custom iso metadata
        log.info('Using following live ISO metadata:')
        log.info('--> Application id: {0}'.format(self.mbrid.get_id()))
        log.info('--> Publisher: {0}'.format(Defaults.get_publisher()))
        log.info('--> Volume id: {0}'.format(self.volume_id))
        custom_iso_args = {
            'meta_data': {
                'publisher': self.publisher,
                'preparer': Defaults.get_preparer(),
                'volume_id': self.volume_id,
                'mbr_id': self.mbrid.get_id(),
                'efi_mode': self.firmware.efi_mode()
            }
        }

        log.info('Setting up live image bootloader configuration')
        if self.firmware.efi_mode():
            # setup bootloader config to boot the ISO via EFI
            # This also embedds an MBR and the respective BIOS modules
            # for compat boot. The complete bootloader setup will be
            # based on grub
            bootloader_config = BootLoaderConfig(
                'grub2',
                self.xml_state,
                root_dir=self.root_dir,
                boot_dir=self.media_dir,
                custom_args={
                    'grub_directory_name':
                    Defaults.get_grub_boot_directory_name(self.root_dir)
                })
            bootloader_config.setup_live_boot_images(mbrid=self.mbrid,
                                                     lookup_path=self.root_dir)
        else:
            # setup bootloader config to boot the ISO via isolinux.
            # This allows for booting on x86 platforms in BIOS mode
            # only.
            bootloader_config = BootLoaderConfig('isolinux',
                                                 self.xml_state,
                                                 root_dir=self.root_dir,
                                                 boot_dir=self.media_dir)
        IsoToolsBase.setup_media_loader_directory(
            self.boot_image.boot_root_directory, self.media_dir,
            bootloader_config.get_boot_theme())
        bootloader_config.write_meta_data()
        bootloader_config.setup_live_image_config(mbrid=self.mbrid)
        bootloader_config.write()

        # call custom editbootconfig script if present
        self.system_setup.call_edit_boot_config_script(
            filesystem='iso:{0}'.format(self.media_dir),
            boot_part_id=1,
            working_directory=self.root_dir)

        # prepare dracut initrd call
        self.boot_image.prepare()

        # create dracut initrd for live image
        log.info('Creating live ISO boot image')
        live_dracut_modules = Defaults.get_live_dracut_modules_from_flag(
            self.live_type)
        live_dracut_modules.append('pollcdrom')
        for dracut_module in live_dracut_modules:
            self.boot_image.include_module(dracut_module)
        self.boot_image.omit_module('multipath')
        self.boot_image.write_system_config_file(
            config={
                'modules': live_dracut_modules,
                'omit_modules': ['multipath']
            },
            config_file=self.root_dir + '/etc/dracut.conf.d/02-livecd.conf')
        self.boot_image.create_initrd(self.mbrid)

        # setup kernel file(s) and initrd in ISO boot layout
        log.info('Setting up kernel file(s) and boot image in ISO boot layout')
        self._setup_live_iso_kernel_and_initrd()

        # calculate size and decide if we need UDF
        if rootsize.accumulate_mbyte_file_sizes() > 4096:
            log.info('ISO exceeds 4G size, using UDF filesystem')
            custom_iso_args['meta_data']['udf'] = True

        # pack system into live boot structure as expected by dracut
        log.info('Packing system into dracut live ISO type: {0}'.format(
            self.live_type))
        root_filesystem = Defaults.get_default_live_iso_root_filesystem()
        filesystem_custom_parameters = {
            'mount_options': self.xml_state.get_fs_mount_option_list(),
            'create_options': self.xml_state.get_fs_create_option_list()
        }
        filesystem_setup = FileSystemSetup(self.xml_state, self.root_dir)
        root_image = NamedTemporaryFile()
        loop_provider = LoopDevice(
            root_image.name, filesystem_setup.get_size_mbytes(root_filesystem),
            self.xml_state.build_type.get_target_blocksize())
        loop_provider.create()
        live_filesystem = FileSystem(name=root_filesystem,
                                     device_provider=loop_provider,
                                     root_dir=self.root_dir + os.sep,
                                     custom_args=filesystem_custom_parameters)
        live_filesystem.create_on_device()
        log.info('--> Syncing data to {0} root image'.format(root_filesystem))
        live_filesystem.sync_data(
            Defaults.get_exclude_list_for_root_data_sync())
        log.info('--> Creating squashfs container for root image')
        self.live_container_dir = mkdtemp(prefix='live-container.',
                                          dir=self.target_dir)
        Path.create(self.live_container_dir + '/LiveOS')
        shutil.copy(root_image.name,
                    self.live_container_dir + '/LiveOS/rootfs.img')
        live_container_image = FileSystem(
            name='squashfs',
            device_provider=None,
            root_dir=self.live_container_dir,
            custom_args={
                'compression':
                self.xml_state.build_type.get_squashfscompression()
            })
        container_image = NamedTemporaryFile()
        live_container_image.create_on_file(container_image.name)
        Path.create(self.media_dir + '/LiveOS')
        shutil.copy(container_image.name,
                    self.media_dir + '/LiveOS/squashfs.img')

        # create iso filesystem from media_dir
        log.info('Creating live ISO image')
        iso_image = FileSystemIsoFs(device_provider=None,
                                    root_dir=self.media_dir,
                                    custom_args=custom_iso_args)
        iso_image.create_on_file(self.isoname)

        # include metadata for checkmedia tool
        if self.xml_state.build_type.get_mediacheck() is True:
            Iso.set_media_tag(self.isoname)

        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(), self.isoname)
        self.result.add(key='live_image',
                        filename=self.isoname,
                        use_for_bundle=True,
                        compress=False,
                        shasum=True)
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        return self.result

    def _setup_live_iso_kernel_and_initrd(self):
        """
        Copy kernel and initrd from the root tree into the iso boot structure
        """
        boot_path = ''.join([self.media_dir, '/boot/', self.arch, '/loader'])
        Path.create(boot_path)

        # Move kernel files to iso filesystem structure
        kernel = Kernel(self.boot_image.boot_root_directory)
        if kernel.get_kernel():
            kernel.copy_kernel(boot_path, '/linux')
        else:
            raise KiwiLiveBootImageError(
                'No kernel in boot image tree {0} found'.format(
                    self.boot_image.boot_root_directory))
        if self.xml_state.is_xen_server():
            if kernel.get_xen_hypervisor():
                kernel.copy_xen_hypervisor(boot_path, '/xen.gz')
            else:
                raise KiwiLiveBootImageError(
                    'No hypervisor in boot image tree {0} found'.format(
                        self.boot_image.boot_root_directory))

        # Move initrd to iso filesystem structure
        if os.path.exists(self.boot_image.initrd_filename):
            shutil.move(self.boot_image.initrd_filename, boot_path + '/initrd')
        else:
            raise KiwiLiveBootImageError(
                'No boot image {0} in boot image tree {1} found'.format(
                    self.boot_image.initrd_filename,
                    self.boot_image.boot_root_directory))

    def __del__(self):
        if self.media_dir or self.live_container_dir:
            log.info('Cleaning up {0} instance'.format(type(self).__name__))
            if self.media_dir:
                Path.wipe(self.media_dir)
            if self.live_container_dir:
                Path.wipe(self.live_container_dir)
Esempio n. 27
0
class TestResultBundleTask:
    @fixture(autouse=True)
    def inject_fixtures(self, caplog):
        self._caplog = caplog

    def setup(self):
        sys.argv = [
            sys.argv[0], 'result', 'bundle', '--target-dir', 'target_dir',
            '--bundle-dir', 'bundle_dir', '--id', 'Build_42'
        ]
        self.abs_target_dir = os.path.abspath('target_dir')
        self.abs_bundle_dir = os.path.abspath('bundle_dir')

        self.xml_state = Mock()
        self.xml_state.get_image_version = Mock(return_value='1.2.3')
        self.xml_state.xml_data.get_name = Mock(return_value='test-image')

        self.result = Result(self.xml_state)
        self.result.add(key='keyname',
                        filename='test-image-1.2.3',
                        use_for_bundle=True,
                        compress=True,
                        shasum=True)

        kiwi.tasks.result_bundle.Help = Mock(return_value=Mock())
        self.task = ResultBundleTask()

        runtime_config = Mock()
        runtime_config.is_bundle_compression_requested = Mock(
            return_value=True)
        self.task.runtime_config = runtime_config

    def teardown(self):
        sys.argv = argv_kiwi_tests

    def _init_command_args(self):
        self.task.command_args = {}
        self.task.command_args['help'] = False
        self.task.command_args['bundle'] = False
        self.task.command_args['--target-dir'] = 'target_dir'
        self.task.command_args['--bundle-dir'] = 'bundle_dir'
        self.task.command_args['--id'] = 'Build_42'
        self.task.command_args['--zsync-source'] = None

    def test_process_invalid_bundle_directory(self):
        self._init_command_args()
        self.task.command_args['--bundle-dir'] = \
            self.task.command_args['--target-dir']
        self.task.command_args['bundle'] = True
        with raises(KiwiBundleError):
            self.task.process()

    @patch('kiwi.tasks.result_bundle.Result.load')
    @patch('kiwi.tasks.result_bundle.Command.run')
    @patch('kiwi.tasks.result_bundle.Path.create')
    @patch('kiwi.tasks.result_bundle.Path.which')
    @patch('kiwi.tasks.result_bundle.Compress')
    @patch('kiwi.tasks.result_bundle.Checksum')
    @patch('os.path.exists')
    def test_process_result_bundle(self, mock_exists, mock_checksum,
                                   mock_compress, mock_path_which,
                                   mock_path_create, mock_command, mock_load):
        # This file won't be copied with build id
        self.result.add(key='noversion',
                        filename='test-image-noversion',
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)

        checksum = Mock()
        compress = Mock()
        mock_path_which.return_value = 'zsyncmake'
        compress.compressed_filename = 'compressed_filename'
        mock_compress.return_value = compress
        mock_checksum.return_value = checksum
        mock_exists.return_value = False
        mock_load.return_value = self.result
        self._init_command_args()
        self.task.command_args['bundle'] = True
        self.task.command_args['--zsync-source'] = 'http://example.com/zsync'

        m_open = mock_open()
        with patch('builtins.open', m_open, create=True):
            self.task.process()

        mock_load.assert_called_once_with(
            os.sep.join([self.abs_target_dir, 'kiwi.result']))
        mock_path_create.assert_called_once_with(self.abs_bundle_dir)
        assert mock_command.call_args_list == [
            call([
                'cp', 'test-image-1.2.3',
                os.sep.join([self.abs_bundle_dir, 'test-image-1.2.3-Build_42'])
            ]),
            call([
                'zsyncmake', '-e', '-u',
                'http://example.com/zsync/compressed_filename', '-o',
                'compressed_filename.zsync', 'compressed_filename'
            ]),
            call([
                'cp', 'test-image-noversion',
                os.sep.join([self.abs_bundle_dir, 'test-image-noversion'])
            ])
        ]
        mock_compress.assert_called_once_with(
            os.sep.join([self.abs_bundle_dir, 'test-image-1.2.3-Build_42']))
        mock_checksum.assert_called_once_with(compress.compressed_filename)
        checksum.sha256.assert_called_once_with()
        m_open.return_value.write.assert_called_once_with(
            '{0}  compressed_filename'.format(checksum.sha256.return_value))

    @patch('kiwi.tasks.result_bundle.Result.load')
    @patch('kiwi.tasks.result_bundle.Command.run')
    @patch('kiwi.tasks.result_bundle.Path.create')
    @patch('os.path.exists')
    def test_process_result_bundle_name_includes_version(
            self, mock_exists, mock_path_create, mock_command, mock_load):
        result = Result(self.xml_state)
        result.add(key='nameincludesversion',
                   filename='test-1.2.3-image-1.2.3',
                   use_for_bundle=True,
                   compress=False,
                   shasum=False)

        self.xml_state.xml_data.get_name = Mock(
            return_value='test-1.2.3-image')

        mock_exists.return_value = False
        mock_load.return_value = result
        self._init_command_args()

        self.task.process()

        mock_load.assert_called_once_with(
            os.sep.join([self.abs_target_dir, 'kiwi.result']))
        mock_path_create.assert_called_once_with(self.abs_bundle_dir)
        assert mock_command.call_args_list == [
            call([
                'cp', 'test-1.2.3-image-1.2.3',
                os.sep.join(
                    [self.abs_bundle_dir, 'test-1.2.3-image-1.2.3-Build_42'])
            ])
        ]

    @patch('kiwi.tasks.result_bundle.Result.load')
    @patch('kiwi.tasks.result_bundle.Command.run')
    @patch('kiwi.tasks.result_bundle.Path.create')
    @patch('kiwi.tasks.result_bundle.Path.which')
    @patch('kiwi.tasks.result_bundle.Compress')
    @patch('kiwi.tasks.result_bundle.Checksum')
    @patch('os.path.exists')
    def test_process_result_bundle_zsyncmake_missing(
            self, mock_exists, mock_checksum, mock_compress, mock_path_which,
            mock_path_create, mock_command, mock_load):
        checksum = Mock()
        compress = Mock()
        mock_path_which.return_value = None
        compress.compressed_filename = 'compressed_filename'
        mock_compress.return_value = compress
        mock_checksum.return_value = checksum
        mock_exists.return_value = False
        mock_load.return_value = self.result
        self._init_command_args()
        self.task.command_args['bundle'] = True
        self.task.command_args['--zsync-source'] = 'http://example.com/zsync'

        with patch('builtins.open'):
            with self._caplog.at_level(logging.WARNING):
                self.task.process()
                assert '--> zsyncmake missing, zsync setup skipped' in \
                    self._caplog.text

    def test_process_result_bundle_help(self):
        self._init_command_args()
        self.task.command_args['help'] = True
        self.task.command_args['bundle'] = True
        self.task.process()
        self.task.manual.show.assert_called_once_with('kiwi::result::bundle')
Esempio n. 28
0
class DiskBuilder(object):
    """
    **Disk image builder**

    :param object xml_state: Instance of :class:`XMLState`
    :param str target_dir: Target directory path name
    :param str root_dir: Root directory path name
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * signing_keys: list of package signing keys
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.arch = platform.machine()
        if self.arch == 'i686' or self.arch == 'i586':
            self.arch = 'ix86'
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.spare_part_mbsize = xml_state.get_build_type_spare_part_size()
        self.persistency_type = xml_state.build_type.get_devicepersistency()
        self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot()
        self.custom_root_mount_args = xml_state.get_fs_mount_option_list()
        self.build_type_name = xml_state.get_build_type_name()
        self.image_format = xml_state.build_type.get_format()
        self.install_iso = xml_state.build_type.get_installiso()
        self.install_stick = xml_state.build_type.get_installstick()
        self.install_pxe = xml_state.build_type.get_installpxe()
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.volume_manager_name = xml_state.get_volume_management()
        self.volumes = xml_state.get_volumes()
        self.volume_group_name = xml_state.get_volume_group_name()
        self.mdraid = xml_state.build_type.get_mdraid()
        self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr()
        self.force_mbr = xml_state.build_type.get_force_mbr()
        self.luks = xml_state.build_type.get_luks()
        self.luks_os = xml_state.build_type.get_luksOS()
        self.xen_server = xml_state.is_xen_server()
        self.requested_filesystem = xml_state.build_type.get_filesystem()
        self.requested_boot_filesystem = \
            xml_state.build_type.get_bootfilesystem()
        self.bootloader = xml_state.build_type.get_bootloader()
        self.initrd_system = xml_state.get_initrd_system()
        self.target_removable = xml_state.build_type.get_target_removable()
        self.root_filesystem_is_multipath = \
            xml_state.get_oemconfig_oem_multipath_scan()
        self.disk_setup = DiskSetup(
            xml_state, root_dir
        )
        self.unpartitioned_bytes = \
            xml_state.get_build_type_unpartitioned_bytes()
        self.custom_args = custom_args

        self.signing_keys = None
        if custom_args and 'signing_keys' in custom_args:
            self.signing_keys = custom_args['signing_keys']

        self.boot_image = BootImage(
            xml_state, target_dir, root_dir,
            signing_keys=self.signing_keys, custom_args=self.custom_args
        )
        self.firmware = FirmWare(
            xml_state
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.diskname = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + self.arch,
                '-' + xml_state.get_image_version(),
                '.raw'
            ]
        )
        self.install_media = self._install_image_requested()
        self.generic_fstab_entries = []

        # an instance of a class with the sync_data capability
        # representing the entire image system except for the boot/ area
        # which could live on another part of the disk
        self.system = None

        # an instance of a class with the sync_data capability
        # representing the boot/ area of the disk if not part of
        # self.system
        self.system_boot = None

        # an instance of a class with the sync_data capability
        # representing the boot/efi area of the disk
        self.system_efi = None

        # result store
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a bootable disk image and optional installation image
        The installation image is a bootable hybrid ISO image which
        embeds the disk image and an image installer

        Image types which triggers this builder are:

        * image="oem"
        * image="vmx"

        :return: result

        :rtype: instance of :class:`Result`
        """
        disk = DiskBuilder(
            self.xml_state, self.target_dir, self.root_dir, self.custom_args
        )
        result = disk.create_disk()

        # cleanup disk resources taken prior to next steps
        del disk

        disk_installer = DiskBuilder(
            self.xml_state, self.target_dir, self.root_dir
        )
        result = disk_installer.create_install_media(result)

        disk_format = DiskBuilder(
            self.xml_state, self.target_dir, self.root_dir
        )

        disk_format.append_unpartitioned_space()
        result = disk_format.create_disk_format(result)

        return result

    def create_disk(self):
        """
        Build a bootable raw disk image

        :raises KiwiInstallMediaError:
            if install media is required and image type is not oem
        :raises KiwiVolumeManagerSetupError:
            root overlay at the same time volumes are defined is not supported

        :return: result

        :rtype: instance of :class:`Result`
        """
        if self.install_media and self.build_type_name != 'oem':
            raise KiwiInstallMediaError(
                'Install media requires oem type setup, got %s' %
                self.build_type_name
            )

        if self.root_filesystem_is_overlay and self.volume_manager_name:
            raise KiwiVolumeManagerSetupError(
                'Volume management together with root overlay is not supported'
            )

        # setup recovery archive, cleanup and create archive if requested
        self.system_setup.create_recovery_archive()

        # prepare boot(initrd) root system
        log.info('Preparing boot system')
        self.boot_image.prepare()

        # precalculate needed disk size
        disksize_mbytes = self.disk_setup.get_disksize_mbytes()

        # create the disk
        log.info('Creating raw disk image %s', self.diskname)
        loop_provider = LoopDevice(
            self.diskname, disksize_mbytes, self.blocksize
        )
        loop_provider.create()

        self.disk = Disk(
            self.firmware.get_partition_table_type(), loop_provider,
            self.xml_state.get_disk_start_sector()
        )

        # create the bootloader instance
        self.bootloader_config = BootLoaderConfig(
            self.bootloader, self.xml_state, self.root_dir, {
                'targetbase':
                    loop_provider.get_device(),
                'grub_directory_name':
                    Defaults.get_grub_boot_directory_name(self.root_dir)
            }
        )

        # create disk partitions and instance device map
        device_map = self._build_and_map_disk_partitions()

        # create raid on current root device if requested
        if self.mdraid:
            self.raid_root = RaidDevice(device_map['root'])
            self.raid_root.create_degraded_raid(raid_level=self.mdraid)
            device_map['root'] = self.raid_root.get_device()
            self.disk.public_partition_id_map['kiwi_RaidPart'] = \
                self.disk.public_partition_id_map['kiwi_RootPart']
            self.disk.public_partition_id_map['kiwi_RaidDev'] = \
                device_map['root'].get_device()

        # create luks on current root device if requested
        if self.luks:
            self.luks_root = LuksDevice(device_map['root'])
            self.luks_root.create_crypto_luks(
                passphrase=self.luks, os=self.luks_os
            )
            device_map['root'] = self.luks_root.get_device()

        # create filesystems on boot partition(s) if any
        self._build_boot_filesystems(device_map)

        # create volumes and filesystems for root system
        if self.volume_manager_name:
            volume_manager_custom_parameters = {
                'fs_mount_options':
                    self.custom_root_mount_args,
                'root_label':
                    self.disk_setup.get_root_label(),
                'root_is_snapshot':
                    self.xml_state.build_type.get_btrfs_root_is_snapshot(),
                'root_is_readonly_snapshot':
                    self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot(),
                'image_type':
                    self.xml_state.get_build_type_name()
            }
            volume_manager = VolumeManager(
                self.volume_manager_name, device_map['root'],
                self.root_dir + '/',
                self.volumes,
                volume_manager_custom_parameters
            )
            volume_manager.setup(
                self.volume_group_name
            )
            volume_manager.create_volumes(
                self.requested_filesystem
            )
            volume_manager.mount_volumes()
            self.generic_fstab_entries += volume_manager.get_fstab(
                self.persistency_type, self.requested_filesystem
            )
            self.system = volume_manager
            device_map['root'] = volume_manager.get_device()['root']
        else:
            log.info(
                'Creating root(%s) filesystem on %s',
                self.requested_filesystem, device_map['root'].get_device()
            )
            filesystem_custom_parameters = {
                'mount_options': self.custom_root_mount_args
            }
            filesystem = FileSystem(
                self.requested_filesystem, device_map['root'],
                self.root_dir + '/',
                filesystem_custom_parameters
            )
            filesystem.create_on_device(
                label=self.disk_setup.get_root_label()
            )
            self.system = filesystem

        # create a random image identifier
        self.mbrid = SystemIdentifier()
        self.mbrid.calculate_id()

        # create first stage metadata to boot image
        self._write_partition_id_config_to_boot_image()

        self._write_recovery_metadata_to_boot_image()

        self._write_raid_config_to_boot_image()

        self._write_generic_fstab_to_boot_image(device_map)

        self.system_setup.export_modprobe_setup(
            self.boot_image.boot_root_directory
        )

        # create first stage metadata to system image
        self._write_image_identifier_to_system_image()

        self._write_crypttab_to_system_image()

        self._write_generic_fstab_to_system_image(device_map)

        if self.initrd_system == 'dracut':
            self._create_dracut_config()

        # create initrd cpio archive
        self.boot_image.create_initrd(self.mbrid)

        # create dracut config omitting one time kiwi dracut modules
        if self.initrd_system == 'dracut':
            self._create_system_dracut_config()

        # create second stage metadata to system image
        self._copy_first_boot_files_to_system_image()

        self._write_bootloader_config_to_system_image(device_map)

        self.mbrid.write_to_disk(
            self.disk.storage_provider
        )

        # set SELinux file security contexts if context exists
        self._setup_selinux_file_contexts()

        # syncing system data to disk image
        log.info('Syncing system to image')
        if self.system_efi:
            log.info('--> Syncing EFI boot data to EFI partition')
            self.system_efi.sync_data()

        if self.system_boot:
            log.info('--> Syncing boot data at extra partition')
            self.system_boot.sync_data(
                self._get_exclude_list_for_boot_data_sync()
            )

        log.info('--> Syncing root filesystem data')
        if self.root_filesystem_is_overlay:
            squashed_root_file = NamedTemporaryFile()
            squashed_root = FileSystemSquashFs(
                device_provider=None, root_dir=self.root_dir
            )
            squashed_root.create_on_file(
                filename=squashed_root_file.name,
                exclude=self._get_exclude_list_for_root_data_sync(device_map)
            )
            Command.run(
                [
                    'dd',
                    'if=%s' % squashed_root_file.name,
                    'of=%s' % device_map['readonly'].get_device()
                ]
            )
        else:
            self.system.sync_data(
                self._get_exclude_list_for_root_data_sync(device_map)
            )

        # install boot loader
        self._install_bootloader(device_map)

        # set root filesystem properties
        self._setup_property_root_is_readonly_snapshot()

        # prepare for install media if requested
        if self.install_media:
            log.info('Saving boot image instance to file')
            self.boot_image.dump(
                self.target_dir + '/boot_image.pickledump'
            )

        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.diskname
        )
        # store image file name in result
        self.result.add(
            key='disk_image',
            filename=self.diskname,
            use_for_bundle=True if not self.image_format else False,
            compress=True,
            shasum=True
        )

        # create image root metadata
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )

        return self.result

    def create_disk_format(self, result_instance):
        """
        Create a bootable disk format from a previously
        created raw disk image

        :param object result_instance: instance of :class:`Result`

        :return: updated result_instance

        :rtype: instance of :class:`Result`
        """
        if self.image_format:
            log.info('Creating %s Disk Format', self.image_format)
            disk_format = DiskFormat(
                self.image_format, self.xml_state,
                self.root_dir, self.target_dir
            )
            disk_format.create_image_format()
            disk_format.store_to_result(result_instance)

        return result_instance

    def append_unpartitioned_space(self):
        """
        Extends the raw disk if an unpartitioned area is specified
        """
        if self.unpartitioned_bytes:
            log.info(
                'Expanding disk with %d bytes of unpartitioned space',
                self.unpartitioned_bytes
            )
            disk_format = DiskFormat(
                'raw', self.xml_state, self.root_dir, self.target_dir
            )
            disk_format.resize_raw_disk(self.unpartitioned_bytes, append=True)
            firmware = FirmWare(self.xml_state)
            loop_provider = LoopDevice(disk_format.diskname)
            loop_provider.create(overwrite=False)
            partitioner = Partitioner(
                firmware.get_partition_table_type(), loop_provider
            )
            partitioner.resize_table()

    def create_install_media(self, result_instance):
        """
        Build an installation image. The installation image is a
        bootable hybrid ISO image which embeds the raw disk image
        and an image installer

        :param object result_instance: instance of :class:`Result`

        :return: updated result_instance with installation media

        :rtype: instance of :class:`Result`
        """
        if self.install_media:
            install_image = InstallImageBuilder(
                self.xml_state, self.root_dir, self.target_dir,
                self._load_boot_image_instance(), self.custom_args
            )

            if self.install_iso or self.install_stick:
                log.info('Creating hybrid ISO installation image')
                install_image.create_install_iso()
                result_instance.add(
                    key='installation_image',
                    filename=install_image.isoname,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )

            if self.install_pxe:
                log.info('Creating PXE installation archive')
                install_image.create_install_pxe_archive()
                result_instance.add(
                    key='installation_pxe_archive',
                    filename=install_image.pxename,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )

        return result_instance

    def _load_boot_image_instance(self):
        boot_image_dump_file = self.target_dir + '/boot_image.pickledump'
        if not os.path.exists(boot_image_dump_file):
            raise KiwiInstallMediaError(
                'No boot image instance dump %s found' % boot_image_dump_file
            )
        try:
            with open(boot_image_dump_file, 'rb') as boot_image_dump:
                boot_image = pickle.load(boot_image_dump)
            boot_image.enable_cleanup()
            Path.wipe(boot_image_dump_file)
        except Exception as e:
            raise KiwiInstallMediaError(
                'Failed to load boot image dump: %s' % type(e).__name__
            )
        return boot_image

    def _setup_selinux_file_contexts(self):
        security_context = '/etc/selinux/targeted/contexts/files/file_contexts'
        if os.path.exists(self.root_dir + security_context):
            self.system_setup.set_selinux_file_contexts(
                security_context
            )

    def _install_image_requested(self):
        if self.install_iso or self.install_stick or self.install_pxe:
            return True

    def _get_exclude_list_for_root_data_sync(self, device_map):
        exclude_list = Defaults.get_exclude_list_for_root_data_sync()
        if 'boot' in device_map and self.bootloader == 'grub2_s390x_emu':
            exclude_list.append('boot/zipl/*')
            exclude_list.append('boot/zipl/.*')
        elif 'boot' in device_map:
            exclude_list.append('boot/*')
            exclude_list.append('boot/.*')
        if 'efi' in device_map:
            exclude_list.append('boot/efi/*')
            exclude_list.append('boot/efi/.*')
        return exclude_list

    def _get_exclude_list_for_boot_data_sync(self):
        return ['efi/*']

    def _build_boot_filesystems(self, device_map):
        if 'efi' in device_map:
            log.info(
                'Creating EFI(fat16) filesystem on %s',
                device_map['efi'].get_device()
            )
            filesystem = FileSystem(
                'fat16', device_map['efi'], self.root_dir + '/boot/efi/'
            )
            filesystem.create_on_device(
                label=self.disk_setup.get_efi_label()
            )
            self.system_efi = filesystem

        if 'boot' in device_map:
            boot_filesystem = self.requested_boot_filesystem
            if not boot_filesystem:
                boot_filesystem = self.requested_filesystem
            boot_directory = self.root_dir + '/boot/'
            if self.bootloader == 'grub2_s390x_emu':
                boot_directory = self.root_dir + '/boot/zipl/'
                boot_filesystem = 'ext2'
            log.info(
                'Creating boot(%s) filesystem on %s',
                boot_filesystem, device_map['boot'].get_device()
            )
            filesystem = FileSystem(
                boot_filesystem, device_map['boot'], boot_directory
            )
            filesystem.create_on_device(
                label=self.disk_setup.get_boot_label()
            )
            self.system_boot = filesystem

    def _build_and_map_disk_partitions(self):               # noqa: C901
        self.disk.wipe()
        if self.firmware.legacy_bios_mode():
            log.info('--> creating EFI CSM(legacy bios) partition')
            self.disk.create_efi_csm_partition(
                self.firmware.get_legacy_bios_partition_size()
            )

        if self.firmware.efi_mode():
            log.info('--> creating EFI partition')
            self.disk.create_efi_partition(
                self.firmware.get_efi_partition_size()
            )

        if self.firmware.ofw_mode():
            log.info('--> creating PReP partition')
            self.disk.create_prep_partition(
                self.firmware.get_prep_partition_size()
            )

        if self.disk_setup.need_boot_partition():
            log.info('--> creating boot partition')
            self.disk.create_boot_partition(
                self.disk_setup.boot_partition_size()
            )

        if self.spare_part_mbsize:
            log.info('--> creating spare partition')
            self.disk.create_spare_partition(
                self.spare_part_mbsize
            )

        if self.root_filesystem_is_overlay:
            log.info('--> creating readonly root partition')
            squashed_root_file = NamedTemporaryFile()
            squashed_root = FileSystemSquashFs(
                device_provider=None, root_dir=self.root_dir
            )
            squashed_root.create_on_file(
                filename=squashed_root_file.name,
                exclude=[Defaults.get_shared_cache_location()]
            )
            squashed_rootfs_mbsize = os.path.getsize(
                squashed_root_file.name
            ) / 1048576
            self.disk.create_root_readonly_partition(
                int(squashed_rootfs_mbsize + 50)
            )

        if self.volume_manager_name and self.volume_manager_name == 'lvm':
            log.info('--> creating LVM root partition')
            self.disk.create_root_lvm_partition('all_free')

        elif self.mdraid:
            log.info('--> creating mdraid root partition')
            self.disk.create_root_raid_partition('all_free')

        else:
            log.info('--> creating root partition')
            self.disk.create_root_partition('all_free')

        if self.firmware.bios_mode():
            log.info('--> setting active flag to primary boot partition')
            self.disk.activate_boot_partition()

        if self.firmware.ofw_mode():
            log.info('--> setting active flag to primary PReP partition')
            self.disk.activate_boot_partition()

        if self.firmware.efi_mode():
            if self.force_mbr:
                log.info('--> converting partition table to MBR')
                self.disk.create_mbr()
            elif self.hybrid_mbr:
                log.info('--> converting partition table to hybrid GPT/MBR')
                self.disk.create_hybrid_mbr()

        self.disk.map_partitions()

        return self.disk.get_device()

    def _create_dracut_config(self):
        dracut_config = [
            'hostonly="no"',
            'dracut_rescue_image="no"'
        ]
        dracut_modules = []
        dracut_modules_omit = ['kiwi-live', 'kiwi-dump']
        if self.root_filesystem_is_multipath is False:
            dracut_modules_omit.append('multipath')
        if self.root_filesystem_is_overlay:
            dracut_modules.append('kiwi-overlay')
        else:
            dracut_modules_omit.append('kiwi-overlay')
        if self.build_type_name == 'oem':
            dracut_modules.append('kiwi-lib')
            dracut_modules.append('kiwi-repart')
        self._write_dracut_config(
            config=dracut_config,
            modules=dracut_modules,
            omit_modules=dracut_modules_omit
        )

    def _create_system_dracut_config(self):
        dracut_modules = []
        dracut_modules_omit = ['kiwi-live', 'kiwi-dump', 'kiwi-repart']
        if self.root_filesystem_is_overlay:
            dracut_modules.append('kiwi-overlay')
        else:
            dracut_modules_omit.append('kiwi-overlay')
        self._write_dracut_config(
            config=[], modules=dracut_modules, omit_modules=dracut_modules_omit
        )

    def _write_dracut_config(self, config, modules, omit_modules):
        dracut_config_file = ''.join(
            [self.root_dir, Defaults.get_dracut_conf_name()]
        )
        if modules:
            config.append(
                'add_dracutmodules+=" {0} "'.format(' '.join(modules))
            )
        if omit_modules:
            config.append(
                'omit_dracutmodules+=" {0} "'.format(' '.join(omit_modules))
            )
        with open(dracut_config_file, 'w') as dracut_config:
            for entry in config:
                dracut_config.write(entry + os.linesep)

    def _write_partition_id_config_to_boot_image(self):
        log.info('Creating config.partids in boot system')
        filename = ''.join(
            [self.boot_image.boot_root_directory, '/config.partids']
        )
        partition_id_map = self.disk.get_public_partition_id_map()
        with open(filename, 'w') as partids:
            for id_name, id_value in list(partition_id_map.items()):
                partids.write('{0}="{1}"{2}'.format(
                    id_name, id_value, os.linesep)
                )
        self.boot_image.include_file(
            os.sep + os.path.basename(filename)
        )

    def _write_raid_config_to_boot_image(self):
        if self.mdraid:
            log.info('Creating etc/mdadm.conf in boot system')
            filename = ''.join(
                [self.boot_image.boot_root_directory, '/etc/mdadm.conf']
            )
            self.raid_root.create_raid_config(filename)
            self.boot_image.include_file(
                os.sep + os.sep.join(['etc', os.path.basename(filename)])
            )

    def _write_crypttab_to_system_image(self):
        if self.luks:
            log.info('Creating etc/crypttab')
            filename = ''.join(
                [self.root_dir, '/etc/crypttab']
            )
            self.luks_root.create_crypttab(filename)
            self.boot_image.include_file(
                os.sep + os.sep.join(['etc', os.path.basename(filename)])
            )

    def _write_generic_fstab_to_system_image(self, device_map):
        log.info('Creating generic system etc/fstab')
        self._write_generic_fstab(device_map, self.system_setup)

    def _write_generic_fstab_to_boot_image(self, device_map):
        if self.initrd_system == 'kiwi':
            log.info('Creating generic boot image etc/fstab')
            self._write_generic_fstab(device_map, self.boot_image.setup)

    def _write_generic_fstab(self, device_map, setup):
        root_is_snapshot = \
            self.xml_state.build_type.get_btrfs_root_is_snapshot()
        root_is_readonly_snapshot = \
            self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()

        fs_check_interval = '1 1'
        custom_root_mount_args = list(self.custom_root_mount_args)
        if root_is_snapshot and root_is_readonly_snapshot:
            custom_root_mount_args += ['ro']
            fs_check_interval = '0 0'

        self._add_generic_fstab_entry(
            device_map['root'].get_device(), '/',
            custom_root_mount_args, fs_check_interval
        )
        if 'boot' in device_map:
            if self.bootloader == 'grub2_s390x_emu':
                boot_mount_point = '/boot/zipl'
            else:
                boot_mount_point = '/boot'
            self._add_generic_fstab_entry(
                device_map['boot'].get_device(), boot_mount_point
            )
        if 'efi' in device_map:
            self._add_generic_fstab_entry(
                device_map['efi'].get_device(), '/boot/efi'
            )
        setup.create_fstab(
            self.generic_fstab_entries
        )

    def _add_generic_fstab_entry(
        self, device, mount_point, options=None, check='0 0'
    ):
        if not options:
            options = ['defaults']
        block_operation = BlockID(device)
        blkid_type = 'LABEL' if self.persistency_type == 'by-label' else 'UUID'
        device_id = block_operation.get_blkid(blkid_type)
        fstab_entry = ' '.join(
            [
                blkid_type + '=' + device_id, mount_point,
                block_operation.get_filesystem(), ','.join(options), check
            ]
        )
        if fstab_entry not in self.generic_fstab_entries:
            self.generic_fstab_entries.append(
                fstab_entry
            )

    def _write_image_identifier_to_system_image(self):
        log.info('Creating image identifier: %s', self.mbrid.get_id())
        self.mbrid.write(
            self.root_dir + '/boot/mbrid'
        )

    def _write_recovery_metadata_to_boot_image(self):
        if os.path.exists(self.root_dir + '/recovery.partition.size'):
            log.info('Copying recovery metadata to boot image')
            recovery_metadata = ''.join(
                [self.root_dir, '/recovery.partition.size']
            )
            Command.run(
                ['cp', recovery_metadata, self.boot_image.boot_root_directory]
            )
            self.boot_image.include_file(
                os.sep + os.path.basename(recovery_metadata)
            )

    def _write_bootloader_config_to_system_image(self, device_map):
        if self.bootloader is not 'custom':
            log.info('Creating %s bootloader configuration', self.bootloader)
            boot_options = []
            if self.mdraid:
                boot_options.append('rd.auto')
            boot_names = self.boot_image.get_boot_names()
            boot_device = device_map['root']
            if 'boot' in device_map:
                boot_device = device_map['boot']

            root_uuid = self.disk.get_uuid(
                device_map['root'].get_device()
            )
            boot_uuid = self.disk.get_uuid(
                boot_device.get_device()
            )
            self.bootloader_config.setup_disk_boot_images(boot_uuid)
            self.bootloader_config.setup_disk_image_config(
                boot_uuid=boot_uuid,
                root_uuid=root_uuid,
                kernel=boot_names.kernel_name,
                initrd=boot_names.initrd_name,
                boot_options=' '.join(boot_options)
            )
            self.bootloader_config.write()

            log.info('Creating config.bootoptions')
            filename = ''.join(
                [self.boot_image.boot_root_directory, '/config.bootoptions']
            )
            kexec_boot_options = ' '.join(
                [
                    self.bootloader_config.get_boot_cmdline(root_uuid)
                ] + boot_options
            )
            with open(filename, 'w') as boot_options:
                boot_options.write(
                    '{0}{1}'.format(kexec_boot_options, os.linesep)
                )

        partition_id_map = self.disk.get_public_partition_id_map()
        boot_partition_id = partition_id_map['kiwi_RootPart']
        if 'kiwi_BootPart' in partition_id_map:
            boot_partition_id = partition_id_map['kiwi_BootPart']

        self.system_setup.call_edit_boot_config_script(
            self.requested_filesystem, boot_partition_id
        )

    def _install_bootloader(self, device_map):
        root_device = device_map['root']
        boot_device = root_device
        if 'boot' in device_map:
            boot_device = device_map['boot']

        if 'readonly' in device_map:
            root_device = device_map['readonly']

        custom_install_arguments = {
            'boot_device': boot_device.get_device(),
            'root_device': root_device.get_device(),
            'firmware': self.firmware,
            'target_removable': self.target_removable
        }

        if 'efi' in device_map:
            efi_device = device_map['efi']
            custom_install_arguments.update(
                {'efi_device': efi_device.get_device()}
            )

        if 'prep' in device_map:
            prep_device = device_map['prep']
            custom_install_arguments.update(
                {'prep_device': prep_device.get_device()}
            )

        if self.volume_manager_name:
            self.system.umount_volumes()
            custom_install_arguments.update(
                {'system_volumes': self.system.get_volumes()}
            )

        if self.bootloader is not 'custom':
            log.debug(
                "custom arguments for bootloader installation %s",
                custom_install_arguments
            )
            bootloader = BootLoaderInstall(
                self.bootloader, self.root_dir, self.disk.storage_provider,
                custom_install_arguments
            )
            if bootloader.install_required():
                bootloader.install()

        self.system_setup.call_edit_boot_install_script(
            self.diskname, boot_device.get_device()
        )

    def _setup_property_root_is_readonly_snapshot(self):
        if self.volume_manager_name:
            root_is_snapshot = \
                self.xml_state.build_type.get_btrfs_root_is_snapshot()
            root_is_readonly_snapshot = \
                self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot()
            if root_is_snapshot and root_is_readonly_snapshot:
                log.info(
                    'Setting root filesystem into read-only mode'
                )
                self.system.mount_volumes()
                self.system.set_property_readonly_root()
                self.system.umount_volumes()

    def _copy_first_boot_files_to_system_image(self):
        boot_names = self.boot_image.get_boot_names()
        if self.initrd_system == 'kiwi':
            log.info('Copy boot files to system image')
            kernel = Kernel(self.boot_image.boot_root_directory)

            log.info('--> boot image kernel as %s', boot_names.kernel_name)
            kernel.copy_kernel(
                self.root_dir, ''.join(['/boot/', boot_names.kernel_name])
            )

            if self.xen_server:
                if kernel.get_xen_hypervisor():
                    log.info('--> boot image Xen hypervisor as xen.gz')
                    kernel.copy_xen_hypervisor(
                        self.root_dir, '/boot/xen.gz'
                    )
                else:
                    raise KiwiDiskBootImageError(
                        'No hypervisor in boot image tree %s found' %
                        self.boot_image.boot_root_directory
                    )

        log.info('--> initrd archive as %s', boot_names.initrd_name)
        Command.run(
            [
                'mv', self.boot_image.initrd_filename,
                self.root_dir + ''.join(['/boot/', boot_names.initrd_name])
            ]
        )
Esempio n. 29
0
File: kis.py Progetto: jfkw/kiwi
class KisBuilder:
    """
    **Filesystem based image builder.**

    :param object xml_state: instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: system image root directory
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * signing_keys: list of package signing keys
        * xz_options: string of XZ compression parameters
    """
    def __init__(
        self, xml_state: XMLState, target_dir: str,
        root_dir: str, custom_args: Dict = None
    ):
        self.target_dir = target_dir
        self.compressed = xml_state.build_type.get_compressed()
        self.xen_server = xml_state.is_xen_server()
        self.custom_cmdline = xml_state.build_type.get_kernelcmdline()
        self.filesystem = FileSystemBuilder(
            xml_state, target_dir, root_dir + '/'
        ) if xml_state.build_type.get_filesystem() else None
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=root_dir
        )
        self.initrd_system = xml_state.get_initrd_system()

        self.boot_signing_keys = custom_args['signing_keys'] if custom_args \
            and 'signing_keys' in custom_args else None

        self.xz_options = custom_args['xz_options'] if custom_args \
            and 'xz_options' in custom_args else None

        self.boot_image_task = BootImage.new(
            xml_state, target_dir, root_dir,
            signing_keys=self.boot_signing_keys
        )
        self.image_name = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + Defaults.get_platform_name(),
                '-' + xml_state.get_image_version()
            ]
        )
        self.image: str = ''
        self.append_file = ''.join([self.image_name, '.append'])
        self.archive_name = ''.join([self.image_name, '.tar'])
        self.checksum_name = ''.join([self.image_name, '.md5'])
        self.kernel_filename: str = ''
        self.hypervisor_filename: str = ''
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self) -> Result:
        """
        Build a component image consisting out of a boot image(initrd)
        plus its appropriate kernel files and the root filesystem
        image with a checksum.

        Image types which triggers this builder are:

        * image="kis"
        * image="pxe"

        :raises KiwiKisBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        if self.filesystem:
            log.info('Creating root filesystem image')
            self.filesystem.create()
            os.rename(
                self.filesystem.filename, self.image_name
            )
            self.image = self.image_name
            if self.compressed:
                log.info('xz compressing root filesystem image')
                compress = Compress(self.image)
                compress.xz(self.xz_options)
                self.image = compress.compressed_filename

            log.info('Creating root filesystem MD5 checksum')
            checksum = Checksum(self.image)
            checksum.md5(self.checksum_name)

        # prepare boot(initrd) root system
        log.info('Creating boot image')
        self.boot_image_task.prepare()

        # export modprobe configuration to boot image
        self.system_setup.export_modprobe_setup(
            self.boot_image_task.boot_root_directory
        )

        # extract kernel from boot(initrd) root system
        kernel = Kernel(self.boot_image_task.boot_root_directory)
        kernel_data = kernel.get_kernel()
        if kernel_data:
            self.kernel_filename = ''.join(
                [
                    os.path.basename(self.image_name), '-',
                    kernel_data.version, '.kernel'
                ]
            )
            kernel.copy_kernel(
                self.target_dir, self.kernel_filename
            )
        else:
            raise KiwiKisBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image_task.boot_root_directory
            )

        # extract hypervisor from boot(initrd) root system
        if self.xen_server:
            hypervisor_data = kernel.get_xen_hypervisor()
            if hypervisor_data:
                self.hypervisor_filename = ''.join(
                    [
                        os.path.basename(self.image_name),
                        '-', hypervisor_data.name
                    ]
                )
                kernel.copy_xen_hypervisor(
                    self.target_dir, self.hypervisor_filename
                )
                self.result.add(
                    key='xen_hypervisor',
                    filename=self.target_dir + '/' + self.hypervisor_filename,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )
            else:
                raise KiwiKisBootImageError(
                    'No hypervisor in boot image tree %s found' %
                    self.boot_image_task.boot_root_directory
                )

        # create initrd
        self.boot_image_task.create_initrd()

        # create append information
        # this information helps to configure the deployment infrastructure
        if self.filesystem and self.filesystem.root_uuid \
           and self.initrd_system == 'dracut':
            cmdline = 'root=UUID={}'.format(self.filesystem.root_uuid)
            if self.custom_cmdline:
                cmdline += ' {}'.format(self.custom_cmdline)
            with open(self.append_file, 'w') as append:
                append.write(cmdline)

        # put results into a tarball
        if not self.xz_options:
            self.xz_options = Defaults.get_xz_compression_options()

        kis_tarball_files = [
            self.kernel_filename,
            os.path.basename(self.boot_image_task.initrd_filename),
            os.path.basename(self.checksum_name),
        ]

        if self.image:
            kis_tarball_files.append(os.path.basename(self.image))

        if self.filesystem and self.filesystem.root_uuid \
           and self.initrd_system == 'dracut':
            kis_tarball_files.append(os.path.basename(self.append_file))

        kis_tarball = ArchiveTar(
            self.archive_name,
            create_from_file_list=True,
            file_list=kis_tarball_files
        )

        if self.compressed:
            self.archive_name = kis_tarball.create(self.target_dir)
        else:
            self.archive_name = kis_tarball.create_xz_compressed(
                self.target_dir, xz_options=self.xz_options
            )

        Result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.archive_name
        )
        # store results
        self.result.add(
            key='kis_archive',
            filename=self.archive_name,
            use_for_bundle=True,
            compress=self.runtime_config.get_bundle_compression(
                default=False
            ),
            shasum=True
        )

        # create image root metadata
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_changes',
            filename=self.system_setup.export_package_changes(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=True,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result
Esempio n. 30
0
    def process(self):
        """
        Create result bundle from the image build results in the
        specified target directory. Each result image will contain
        the specified bundle identifier as part of its filename.
        Uncompressed image files will also become xz compressed
        and a sha sum will be created from every result image
        """
        self.manual = Help()
        if self._help():
            return

        if self.command_args['--package-as-rpm']:
            Privileges.check_for_root_permissions()

        # load serialized result object from target directory
        result_directory = os.path.abspath(self.command_args['--target-dir'])
        bundle_directory = os.path.abspath(self.command_args['--bundle-dir'])
        if result_directory == bundle_directory:
            raise KiwiBundleError(
                'Bundle directory must be different from target directory')

        log.info('Bundle build results from %s', result_directory)
        result = Result.load(result_directory + '/kiwi.result')
        image_version = result.xml_state.get_image_version()
        image_name = result.xml_state.xml_data.get_name()
        image_description = result.xml_state.get_description_section()
        ordered_results = OrderedDict(sorted(result.get_results().items()))

        # hard link bundle files, compress and build checksum
        if self.command_args['--package-as-rpm']:
            Path.wipe(bundle_directory)
        if not os.path.exists(bundle_directory):
            Path.create(bundle_directory)

        bundle_file_format_name = ''
        if 'bundle_format' in ordered_results:
            bundle_format = ordered_results['bundle_format']
            tags = bundle_format['tags']
            bundle_file_format_name = bundle_format['pattern']
            # Insert image name
            bundle_file_format_name = bundle_file_format_name.replace(
                '%N', tags.N)
            # Insert Concatenated profile name (_)
            bundle_file_format_name = bundle_file_format_name.replace(
                '%P', tags.P)
            # Insert Architecture name
            bundle_file_format_name = bundle_file_format_name.replace(
                '%A', tags.A)
            # Insert Image build type name
            bundle_file_format_name = bundle_file_format_name.replace(
                '%T', tags.T)
            # Insert Image Major version number
            bundle_file_format_name = bundle_file_format_name.replace(
                '%M', format(tags.M))
            # Insert Image Minor version number
            bundle_file_format_name = bundle_file_format_name.replace(
                '%m', format(tags.m))
            # Insert Image Patch version number
            bundle_file_format_name = bundle_file_format_name.replace(
                '%p', format(tags.p))
            # Insert Bundle ID
            bundle_file_format_name = bundle_file_format_name.replace(
                '%I', self.command_args['--id'])
            del (ordered_results['bundle_format'])

        for result_file in list(ordered_results.values()):
            if result_file.use_for_bundle:
                extension = result_file.filename.split('.').pop()
                if bundle_file_format_name:
                    bundle_file_basename = '.'.join(
                        [bundle_file_format_name, extension])
                else:
                    bundle_file_basename = os.path.basename(
                        result_file.filename)
                    # The bundle id is only taken into account for image results
                    # which contains the image version appended in its file name
                    part_name = list(
                        bundle_file_basename.partition(image_name))
                    bundle_file_basename = ''.join([
                        part_name[0], part_name[1], part_name[2].replace(
                            image_version,
                            image_version + '-' + self.command_args['--id'])
                    ])
                log.info('Creating %s', bundle_file_basename)
                bundle_file = ''.join(
                    [bundle_directory, '/', bundle_file_basename])
                Command.run(['cp', result_file.filename, bundle_file])
                if result_file.compress:
                    log.info('--> XZ compressing')
                    compress = Compress(bundle_file)
                    compress.xz(self.runtime_config.get_xz_options())
                    bundle_file = compress.compressed_filename

                if self.command_args['--zsync-source'] and result_file.shasum:
                    # Files with a checksum are considered to be image files
                    # and are therefore eligible to be provided via the
                    # requested Partial/differential file download based on
                    # zsync
                    zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK)
                    if zsyncmake:
                        log.info('--> Creating zsync control file')
                        Command.run([
                            zsyncmake, '-e', '-u',
                            os.sep.join([
                                self.command_args['--zsync-source'],
                                os.path.basename(bundle_file)
                            ]), '-o', bundle_file + '.zsync', bundle_file
                        ])
                    else:
                        log.warning(
                            '--> zsyncmake missing, zsync setup skipped')

                if result_file.shasum:
                    log.info('--> Creating SHA 256 sum')
                    checksum = Checksum(bundle_file)
                    with open(bundle_file + '.sha256', 'w') as shasum:
                        shasum.write('{0}  {1}{2}'.format(
                            checksum.sha256(), os.path.basename(bundle_file),
                            os.linesep))
        if self.command_args['--package-as-rpm']:
            ResultBundleTask._build_rpm_package(
                bundle_directory, bundle_file_format_name or image_name,
                image_version, image_description.specification,
                list(glob.iglob(f'{bundle_directory}/*')))
Esempio n. 31
0
File: kis.py Progetto: jfkw/kiwi
    def create(self) -> Result:
        """
        Build a component image consisting out of a boot image(initrd)
        plus its appropriate kernel files and the root filesystem
        image with a checksum.

        Image types which triggers this builder are:

        * image="kis"
        * image="pxe"

        :raises KiwiKisBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        if self.filesystem:
            log.info('Creating root filesystem image')
            self.filesystem.create()
            os.rename(
                self.filesystem.filename, self.image_name
            )
            self.image = self.image_name
            if self.compressed:
                log.info('xz compressing root filesystem image')
                compress = Compress(self.image)
                compress.xz(self.xz_options)
                self.image = compress.compressed_filename

            log.info('Creating root filesystem MD5 checksum')
            checksum = Checksum(self.image)
            checksum.md5(self.checksum_name)

        # prepare boot(initrd) root system
        log.info('Creating boot image')
        self.boot_image_task.prepare()

        # export modprobe configuration to boot image
        self.system_setup.export_modprobe_setup(
            self.boot_image_task.boot_root_directory
        )

        # extract kernel from boot(initrd) root system
        kernel = Kernel(self.boot_image_task.boot_root_directory)
        kernel_data = kernel.get_kernel()
        if kernel_data:
            self.kernel_filename = ''.join(
                [
                    os.path.basename(self.image_name), '-',
                    kernel_data.version, '.kernel'
                ]
            )
            kernel.copy_kernel(
                self.target_dir, self.kernel_filename
            )
        else:
            raise KiwiKisBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image_task.boot_root_directory
            )

        # extract hypervisor from boot(initrd) root system
        if self.xen_server:
            hypervisor_data = kernel.get_xen_hypervisor()
            if hypervisor_data:
                self.hypervisor_filename = ''.join(
                    [
                        os.path.basename(self.image_name),
                        '-', hypervisor_data.name
                    ]
                )
                kernel.copy_xen_hypervisor(
                    self.target_dir, self.hypervisor_filename
                )
                self.result.add(
                    key='xen_hypervisor',
                    filename=self.target_dir + '/' + self.hypervisor_filename,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )
            else:
                raise KiwiKisBootImageError(
                    'No hypervisor in boot image tree %s found' %
                    self.boot_image_task.boot_root_directory
                )

        # create initrd
        self.boot_image_task.create_initrd()

        # create append information
        # this information helps to configure the deployment infrastructure
        if self.filesystem and self.filesystem.root_uuid \
           and self.initrd_system == 'dracut':
            cmdline = 'root=UUID={}'.format(self.filesystem.root_uuid)
            if self.custom_cmdline:
                cmdline += ' {}'.format(self.custom_cmdline)
            with open(self.append_file, 'w') as append:
                append.write(cmdline)

        # put results into a tarball
        if not self.xz_options:
            self.xz_options = Defaults.get_xz_compression_options()

        kis_tarball_files = [
            self.kernel_filename,
            os.path.basename(self.boot_image_task.initrd_filename),
            os.path.basename(self.checksum_name),
        ]

        if self.image:
            kis_tarball_files.append(os.path.basename(self.image))

        if self.filesystem and self.filesystem.root_uuid \
           and self.initrd_system == 'dracut':
            kis_tarball_files.append(os.path.basename(self.append_file))

        kis_tarball = ArchiveTar(
            self.archive_name,
            create_from_file_list=True,
            file_list=kis_tarball_files
        )

        if self.compressed:
            self.archive_name = kis_tarball.create(self.target_dir)
        else:
            self.archive_name = kis_tarball.create_xz_compressed(
                self.target_dir, xz_options=self.xz_options
            )

        Result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.archive_name
        )
        # store results
        self.result.add(
            key='kis_archive',
            filename=self.archive_name,
            use_for_bundle=True,
            compress=self.runtime_config.get_bundle_compression(
                default=False
            ),
            shasum=True
        )

        # create image root metadata
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_changes',
            filename=self.system_setup.export_package_changes(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=True,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result
Esempio n. 32
0
class ContainerBuilder(object):
    """
    Container image builder

    Attributes

    * :attr:`root_dir`
        root directory path name

    * :attr:`target_dir`
        target directory path name

    * :attr:`container_config`
        Instance of xml_parse::containerconfig

    * :attr:`requested_container_type`
        Configured container type

    * :attr:`system_setup`
        Instance of SystemSetup

    * :attr:`filename`
        File name of the container image

    * :attr:`result`
        Instance of Result
    """
    def __init__(self, xml_state, target_dir, root_dir):
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.container_config = xml_state.get_container_config()
        self.requested_container_type = xml_state.get_build_type_name()
        self.base_image = None
        self.base_image_md5 = None

        if xml_state.get_derived_from_image_uri():
            # The base image is expected to be unpacked by the kiwi
            # prepare step and stored inside of the root_dir/image directory.
            # In addition a md5 file of the image is expected too
            self.base_image = Defaults.get_imported_root_image(
                self.root_dir
            )
            self.base_image_md5 = ''.join([self.base_image, '.md5'])

            if not os.path.exists(self.base_image):
                raise KiwiContainerBuilderError(
                    'Unpacked Base image {0} not found'.format(
                        self.base_image
                    )
                )
            if not os.path.exists(self.base_image_md5):
                raise KiwiContainerBuilderError(
                    'Base image MD5 sum {0} not found at'.format(
                        self.base_image_md5
                    )
                )

        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.filename = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version(),
                '.', self.requested_container_type, '.tar.xz'
            ]
        )
        self.result = Result(xml_state)

    def create(self):
        """
        Builds a container image which is usually a tarball including
        container specific metadata.

        Image types which triggers this builder are:

        * image="docker"
        """
        if not self.base_image:
            log.info(
                'Setting up %s container', self.requested_container_type
            )
            container_setup = ContainerSetup(
                self.requested_container_type, self.root_dir,
                self.container_config
            )
            container_setup.setup()
        else:
            checksum = Checksum(self.base_image)
            if not checksum.matches(checksum.md5(), self.base_image_md5):
                raise KiwiContainerBuilderError(
                    'base image file {0} checksum validation failed'.format(
                        self.base_image
                    )
                )

        log.info(
            '--> Creating container image'
        )
        container_image = ContainerImage(
            self.requested_container_type, self.root_dir, self.container_config
        )
        container_image.create(
            self.filename, self.base_image
        )
        self.result.add(
            key='container',
            filename=self.filename,
            use_for_bundle=True,
            compress=False,
            shasum=True
        )
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_rpm_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_rpm_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result
Esempio n. 33
0
class TestResultBundleTask(object):
    def setup(self):
        sys.argv = [
            sys.argv[0], 'result', 'bundle', '--target-dir', 'target_dir',
            '--bundle-dir', 'bundle_dir', '--id', 'Build_42'
        ]
        self.abs_target_dir = os.path.abspath('target_dir')
        self.abs_bundle_dir = os.path.abspath('bundle_dir')
        self.context_manager_mock = mock.Mock()
        self.file_mock = mock.Mock()
        self.enter_mock = mock.Mock()
        self.exit_mock = mock.Mock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, '__enter__', self.enter_mock)
        setattr(self.context_manager_mock, '__exit__', self.exit_mock)

        self.file_mock.read.return_value = b'data'

        self.xml_state = mock.Mock()
        self.xml_state.get_image_version = mock.Mock(
            return_value='1.2.3'
        )

        self.result = Result(self.xml_state)
        self.result.add(
            key='keyname', filename='filename-1.2.3',
            use_for_bundle=True, compress=True, shasum=True
        )

        kiwi.tasks.result_bundle.Help = mock.Mock(
            return_value=mock.Mock()
        )
        self.task = ResultBundleTask()

    def teardown(self):
        sys.argv = argv_kiwi_tests

    def _init_command_args(self):
        self.task.command_args = {}
        self.task.command_args['help'] = False
        self.task.command_args['bundle'] = False
        self.task.command_args['--target-dir'] = 'target_dir'
        self.task.command_args['--bundle-dir'] = 'bundle_dir'
        self.task.command_args['--id'] = 'Build_42'

    @raises(KiwiBundleError)
    def test_process_invalid_bundle_directory(self):
        self._init_command_args()
        self.task.command_args['--bundle-dir'] = \
            self.task.command_args['--target-dir']
        self.task.command_args['bundle'] = True
        self.task.process()

    @patch('kiwi.tasks.result_bundle.Result.load')
    @patch('kiwi.tasks.result_bundle.Command.run')
    @patch('kiwi.tasks.result_bundle.Path.create')
    @patch('kiwi.tasks.result_bundle.Compress')
    @patch('kiwi.tasks.result_bundle.Checksum')
    @patch('os.path.exists')
    @patch_open
    def test_process_result_bundle(
        self, mock_open, mock_exists, mock_checksum, mock_compress,
        mock_path, mock_command, mock_load
    ):
        checksum = mock.Mock()
        compress = mock.Mock()
        compress.compressed_filename = 'compressed_filename'
        mock_compress.return_value = compress
        mock_checksum.return_value = checksum
        mock_exists.return_value = False
        mock_open.return_value = self.context_manager_mock
        mock_load.return_value = self.result
        self._init_command_args()
        self.task.command_args['bundle'] = True

        self.task.process()

        mock_load.assert_called_once_with(os.sep.join([self.abs_target_dir, 'kiwi.result']))
        mock_path.assert_called_once_with(self.abs_bundle_dir)
        mock_command.assert_called_once_with(
            [
                'cp', 'filename-1.2.3',
                os.sep.join([self.abs_bundle_dir, 'filename-1.2.3-Build_42'])
            ]
        )
        mock_compress.assert_called_once_with(
            os.sep.join([self.abs_bundle_dir, 'filename-1.2.3-Build_42'])
        )
        mock_checksum.assert_called_once_with(
            compress.compressed_filename
        )
        checksum.sha256.assert_called_once_with()

    def test_process_result_bundle_help(self):
        self._init_command_args()
        self.task.command_args['help'] = True
        self.task.command_args['bundle'] = True
        self.task.process()
        self.task.manual.show.assert_called_once_with(
            'kiwi::result::bundle'
        )
Esempio n. 34
0
class FileSystemBuilder(object):
    """
    Filesystem image builder

    Attributes

    * :attr:`label`
        filesystem label

    * :attr:`root_dir`
        root directory path name

    * :attr:`target_dir`
        target directory path name

    * :attr:`requested_image_type`
        Configured image type

    * :attr:`requested_filesystem`
        Requested filesystem name

    * :attr:`system_setup`
        Instance of SystemSetup

    * :attr:`filename`
        File name of the filesystem image

    * :attr:`blocksize`
        Configured disk blocksize

    * :attr:`filesystem_setup`
        Instance of FileSystemSetup

    * :attr:`filesystems_no_device_node`
        List of filesystems which are created from a data tree
        and do not require a block device e.g loop

    * :attr:`filesystem_custom_parameters`
        Configured custom filesystem mount and creation arguments

    * :attr:`result`
        Instance of Result
    """
    def __init__(self, xml_state, target_dir, root_dir):
        self.label = None
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.requested_image_type = xml_state.get_build_type_name()
        if self.requested_image_type == 'pxe':
            self.requested_filesystem = xml_state.build_type.get_filesystem()
        else:
            self.requested_filesystem = self.requested_image_type
        if not self.requested_filesystem:
            raise KiwiFileSystemSetupError(
                'No filesystem configured in %s type' %
                self.requested_image_type)
        self.filesystem_custom_parameters = {
            'mount_options': xml_state.get_fs_mount_option_list()
        }
        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.filename = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '.' + platform.machine(),
            '-' + xml_state.get_image_version(), '.', self.requested_filesystem
        ])
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.filesystem_setup = FileSystemSetup(xml_state, root_dir)
        self.filesystems_no_device_node = ['squashfs']
        self.result = Result(xml_state)

    def create(self):
        """
        Build a mountable filesystem image

        Image types which triggers this builder are:

        * image="ext2"
        * image="ext3"
        * image="ext4"
        * image="btrfs"
        * image="xfs"
        """
        log.info('Creating %s filesystem', self.requested_filesystem)
        supported_filesystems = Defaults.get_filesystem_image_types()
        if self.requested_filesystem not in supported_filesystems:
            raise KiwiFileSystemSetupError('Unknown filesystem: %s' %
                                           self.requested_filesystem)
        if self.requested_filesystem not in self.filesystems_no_device_node:
            self._operate_on_loop()
        else:
            self._operate_on_file()
        self.result.add(key='filesystem_image',
                        filename=self.filename,
                        use_for_bundle=True,
                        compress=True,
                        shasum=True)
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        return self.result

    def _operate_on_loop(self):
        filesystem = None
        loop_provider = LoopDevice(self.filename,
                                   self.filesystem_setup.get_size_mbytes(),
                                   self.blocksize)
        loop_provider.create()
        filesystem = FileSystem(self.requested_filesystem, loop_provider,
                                self.root_dir,
                                self.filesystem_custom_parameters)
        filesystem.create_on_device(self.label)
        log.info('--> Syncing data to filesystem on %s',
                 loop_provider.get_device())
        exclude_list = [
            'image', '.profile', '.kconfig',
            Defaults.get_shared_cache_location()
        ]
        filesystem.sync_data(exclude_list)

    def _operate_on_file(self):
        default_provider = DeviceProvider()
        filesystem = FileSystem(self.requested_filesystem, default_provider,
                                self.root_dir,
                                self.filesystem_custom_parameters)
        filesystem.create_on_file(self.filename, self.label)
Esempio n. 35
0
class TestResult(object):
    def setup(self):
        self.context_manager_mock = mock.MagicMock()
        self.file_mock = mock.MagicMock()
        self.enter_mock = mock.MagicMock()
        self.exit_mock = mock.MagicMock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, '__enter__', self.enter_mock)
        setattr(self.context_manager_mock, '__exit__', self.exit_mock)

        self.xml_state = mock.Mock()

        self.result = Result(self.xml_state)

    def test_add(self):
        self.result.add('foo', 'bar')
        result = self.result.get_results()
        assert result['foo'].filename == 'bar'
        assert result['foo'].use_for_bundle is True
        assert result['foo'].compress is False

    @patch('kiwi.logger.log.info')
    def test_print_results_no_data(self, mock_info):
        self.result.print_results()
        assert mock_info.called == 0

    @patch('kiwi.logger.log.info')
    def test_print_results_data(self, mock_info):
        self.result.add('foo', 'bar')
        self.result.print_results()
        assert mock_info.called

    @patch('pickle.dump')
    @patch_open
    def test_dump(self, mock_open, mock_pickle_dump):
        mock_open.return_value = self.context_manager_mock
        self.result.dump('kiwi.result')
        mock_open.assert_called_once_with(
            'kiwi.result', 'wb'
        )
        mock_pickle_dump.assert_called_once_with(
            self.result, self.file_mock
        )

    @patch('pickle.dump')
    @patch_open
    @raises(KiwiResultError)
    def test_dump_failed(self, mock_open, mock_pickle_dump):
        mock_pickle_dump.side_effect = Exception
        self.result.dump('kiwi.result')

    @patch('pickle.load')
    @patch('os.path.exists')
    @patch_open
    def test_load(self, mock_open, mock_exists, mock_pickle_load):
        mock_open.return_value = self.context_manager_mock
        mock_exists.return_value = True
        Result.load('kiwi.result')
        mock_open.assert_called_once_with(
            'kiwi.result', 'rb'
        )
        mock_pickle_load.assert_called_once_with(
            self.file_mock
        )

    @patch('os.path.exists')
    @raises(KiwiResultError)
    def test_load_result_not_present(self, mock_exists):
        mock_exists.return_value = False
        Result.load('kiwi.result')

    @patch('pickle.load')
    @patch('os.path.exists')
    @raises(KiwiResultError)
    def test_load_failed(self, mock_exists, mock_pickle_load):
        mock_exists.return_value = True
        mock_pickle_load.side_effect = Exception
        Result.load('kiwi.result')
Esempio n. 36
0
class FileSystemBuilder:
    """
    **Filesystem image builder**

    :param obsject xml_state: Instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: root directory path name
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * None
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.label = None
        self.root_uuid = None
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.requested_image_type = xml_state.get_build_type_name()
        if self.requested_image_type in Defaults.get_kis_image_types():
            self.requested_filesystem = xml_state.build_type.get_filesystem()
        else:
            self.requested_filesystem = self.requested_image_type
        if not self.requested_filesystem:
            raise KiwiFileSystemSetupError(
                'No filesystem configured in %s type' %
                self.requested_image_type
            )
        self.filesystem_custom_parameters = {
            'mount_options': xml_state.get_fs_mount_option_list(),
            'create_options': xml_state.get_fs_create_option_list()
        }
        if self.requested_filesystem == 'squashfs':
            self.filesystem_custom_parameters['compression'] = \
                xml_state.build_type.get_squashfscompression()

        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.filename = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + Defaults.get_platform_name(),
                '-' + xml_state.get_image_version(),
                '.', self.requested_filesystem
            ]
        )
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.filesystem_setup = FileSystemSetup(xml_state, root_dir)
        self.filesystems_no_device_node = [
            'squashfs'
        ]
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a mountable filesystem image

        Image types which triggers this builder are:

        * image="ext2"
        * image="ext3"
        * image="ext4"
        * image="btrfs"
        * image="xfs"

        :return: result

        :rtype: instance of :class:`Result`
        """
        log.info(
            'Creating %s filesystem', self.requested_filesystem
        )
        supported_filesystems = Defaults.get_filesystem_image_types()
        if self.requested_filesystem not in supported_filesystems:
            raise KiwiFileSystemSetupError(
                'Unknown filesystem: %s' % self.requested_filesystem
            )
        if self.requested_filesystem not in self.filesystems_no_device_node:
            self._operate_on_loop()
        else:
            self._operate_on_file()
        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.filename
        )
        self.result.add(
            key='filesystem_image',
            filename=self.filename,
            use_for_bundle=True,
            compress=self.runtime_config.get_bundle_compression(
                default=True
            ),
            shasum=True
        )
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result

    def _operate_on_loop(self):
        filesystem = None
        loop_provider = LoopDevice(
            self.filename,
            self.filesystem_setup.get_size_mbytes(),
            self.blocksize
        )
        loop_provider.create()
        filesystem = FileSystem.new(
            self.requested_filesystem, loop_provider,
            self.root_dir + os.sep, self.filesystem_custom_parameters
        )
        filesystem.create_on_device(self.label)
        self.root_uuid = loop_provider.get_uuid(loop_provider.get_device())
        log.info(
            '--> Syncing data to filesystem on %s', loop_provider.get_device()
        )
        filesystem.sync_data(
            Defaults.get_exclude_list_for_root_data_sync()
        )

    def _operate_on_file(self):
        default_provider = DeviceProvider()
        filesystem = FileSystem.new(
            self.requested_filesystem, default_provider,
            self.root_dir, self.filesystem_custom_parameters
        )
        filesystem.create_on_file(
            self.filename, self.label,
            Defaults.get_exclude_list_for_root_data_sync()
        )
Esempio n. 37
0
class LiveImageBuilder(object):
    """
    **Live image builder**

    :param object xml_state: instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: root directory path name
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.media_dir = None
        self.live_container_dir = None
        self.arch = platform.machine()
        if self.arch == 'i686' or self.arch == 'i586':
            self.arch = 'ix86'
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.live_type = xml_state.build_type.get_flags()
        self.volume_id = xml_state.build_type.get_volid() or \
            Defaults.get_volume_id()
        self.mbrid = SystemIdentifier()
        self.mbrid.calculate_id()
        self.filesystem_custom_parameters = {
            'mount_options': xml_state.get_fs_mount_option_list()
        }
        self.publisher = xml_state.build_type.get_publisher() or \
            Defaults.get_publisher()

        if not self.live_type:
            self.live_type = Defaults.get_default_live_iso_type()

        self.boot_image = BootImageDracut(
            xml_state, target_dir, self.root_dir
        )
        self.firmware = FirmWare(
            xml_state
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.isoname = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version(),
                '.iso'
            ]
        )
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a bootable hybrid live ISO image

        Image types which triggers this builder are:

        * image="iso"

        :raises KiwiLiveBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        # media dir to store CD contents
        self.media_dir = mkdtemp(
            prefix='live-media.', dir=self.target_dir
        )

        # unpack cdroot user files to media dir
        self.system_setup.import_cdroot_files(self.media_dir)

        rootsize = SystemSize(self.media_dir)

        # custom iso metadata
        log.info('Using following live ISO metadata:')
        log.info('--> Application id: {0}'.format(self.mbrid.get_id()))
        log.info('--> Publisher: {0}'.format(Defaults.get_publisher()))
        log.info('--> Volume id: {0}'.format(self.volume_id))
        custom_iso_args = {
            'meta_data': {
                'publisher': self.publisher,
                'preparer': Defaults.get_preparer(),
                'volume_id': self.volume_id,
                'mbr_id': self.mbrid.get_id(),
                'efi_mode': self.firmware.efi_mode()
            }
        }

        # pack system into live boot structure as expected by dracut
        log.info(
            'Packing system into dracut live ISO type: {0}'.format(
                self.live_type
            )
        )
        root_filesystem = Defaults.get_default_live_iso_root_filesystem()
        filesystem_custom_parameters = {
            'mount_options': self.xml_state.get_fs_mount_option_list()
        }
        filesystem_setup = FileSystemSetup(
            self.xml_state, self.root_dir
        )
        root_image = NamedTemporaryFile()
        loop_provider = LoopDevice(
            root_image.name,
            filesystem_setup.get_size_mbytes(root_filesystem),
            self.xml_state.build_type.get_target_blocksize()
        )
        loop_provider.create()
        live_filesystem = FileSystem(
            name=root_filesystem,
            device_provider=loop_provider,
            root_dir=self.root_dir + os.sep,
            custom_args=filesystem_custom_parameters
        )
        live_filesystem.create_on_device()
        log.info(
            '--> Syncing data to {0} root image'.format(root_filesystem)
        )
        live_filesystem.sync_data(
            Defaults.get_exclude_list_for_root_data_sync()
        )
        log.info('--> Creating squashfs container for root image')
        self.live_container_dir = mkdtemp(
            prefix='live-container.', dir=self.target_dir
        )
        Path.create(self.live_container_dir + '/LiveOS')
        shutil.copy(
            root_image.name, self.live_container_dir + '/LiveOS/rootfs.img'
        )
        live_container_image = FileSystem(
            name='squashfs',
            device_provider=None,
            root_dir=self.live_container_dir
        )
        container_image = NamedTemporaryFile()
        live_container_image.create_on_file(
            container_image.name
        )
        Path.create(self.media_dir + '/LiveOS')
        shutil.copy(
            container_image.name, self.media_dir + '/LiveOS/squashfs.img'
        )

        # setup bootloader config to boot the ISO via isolinux
        log.info('Setting up isolinux bootloader configuration')
        bootloader_config_isolinux = BootLoaderConfig(
            'isolinux', self.xml_state, self.media_dir
        )
        bootloader_config_isolinux.setup_live_boot_images(
            mbrid=None,
            lookup_path=self.boot_image.boot_root_directory
        )
        bootloader_config_isolinux.setup_live_image_config(
            mbrid=None
        )
        bootloader_config_isolinux.write()

        # setup bootloader config to boot the ISO via EFI
        if self.firmware.efi_mode():
            log.info('Setting up EFI grub bootloader configuration')
            bootloader_config_grub = BootLoaderConfig(
                'grub2', self.xml_state, self.media_dir, {
                    'grub_directory_name':
                        Defaults.get_grub_boot_directory_name(self.root_dir)
                }
            )
            bootloader_config_grub.setup_live_boot_images(
                mbrid=self.mbrid, lookup_path=self.root_dir
            )
            bootloader_config_grub.setup_live_image_config(
                mbrid=self.mbrid
            )
            bootloader_config_grub.write()

        # call custom editbootconfig script if present
        self.system_setup.call_edit_boot_config_script(
            filesystem='iso:{0}'.format(self.media_dir), boot_part_id=1,
            working_directory=self.root_dir
        )

        # prepare dracut initrd call
        self.boot_image.prepare()

        # create dracut initrd for live image
        log.info('Creating live ISO boot image')
        self._create_dracut_live_iso_config()
        self.boot_image.create_initrd(self.mbrid)

        # setup kernel file(s) and initrd in ISO boot layout
        log.info('Setting up kernel file(s) and boot image in ISO boot layout')
        self._setup_live_iso_kernel_and_initrd()

        # calculate size and decide if we need UDF
        if rootsize.accumulate_mbyte_file_sizes() > 4096:
            log.info('ISO exceeds 4G size, using UDF filesystem')
            custom_iso_args['meta_data']['udf'] = True

        # create iso filesystem from media_dir
        log.info('Creating live ISO image')
        iso_image = FileSystemIsoFs(
            device_provider=None, root_dir=self.media_dir,
            custom_args=custom_iso_args
        )
        iso_image.create_on_file(self.isoname)

        # include metadata for checkmedia tool
        if self.xml_state.build_type.get_mediacheck() is True:
            Iso.set_media_tag(self.isoname)

        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.isoname
        )
        self.result.add(
            key='live_image',
            filename=self.isoname,
            use_for_bundle=True,
            compress=False,
            shasum=True
        )
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result

    def _create_dracut_live_iso_config(self):
        live_config_file = self.root_dir + '/etc/dracut.conf.d/02-livecd.conf'
        omit_modules = [
            'kiwi-dump', 'kiwi-overlay', 'kiwi-repart', 'kiwi-lib', 'multipath'
        ]
        live_config = [
            'add_dracutmodules+=" {0} pollcdrom "'.format(
                Defaults.get_live_dracut_module_from_flag(self.live_type)
            ),
            'omit_dracutmodules+=" {0} "'.format(' '.join(omit_modules)),
            'hostonly="no"',
            'dracut_rescue_image="no"'
        ]
        with open(live_config_file, 'w') as config:
            for entry in live_config:
                config.write(entry + os.linesep)

    def _setup_live_iso_kernel_and_initrd(self):
        """
        Copy kernel and initrd from the root tree into the iso boot structure
        """
        boot_path = ''.join(
            [self.media_dir, '/boot/', self.arch, '/loader']
        )
        Path.create(boot_path)

        # Move kernel files to iso filesystem structure
        kernel = Kernel(self.boot_image.boot_root_directory)
        if kernel.get_kernel():
            kernel.copy_kernel(boot_path, '/linux')
        else:
            raise KiwiLiveBootImageError(
                'No kernel in boot image tree {0} found'.format(
                    self.boot_image.boot_root_directory
                )
            )
        if self.xml_state.is_xen_server():
            if kernel.get_xen_hypervisor():
                kernel.copy_xen_hypervisor(boot_path, '/xen.gz')
            else:
                raise KiwiLiveBootImageError(
                    'No hypervisor in boot image tree {0} found'.format(
                        self.boot_image.boot_root_directory
                    )
                )

        # Move initrd to iso filesystem structure
        if os.path.exists(self.boot_image.initrd_filename):
            shutil.move(
                self.boot_image.initrd_filename, boot_path + '/initrd'
            )
        else:
            raise KiwiLiveBootImageError(
                'No boot image {0} in boot image tree {1} found'.format(
                    self.boot_image.initrd_filename,
                    self.boot_image.boot_root_directory
                )
            )

    def __del__(self):
        if self.media_dir or self.live_container_dir:
            log.info(
                'Cleaning up {0} instance'.format(type(self).__name__)
            )
            if self.media_dir:
                Path.wipe(self.media_dir)
            if self.live_container_dir:
                Path.wipe(self.live_container_dir)
Esempio n. 38
0
 def test_load_result_not_present(self, mock_exists):
     mock_exists.return_value = False
     with raises(KiwiResultError):
         Result.load('kiwi.result')
Esempio n. 39
0
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.arch = platform.machine()
        if self.arch == 'i686' or self.arch == 'i586':
            self.arch = 'ix86'
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.spare_part_mbsize = xml_state.get_build_type_spare_part_size()
        self.persistency_type = xml_state.build_type.get_devicepersistency()
        self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot()
        self.custom_root_mount_args = xml_state.get_fs_mount_option_list()
        self.build_type_name = xml_state.get_build_type_name()
        self.image_format = xml_state.build_type.get_format()
        self.install_iso = xml_state.build_type.get_installiso()
        self.install_stick = xml_state.build_type.get_installstick()
        self.install_pxe = xml_state.build_type.get_installpxe()
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.volume_manager_name = xml_state.get_volume_management()
        self.volumes = xml_state.get_volumes()
        self.volume_group_name = xml_state.get_volume_group_name()
        self.mdraid = xml_state.build_type.get_mdraid()
        self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr()
        self.force_mbr = xml_state.build_type.get_force_mbr()
        self.luks = xml_state.build_type.get_luks()
        self.luks_os = xml_state.build_type.get_luksOS()
        self.xen_server = xml_state.is_xen_server()
        self.requested_filesystem = xml_state.build_type.get_filesystem()
        self.requested_boot_filesystem = \
            xml_state.build_type.get_bootfilesystem()
        self.bootloader = xml_state.build_type.get_bootloader()
        self.initrd_system = xml_state.get_initrd_system()
        self.target_removable = xml_state.build_type.get_target_removable()
        self.root_filesystem_is_multipath = \
            xml_state.get_oemconfig_oem_multipath_scan()
        self.disk_setup = DiskSetup(
            xml_state, root_dir
        )
        self.unpartitioned_bytes = \
            xml_state.get_build_type_unpartitioned_bytes()
        self.custom_args = custom_args

        self.signing_keys = None
        if custom_args and 'signing_keys' in custom_args:
            self.signing_keys = custom_args['signing_keys']

        self.boot_image = BootImage(
            xml_state, target_dir, root_dir,
            signing_keys=self.signing_keys, custom_args=self.custom_args
        )
        self.firmware = FirmWare(
            xml_state
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.diskname = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + self.arch,
                '-' + xml_state.get_image_version(),
                '.raw'
            ]
        )
        self.install_media = self._install_image_requested()
        self.generic_fstab_entries = []

        # an instance of a class with the sync_data capability
        # representing the entire image system except for the boot/ area
        # which could live on another part of the disk
        self.system = None

        # an instance of a class with the sync_data capability
        # representing the boot/ area of the disk if not part of
        # self.system
        self.system_boot = None

        # an instance of a class with the sync_data capability
        # representing the boot/efi area of the disk
        self.system_efi = None

        # result store
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()
Esempio n. 40
0
class TestResult(object):
    def setup(self):
        self.context_manager_mock = mock.MagicMock()
        self.file_mock = mock.MagicMock()
        self.enter_mock = mock.MagicMock()
        self.exit_mock = mock.MagicMock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, '__enter__', self.enter_mock)
        setattr(self.context_manager_mock, '__exit__', self.exit_mock)

        self.xml_state = mock.Mock()

        self.result = Result(self.xml_state)

    def test_add(self):
        self.result.add('foo', 'bar')
        result = self.result.get_results()
        assert result['foo'].filename == 'bar'
        assert result['foo'].use_for_bundle is True
        assert result['foo'].compress is False

    @patch('kiwi.logger.log.info')
    def test_print_results_no_data(self, mock_info):
        self.result.print_results()
        assert mock_info.called == 0

    @patch('kiwi.logger.log.info')
    def test_print_results_data(self, mock_info):
        self.result.add('foo', 'bar')
        self.result.print_results()
        assert mock_info.called

    @patch('pickle.dump')
    @patch_open
    def test_dump(self, mock_open, mock_pickle_dump):
        mock_open.return_value = self.context_manager_mock
        self.result.dump('kiwi.result')
        mock_open.assert_called_once_with('kiwi.result', 'wb')
        mock_pickle_dump.assert_called_once_with(self.result, self.file_mock)

    @patch('pickle.dump')
    @patch_open
    @raises(KiwiResultError)
    def test_dump_failed(self, mock_open, mock_pickle_dump):
        mock_pickle_dump.side_effect = Exception
        self.result.dump('kiwi.result')

    @patch('pickle.load')
    @patch('os.path.exists')
    @patch_open
    def test_load(self, mock_open, mock_exists, mock_pickle_load):
        mock_open.return_value = self.context_manager_mock
        mock_exists.return_value = True
        Result.load('kiwi.result')
        mock_open.assert_called_once_with('kiwi.result', 'rb')
        mock_pickle_load.assert_called_once_with(self.file_mock)

    @patch('os.path.exists')
    @raises(KiwiResultError)
    def test_load_result_not_present(self, mock_exists):
        mock_exists.return_value = False
        Result.load('kiwi.result')

    @patch('pickle.load')
    @patch('os.path.exists')
    @raises(KiwiResultError)
    def test_load_failed(self, mock_exists, mock_pickle_load):
        mock_exists.return_value = True
        mock_pickle_load.side_effect = Exception
        Result.load('kiwi.result')

    @patch('os.path.getsize')
    def test_build_constraint(self, mock_getsize):
        mock_getsize.return_value = 524288000
        self.result.verify_image_size(524288000, 'foo')

    @patch('os.path.getsize')
    @raises(KiwiResultError)
    def test_build_constraint_failure(self, mock_getsize):
        mock_getsize.return_value = 524288000
        self.result.verify_image_size(524287999, 'foo')
Esempio n. 41
0
 def test_load_result_not_present(self, mock_exists):
     mock_exists.return_value = False
     Result.load('kiwi.result')
Esempio n. 42
0
 def test_load_result_not_present(self, mock_exists):
     mock_exists.return_value = False
     Result.load('kiwi.result')
Esempio n. 43
0
 def test_load_failed(self, mock_exists, mock_pickle_load):
     mock_exists.return_value = True
     mock_pickle_load.side_effect = Exception
     Result.load('kiwi.result')
Esempio n. 44
0
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.arch = platform.machine()
        if self.arch == 'i686' or self.arch == 'i586':
            self.arch = 'ix86'
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.spare_part_mbsize = xml_state.get_build_type_spare_part_size()
        self.spare_part_fs = xml_state.build_type.get_spare_part_fs()
        self.spare_part_is_last = xml_state.build_type.get_spare_part_is_last()
        self.spare_part_mountpoint = \
            xml_state.build_type.get_spare_part_mountpoint()
        self.persistency_type = xml_state.build_type.get_devicepersistency()
        self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot()
        self.custom_root_mount_args = xml_state.get_fs_mount_option_list()
        self.build_type_name = xml_state.get_build_type_name()
        self.image_format = xml_state.build_type.get_format()
        self.install_iso = xml_state.build_type.get_installiso()
        self.install_stick = xml_state.build_type.get_installstick()
        self.install_pxe = xml_state.build_type.get_installpxe()
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.volume_manager_name = xml_state.get_volume_management()
        self.volumes = xml_state.get_volumes()
        self.volume_group_name = xml_state.get_volume_group_name()
        self.mdraid = xml_state.build_type.get_mdraid()
        self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr()
        self.force_mbr = xml_state.build_type.get_force_mbr()
        self.luks = xml_state.build_type.get_luks()
        self.luks_os = xml_state.build_type.get_luksOS()
        self.xen_server = xml_state.is_xen_server()
        self.requested_filesystem = xml_state.build_type.get_filesystem()
        self.requested_boot_filesystem = \
            xml_state.build_type.get_bootfilesystem()
        self.bootloader = xml_state.build_type.get_bootloader()
        self.initrd_system = xml_state.get_initrd_system()
        self.target_removable = xml_state.build_type.get_target_removable()
        self.root_filesystem_is_multipath = \
            xml_state.get_oemconfig_oem_multipath_scan()
        self.disk_setup = DiskSetup(
            xml_state, root_dir
        )
        self.unpartitioned_bytes = \
            xml_state.get_build_type_unpartitioned_bytes()
        self.custom_args = custom_args

        self.signing_keys = None
        if custom_args and 'signing_keys' in custom_args:
            self.signing_keys = custom_args['signing_keys']

        self.boot_image = BootImage(
            xml_state, target_dir, root_dir, signing_keys=self.signing_keys
        )
        self.firmware = FirmWare(
            xml_state
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.diskname = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + self.arch,
                '-' + xml_state.get_image_version(),
                '.raw'
            ]
        )
        self.install_media = self._install_image_requested()
        self.generic_fstab_entries = []

        # an instance of a class with the sync_data capability
        # representing the entire image system except for the boot/ area
        # which could live on another part of the disk
        self.system = None

        # an instance of a class with the sync_data capability
        # representing the boot/ area of the disk if not part of
        # self.system
        self.system_boot = None

        # an instance of a class with the sync_data capability
        # representing the boot/efi area of the disk
        self.system_efi = None

        # an instance of a class with the sync_data capability
        # representing the spare_part_mountpoint area of the disk
        self.system_spare = None

        # result store
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()
Esempio n. 45
0
class TestResultBundleTask(object):
    def setup(self):
        sys.argv = [
            sys.argv[0],
            "result",
            "bundle",
            "--target-dir",
            "target_dir",
            "--bundle-dir",
            "bundle_dir",
            "--id",
            "Build_42",
        ]
        self.context_manager_mock = mock.Mock()
        self.file_mock = mock.Mock()
        self.enter_mock = mock.Mock()
        self.exit_mock = mock.Mock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, "__enter__", self.enter_mock)
        setattr(self.context_manager_mock, "__exit__", self.exit_mock)

        self.file_mock.read.return_value = b"data"

        self.xml_state = mock.Mock()
        self.xml_state.get_image_version = mock.Mock(return_value="1.2.3")

        self.result = Result(self.xml_state)
        self.result.add(key="keyname", filename="filename-1.2.3", use_for_bundle=True, compress=True, shasum=True)

        kiwi.tasks.result_bundle.Help = mock.Mock(return_value=mock.Mock())
        self.task = ResultBundleTask()

    def _init_command_args(self):
        self.task.command_args = {}
        self.task.command_args["help"] = False
        self.task.command_args["bundle"] = False
        self.task.command_args["--target-dir"] = "target_dir"
        self.task.command_args["--bundle-dir"] = "bundle_dir"
        self.task.command_args["--id"] = "Build_42"

    @raises(KiwiBundleError)
    def test_process_invalid_bundle_directory(self):
        self._init_command_args()
        self.task.command_args["--bundle-dir"] = self.task.command_args["--target-dir"]
        self.task.command_args["bundle"] = True
        self.task.process()

    @patch("kiwi.tasks.result_bundle.Result.load")
    @patch("kiwi.tasks.result_bundle.Command.run")
    @patch("kiwi.tasks.result_bundle.Path.create")
    @patch("kiwi.tasks.result_bundle.Compress")
    @patch("kiwi.tasks.result_bundle.Checksum")
    @patch("os.path.exists")
    @patch_open
    def test_process_result_bundle(
        self, mock_open, mock_exists, mock_checksum, mock_compress, mock_path, mock_command, mock_load
    ):
        checksum = mock.Mock()
        compress = mock.Mock()
        compress.compressed_filename = "compressed_filename"
        mock_compress.return_value = compress
        mock_checksum.return_value = checksum
        mock_exists.return_value = False
        mock_open.return_value = self.context_manager_mock
        mock_load.return_value = self.result
        self._init_command_args()
        self.task.command_args["bundle"] = True

        self.task.process()

        mock_load.assert_called_once_with("target_dir/kiwi.result")
        mock_path.assert_called_once_with("bundle_dir")
        mock_command.assert_called_once_with(["cp", "-l", "filename-1.2.3", "bundle_dir/filename-1.2.3-Build_42"])
        mock_compress.assert_called_once_with("bundle_dir/filename-1.2.3-Build_42")
        mock_checksum.assert_called_once_with(compress.compressed_filename)
        checksum.sha256.assert_called_once_with()

    def test_process_result_bundle_help(self):
        self._init_command_args()
        self.task.command_args["help"] = True
        self.task.command_args["bundle"] = True
        self.task.process()
        self.task.manual.show.assert_called_once_with("kiwi::result::bundle")
Esempio n. 46
0
class ArchiveBuilder:
    """
    **Root archive image builder**

    :param obsject xml_state: Instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: root directory path name
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.xml_state = xml_state
        self.requested_archive_type = xml_state.get_build_type_name()
        self.result = Result(xml_state)
        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.filename = self._target_file_for('tar.xz')
        self.xz_options = custom_args['xz_options'] if custom_args \
            and 'xz_options' in custom_args else None

        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Create a root archive tarball

        Build a simple XZ compressed root tarball from the image root tree

        Image types which triggers this builder are:

        * image="tbz"

        :return: result

        :rtype: instance of :class:`Result`
        """
        supported_archives = Defaults.get_archive_image_types()
        if self.requested_archive_type not in supported_archives:
            raise KiwiArchiveSetupError('Unknown archive type: %s' %
                                        self.requested_archive_type)

        if self.requested_archive_type == 'tbz':
            log.info('Creating XZ compressed tar archive')
            archive = ArchiveTar(self._target_file_for('tar'))
            archive.create_xz_compressed(
                self.root_dir,
                xz_options=self.xz_options,
                exclude=Defaults.get_exclude_list_for_root_data_sync())
            self.result.verify_image_size(
                self.runtime_config.get_max_size_constraint(), self.filename)
            self.result.add(key='root_archive',
                            filename=self.filename,
                            use_for_bundle=True,
                            compress=False,
                            shasum=True)
            self.result.add(key='image_packages',
                            filename=self.system_setup.export_package_list(
                                self.target_dir),
                            use_for_bundle=True,
                            compress=False,
                            shasum=False)
            self.result.add(key='image_changes',
                            filename=self.system_setup.export_package_changes(
                                self.target_dir),
                            use_for_bundle=True,
                            compress=True,
                            shasum=False)
            self.result.add(
                key='image_verified',
                filename=self.system_setup.export_package_verification(
                    self.target_dir),
                use_for_bundle=True,
                compress=False,
                shasum=False)
        return self.result

    def _target_file_for(self, suffix):
        return ''.join([
            self.target_dir, '/',
            self.xml_state.xml_data.get_name(),
            '.' + Defaults.get_platform_name(),
            '-' + self.xml_state.get_image_version(), '.', suffix
        ])
Esempio n. 47
0
class ContainerBuilder:
    """
    **Container image builder**

    :param object xml_state: Instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: root directory path name
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * xz_options: string of XZ compression parameters
    """
    def __init__(self,
                 xml_state: XMLState,
                 target_dir: str,
                 root_dir: str,
                 custom_args: Dict = None):
        self.custom_args = custom_args or {}
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.container_config = xml_state.get_container_config()
        self.requested_container_type = xml_state.get_build_type_name()
        self.base_image = None
        self.base_image_md5 = None

        self.container_config['xz_options'] = \
            self.custom_args.get('xz_options')

        self.container_config['metadata_path'] = \
            xml_state.build_type.get_metadata_path()

        if xml_state.get_derived_from_image_uri():
            # The base image is expected to be unpacked by the kiwi
            # prepare step and stored inside of the root_dir/image directory.
            # In addition a md5 file of the image is expected too
            self.base_image = Defaults.get_imported_root_image(self.root_dir)
            self.base_image_md5 = ''.join([self.base_image, '.md5'])

            if not os.path.exists(self.base_image):
                raise KiwiContainerBuilderError(
                    'Unpacked Base image {0} not found'.format(
                        self.base_image))
            if not os.path.exists(self.base_image_md5):
                raise KiwiContainerBuilderError(
                    'Base image MD5 sum {0} not found at'.format(
                        self.base_image_md5))

        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.filename = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '.' + Defaults.get_platform_name(),
            '-' + xml_state.get_image_version(), '.',
            self.requested_container_type,
            '.tar' if self.requested_container_type != 'appx' else ''
        ])
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self) -> Result:
        """
        Builds a container image which is usually a data archive
        including container specific metadata.

        Image types which triggers this builder are:

        * image="docker"
        * image="oci"
        * image="appx"

        :return: result

        :rtype: instance of :class:`Result`
        """
        if not self.base_image:
            log.info('Setting up %s container', self.requested_container_type)
            container_setup = ContainerSetup.new(self.requested_container_type,
                                                 self.root_dir,
                                                 self.container_config)
            container_setup.setup()
        else:
            checksum = Checksum(self.base_image)
            if not checksum.matches(checksum.md5(), self.base_image_md5):
                raise KiwiContainerBuilderError(
                    'base image file {0} checksum validation failed'.format(
                        self.base_image))

        log.info('--> Creating container image')
        container_image = ContainerImage.new(self.requested_container_type,
                                             self.root_dir,
                                             self.container_config)
        self.filename = container_image.create(self.filename, self.base_image)
        Result.verify_image_size(self.runtime_config.get_max_size_constraint(),
                                 self.filename)
        self.result.add(key='container',
                        filename=self.filename,
                        use_for_bundle=True,
                        compress=False,
                        shasum=True)
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_changes',
                        filename=self.system_setup.export_package_changes(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=True,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        return self.result
Esempio n. 48
0
class ContainerBuilder(object):
    """
    **Container image builder**

    :param object xml_state: Instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: root directory path name
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.container_config = xml_state.get_container_config()
        self.requested_container_type = xml_state.get_build_type_name()
        self.base_image = None
        self.base_image_md5 = None

        self.container_config['xz_options'] = custom_args['xz_options'] \
            if custom_args and 'xz_options' in custom_args else None

        if xml_state.get_derived_from_image_uri():
            # The base image is expected to be unpacked by the kiwi
            # prepare step and stored inside of the root_dir/image directory.
            # In addition a md5 file of the image is expected too
            self.base_image = Defaults.get_imported_root_image(
                self.root_dir
            )
            self.base_image_md5 = ''.join([self.base_image, '.md5'])

            if not os.path.exists(self.base_image):
                raise KiwiContainerBuilderError(
                    'Unpacked Base image {0} not found'.format(
                        self.base_image
                    )
                )
            if not os.path.exists(self.base_image_md5):
                raise KiwiContainerBuilderError(
                    'Base image MD5 sum {0} not found at'.format(
                        self.base_image_md5
                    )
                )

        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.filename = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version(),
                '.', self.requested_container_type, '.tar'
            ]
        )
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Builds a container image which is usually a tarball including
        container specific metadata.

        Image types which triggers this builder are:

        * image="docker"

        :return: result

        :rtype: instance of :class:`Result`
        """
        if not self.base_image:
            log.info(
                'Setting up %s container', self.requested_container_type
            )
            container_setup = ContainerSetup(
                self.requested_container_type, self.root_dir,
                self.container_config
            )
            container_setup.setup()
        else:
            checksum = Checksum(self.base_image)
            if not checksum.matches(checksum.md5(), self.base_image_md5):
                raise KiwiContainerBuilderError(
                    'base image file {0} checksum validation failed'.format(
                        self.base_image
                    )
                )

        log.info(
            '--> Creating container image'
        )
        container_image = ContainerImage(
            self.requested_container_type, self.root_dir, self.container_config
        )
        self.filename = container_image.create(
            self.filename, self.base_image
        )
        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.filename
        )
        self.result.add(
            key='container',
            filename=self.filename,
            use_for_bundle=True,
            compress=False,
            shasum=True
        )
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result
Esempio n. 49
0
class FileSystemBuilder(object):
    """
    **Filesystem image builder**

    :param str label: filesystem label
    :param str root_dir: root directory path name
    :param str target_dir: target directory path name
    :param str requested_image_type: configured image type
    :param str requested_filesystem: requested filesystem name
    :param obejct system_setup: instance of :class:`SystemSetup`
    :param str filename: file name of the filesystem image
    :param int blocksize: configured disk blocksize
    :param object filesystem_setup: instance of :class:`FileSystemSetup`
    :param object filesystems_no_device_node: List of filesystems which are
        created from a data tree and do not require a block device e.g loop
    :param dict filesystem_custom_parameters: Configured custom filesystem
        mount and creation arguments
    :param object result: instance of :class:`Result`
    """
    def __init__(self, xml_state, target_dir, root_dir):
        self.label = None
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.requested_image_type = xml_state.get_build_type_name()
        if self.requested_image_type == 'pxe':
            self.requested_filesystem = xml_state.build_type.get_filesystem()
        else:
            self.requested_filesystem = self.requested_image_type
        if not self.requested_filesystem:
            raise KiwiFileSystemSetupError(
                'No filesystem configured in %s type' %
                self.requested_image_type
            )
        self.filesystem_custom_parameters = {
            'mount_options': xml_state.get_fs_mount_option_list()
        }
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=self.root_dir
        )
        self.filename = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version(),
                '.', self.requested_filesystem
            ]
        )
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.filesystem_setup = FileSystemSetup(xml_state, root_dir)
        self.filesystems_no_device_node = [
            'squashfs'
        ]
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a mountable filesystem image

        Image types which triggers this builder are:

        * image="ext2"
        * image="ext3"
        * image="ext4"
        * image="btrfs"
        * image="xfs"

        :return: result

        :rtype: instance of :class:`Result`
        """
        log.info(
            'Creating %s filesystem', self.requested_filesystem
        )
        supported_filesystems = Defaults.get_filesystem_image_types()
        if self.requested_filesystem not in supported_filesystems:
            raise KiwiFileSystemSetupError(
                'Unknown filesystem: %s' % self.requested_filesystem
            )
        if self.requested_filesystem not in self.filesystems_no_device_node:
            self._operate_on_loop()
        else:
            self._operate_on_file()
        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.filename
        )
        self.result.add(
            key='filesystem_image',
            filename=self.filename,
            use_for_bundle=True,
            compress=True,
            shasum=True
        )
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result

    def _operate_on_loop(self):
        filesystem = None
        loop_provider = LoopDevice(
            self.filename,
            self.filesystem_setup.get_size_mbytes(),
            self.blocksize
        )
        loop_provider.create()
        filesystem = FileSystem(
            self.requested_filesystem, loop_provider,
            self.root_dir, self.filesystem_custom_parameters
        )
        filesystem.create_on_device(self.label)
        log.info(
            '--> Syncing data to filesystem on %s', loop_provider.get_device()
        )
        exclude_list = [
            'image', '.profile', '.kconfig',
            Defaults.get_shared_cache_location()
        ]
        filesystem.sync_data(exclude_list)

    def _operate_on_file(self):
        default_provider = DeviceProvider()
        filesystem = FileSystem(
            self.requested_filesystem, default_provider,
            self.root_dir, self.filesystem_custom_parameters
        )
        filesystem.create_on_file(
            self.filename, self.label
        )
Esempio n. 50
0
    def process(self):
        """
        Create result bundle from the image build results in the
        specified target directory. Each result image will contain
        the specified bundle identifier as part of its filename.
        Uncompressed image files will also become xz compressed
        and a sha sum will be created from every result image
        """
        self.manual = Help()
        if self._help():
            return

        # load serialized result object from target directory
        result_directory = os.path.abspath(self.command_args['--target-dir'])
        bundle_directory = os.path.abspath(self.command_args['--bundle-dir'])
        if result_directory == bundle_directory:
            raise KiwiBundleError(
                'Bundle directory must be different from target directory'
            )

        log.info(
            'Bundle build results from %s', result_directory
        )
        result = Result.load(
            result_directory + '/kiwi.result'
        )
        image_version = result.xml_state.get_image_version()
        ordered_results = OrderedDict(sorted(result.get_results().items()))

        # hard link bundle files, compress and build checksum
        if not os.path.exists(bundle_directory):
            Path.create(bundle_directory)
        for result_file in list(ordered_results.values()):
            if result_file.use_for_bundle:
                bundle_file_basename = os.path.basename(result_file.filename)
                # The bundle id is only taken into account for image results
                # which contains the image version in its nane
                bundle_file_basename = bundle_file_basename.replace(
                    image_version,
                    image_version + '-' + self.command_args['--id']
                )
                log.info('Creating %s', bundle_file_basename)
                bundle_file = ''.join(
                    [bundle_directory, '/', bundle_file_basename]
                )
                checksum_file = ''.join(
                    [bundle_directory, '/', bundle_file_basename, '.sha256']
                )
                Command.run(
                    [
                        'cp', result_file.filename, bundle_file
                    ]
                )
                if result_file.compress:
                    log.info('--> XZ compressing')
                    compress = Compress(bundle_file)
                    compress.xz(self.runtime_config.get_xz_options())
                    bundle_file = compress.compressed_filename
                    checksum_file = compress.compressed_filename + '.sha256'
                    if self.command_args['--zsync-source']:
                        zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK)
                        if zsyncmake:
                            log.info('--> Creating zsync control file')
                            Command.run(
                                [
                                    zsyncmake, '-e',
                                    '-u', os.sep.join(
                                        [
                                            self.command_args['--zsync-source'],
                                            os.path.basename(bundle_file)
                                        ]
                                    ),
                                    '-o', bundle_file + '.zsync',
                                    bundle_file
                                ]
                            )
                        else:
                            log.warning(
                                '--> zsyncmake missing, zsync setup skipped'
                            )
                if result_file.shasum:
                    log.info('--> Creating SHA 256 sum')
                    checksum = Checksum(bundle_file)
                    with open(checksum_file, 'w') as shasum:
                        shasum.write(checksum.sha256())
Esempio n. 51
0
class FileSystemBuilder(object):
    """
    **Filesystem image builder**

    :param str label: filesystem label
    :param str root_dir: root directory path name
    :param str target_dir: target directory path name
    :param str requested_image_type: configured image type
    :param str requested_filesystem: requested filesystem name
    :param obejct system_setup: instance of :class:`SystemSetup`
    :param str filename: file name of the filesystem image
    :param int blocksize: configured disk blocksize
    :param object filesystem_setup: instance of :class:`FileSystemSetup`
    :param object filesystems_no_device_node: List of filesystems which are
        created from a data tree and do not require a block device e.g loop
    :param dict filesystem_custom_parameters: Configured custom filesystem
        mount and creation arguments
    :param object result: instance of :class:`Result`
    """
    def __init__(self, xml_state, target_dir, root_dir):
        self.label = None
        self.root_dir = root_dir
        self.target_dir = target_dir
        self.requested_image_type = xml_state.get_build_type_name()
        if self.requested_image_type == 'pxe':
            self.requested_filesystem = xml_state.build_type.get_filesystem()
        else:
            self.requested_filesystem = self.requested_image_type
        if not self.requested_filesystem:
            raise KiwiFileSystemSetupError(
                'No filesystem configured in %s type' %
                self.requested_image_type)
        self.filesystem_custom_parameters = {
            'mount_options': xml_state.get_fs_mount_option_list()
        }
        self.system_setup = SystemSetup(xml_state=xml_state,
                                        root_dir=self.root_dir)
        self.filename = ''.join([
            target_dir, '/',
            xml_state.xml_data.get_name(), '.' + platform.machine(),
            '-' + xml_state.get_image_version(), '.', self.requested_filesystem
        ])
        self.blocksize = xml_state.build_type.get_target_blocksize()
        self.filesystem_setup = FileSystemSetup(xml_state, root_dir)
        self.filesystems_no_device_node = ['squashfs']
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a mountable filesystem image

        Image types which triggers this builder are:

        * image="ext2"
        * image="ext3"
        * image="ext4"
        * image="btrfs"
        * image="xfs"

        :return: result

        :rtype: instance of :class:`Result`
        """
        log.info('Creating %s filesystem', self.requested_filesystem)
        supported_filesystems = Defaults.get_filesystem_image_types()
        if self.requested_filesystem not in supported_filesystems:
            raise KiwiFileSystemSetupError('Unknown filesystem: %s' %
                                           self.requested_filesystem)
        if self.requested_filesystem not in self.filesystems_no_device_node:
            self._operate_on_loop()
        else:
            self._operate_on_file()
        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(), self.filename)
        self.result.add(
            key='filesystem_image',
            filename=self.filename,
            use_for_bundle=True,
            compress=self.runtime_config.get_bundle_compression(default=True),
            shasum=True)
        self.result.add(key='image_packages',
                        filename=self.system_setup.export_package_list(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        self.result.add(key='image_verified',
                        filename=self.system_setup.export_package_verification(
                            self.target_dir),
                        use_for_bundle=True,
                        compress=False,
                        shasum=False)
        return self.result

    def _operate_on_loop(self):
        filesystem = None
        loop_provider = LoopDevice(self.filename,
                                   self.filesystem_setup.get_size_mbytes(),
                                   self.blocksize)
        loop_provider.create()
        filesystem = FileSystem(self.requested_filesystem, loop_provider,
                                self.root_dir + os.sep,
                                self.filesystem_custom_parameters)
        filesystem.create_on_device(self.label)
        log.info('--> Syncing data to filesystem on %s',
                 loop_provider.get_device())
        filesystem.sync_data(Defaults.get_exclude_list_for_root_data_sync())

    def _operate_on_file(self):
        default_provider = DeviceProvider()
        filesystem = FileSystem(self.requested_filesystem, default_provider,
                                self.root_dir,
                                self.filesystem_custom_parameters)
        filesystem.create_on_file(
            self.filename, self.label,
            Defaults.get_exclude_list_for_root_data_sync())
Esempio n. 52
0
class PxeBuilder:
    """
    **Filesystem based PXE image builder.**

    :param object xml_state: instance of :class:`XMLState`
    :param str target_dir: target directory path name
    :param str root_dir: system image root directory
    :param dict custom_args: Custom processing arguments defined as hash keys:
        * signing_keys: list of package signing keys
        * xz_options: string of XZ compression parameters
    """
    def __init__(self, xml_state, target_dir, root_dir, custom_args=None):
        self.target_dir = target_dir
        self.compressed = xml_state.build_type.get_compressed()
        self.xen_server = xml_state.is_xen_server()
        self.filesystem = FileSystemBuilder(
            xml_state, target_dir, root_dir + '/'
        )
        self.system_setup = SystemSetup(
            xml_state=xml_state, root_dir=root_dir
        )

        self.boot_signing_keys = custom_args['signing_keys'] if custom_args \
            and 'signing_keys' in custom_args else None

        self.xz_options = custom_args['xz_options'] if custom_args \
            and 'xz_options' in custom_args else None

        self.boot_image_task = BootImage(
            xml_state, target_dir, root_dir,
            signing_keys=self.boot_signing_keys
        )
        self.image_name = ''.join(
            [
                target_dir, '/',
                xml_state.xml_data.get_name(),
                '.' + platform.machine(),
                '-' + xml_state.get_image_version()
            ]
        )
        self.archive_name = ''.join([self.image_name, '.tar'])
        self.checksum_name = ''.join([self.image_name, '.md5'])
        self.kernel_filename = None
        self.hypervisor_filename = None
        self.result = Result(xml_state)
        self.runtime_config = RuntimeConfig()

    def create(self):
        """
        Build a pxe image set consisting out of a boot image(initrd)
        plus its appropriate kernel files and the root filesystem
        image with a checksum. The result can be used within the kiwi
        PXE boot infrastructure

        Image types which triggers this builder are:

        * image="pxe"

        :raises KiwiPxeBootImageError: if no kernel or hipervisor is found
            in boot image tree
        :return: result

        :rtype: instance of :class:`Result`
        """
        log.info('Creating PXE root filesystem image')
        self.filesystem.create()
        os.rename(
            self.filesystem.filename, self.image_name
        )
        self.image = self.image_name
        if self.compressed:
            log.info('xz compressing root filesystem image')
            compress = Compress(self.image)
            compress.xz(self.xz_options)
            self.image = compress.compressed_filename

        log.info('Creating PXE root filesystem MD5 checksum')
        checksum = Checksum(self.image)
        checksum.md5(self.checksum_name)

        # prepare boot(initrd) root system
        log.info('Creating PXE boot image')
        self.boot_image_task.prepare()

        # export modprobe configuration to boot image
        self.system_setup.export_modprobe_setup(
            self.boot_image_task.boot_root_directory
        )

        # extract kernel from boot(initrd) root system
        kernel = Kernel(self.boot_image_task.boot_root_directory)
        kernel_data = kernel.get_kernel()
        if kernel_data:
            self.kernel_filename = ''.join(
                [
                    os.path.basename(self.image_name), '-',
                    kernel_data.version, '.kernel'
                ]
            )
            kernel.copy_kernel(
                self.target_dir, self.kernel_filename
            )
        else:
            raise KiwiPxeBootImageError(
                'No kernel in boot image tree %s found' %
                self.boot_image_task.boot_root_directory
            )

        # extract hypervisor from boot(initrd) root system
        if self.xen_server:
            kernel_data = kernel.get_xen_hypervisor()
            if kernel_data:
                self.hypervisor_filename = ''.join(
                    [os.path.basename(self.image_name), '-', kernel_data.name]
                )
                kernel.copy_xen_hypervisor(
                    self.target_dir, self.hypervisor_filename
                )
                self.result.add(
                    key='xen_hypervisor',
                    filename=self.target_dir + '/' + self.hypervisor_filename,
                    use_for_bundle=True,
                    compress=False,
                    shasum=True
                )
            else:
                raise KiwiPxeBootImageError(
                    'No hypervisor in boot image tree %s found' %
                    self.boot_image_task.boot_root_directory
                )

        # create initrd for pxe boot
        self.boot_image_task.create_initrd()

        # put results into a tarball
        if not self.xz_options:
            self.xz_options = Defaults.get_xz_compression_options()

        pxe_tarball_files = [
            self.kernel_filename,
            os.path.basename(self.boot_image_task.initrd_filename),
            os.path.basename(self.image),
            os.path.basename(self.checksum_name)
        ]
        pxe_tarball = ArchiveTar(
            self.archive_name,
            create_from_file_list=True,
            file_list=pxe_tarball_files
        )

        if self.compressed:
            self.archive_name = pxe_tarball.create(self.target_dir)
        else:
            self.archive_name = pxe_tarball.create_xz_compressed(
                self.target_dir, xz_options=self.xz_options
            )

        self.result.verify_image_size(
            self.runtime_config.get_max_size_constraint(),
            self.archive_name
        )
        # store results
        self.result.add(
            key='pxe_archive',
            filename=self.archive_name,
            use_for_bundle=True,
            compress=self.runtime_config.get_bundle_compression(
                default=False
            ),
            shasum=True
        )

        # create image root metadata
        self.result.add(
            key='image_packages',
            filename=self.system_setup.export_package_list(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        self.result.add(
            key='image_verified',
            filename=self.system_setup.export_package_verification(
                self.target_dir
            ),
            use_for_bundle=True,
            compress=False,
            shasum=False
        )
        return self.result
Esempio n. 53
0
class TestResultBundleTask(object):
    def setup(self):
        sys.argv = [
            sys.argv[0], 'result', 'bundle', '--target-dir', 'target_dir',
            '--bundle-dir', 'bundle_dir', '--id', 'Build_42'
        ]
        self.context_manager_mock = mock.Mock()
        self.file_mock = mock.Mock()
        self.enter_mock = mock.Mock()
        self.exit_mock = mock.Mock()
        self.enter_mock.return_value = self.file_mock
        setattr(self.context_manager_mock, '__enter__', self.enter_mock)
        setattr(self.context_manager_mock, '__exit__', self.exit_mock)

        self.file_mock.read.return_value = b'data'

        self.xml_state = mock.Mock()
        self.xml_state.get_image_version = mock.Mock(
            return_value='1.2.3'
        )

        self.result = Result(self.xml_state)
        self.result.add(
            key='keyname', filename='filename-1.2.3',
            use_for_bundle=True, compress=True, shasum=True
        )

        kiwi.tasks.result_bundle.Help = mock.Mock(
            return_value=mock.Mock()
        )
        self.task = ResultBundleTask()

    def __init_command_args(self):
        self.task.command_args = {}
        self.task.command_args['help'] = False
        self.task.command_args['bundle'] = False
        self.task.command_args['--target-dir'] = 'target_dir'
        self.task.command_args['--bundle-dir'] = 'bundle_dir'
        self.task.command_args['--id'] = 'Build_42'

    @raises(KiwiBundleError)
    def test_process_invalid_bundle_directory(self):
        self.__init_command_args()
        self.task.command_args['--bundle-dir'] = \
            self.task.command_args['--target-dir']
        self.task.command_args['bundle'] = True
        self.task.process()

    @patch('kiwi.tasks.result_bundle.Result.load')
    @patch('kiwi.tasks.result_bundle.Command.run')
    @patch('kiwi.tasks.result_bundle.Path.create')
    @patch('kiwi.tasks.result_bundle.Compress')
    @patch('kiwi.tasks.result_bundle.Checksum')
    @patch('os.path.exists')
    @patch('builtins.open')
    def test_process_result_bundle(
        self, mock_open, mock_exists, mock_checksum, mock_compress,
        mock_path, mock_command, mock_load
    ):
        checksum = mock.Mock()
        compress = mock.Mock()
        compress.compressed_filename = 'compressed_filename'
        mock_compress.return_value = compress
        mock_checksum.return_value = checksum
        mock_exists.return_value = False
        mock_open.return_value = self.context_manager_mock
        mock_load.return_value = self.result
        self.__init_command_args()
        self.task.command_args['bundle'] = True

        self.task.process()

        mock_load.assert_called_once_with('target_dir/kiwi.result')
        mock_path.assert_called_once_with('bundle_dir')
        mock_command.assert_called_once_with(
            [
                'cp', '-l', 'filename-1.2.3',
                'bundle_dir/filename-1.2.3-Build_42'
            ]
        )
        mock_compress.assert_called_once_with(
            'bundle_dir/filename-1.2.3-Build_42'
        )
        mock_checksum.assert_called_once_with(
            compress.compressed_filename
        )
        checksum.sha256.assert_called_once_with()

    def test_process_result_bundle_help(self):
        self.__init_command_args()
        self.task.command_args['help'] = True
        self.task.command_args['bundle'] = True
        self.task.process()
        self.task.manual.show.assert_called_once_with(
            'kiwi::result::bundle'
        )