def __init__( self, xml_state: XMLState, target_dir: str, root_dir: str, custom_args: Dict = None ): self.custom_args = custom_args or {} self.root_dir = root_dir self.target_dir = target_dir self.bundle_format = xml_state.get_build_type_bundle_format() self.container_config = xml_state.get_container_config() self.requested_container_type = xml_state.get_build_type_name() self.base_image = None self.base_image_md5 = None self.ensure_empty_tmpdirs = True self.container_config['xz_options'] = \ self.custom_args.get('xz_options') self.container_config['metadata_path'] = \ xml_state.build_type.get_metadata_path() if xml_state.get_derived_from_image_uri(): # The base image is expected to be unpacked by the kiwi # prepare step and stored inside of the root_dir/image directory. # In addition a md5 file of the image is expected too self.base_image = Defaults.get_imported_root_image( self.root_dir ) self.base_image_md5 = ''.join([self.base_image, '.md5']) if not os.path.exists(self.base_image): raise KiwiContainerBuilderError( 'Unpacked Base image {0} not found'.format( self.base_image ) ) if not os.path.exists(self.base_image_md5): raise KiwiContainerBuilderError( 'Base image MD5 sum {0} not found at'.format( self.base_image_md5 ) ) if xml_state.build_type.get_ensure_empty_tmpdirs() is False: self.ensure_empty_tmpdirs = False self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.filename = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + Defaults.get_platform_name(), '-' + xml_state.get_image_version(), '.', self.requested_container_type, '.tar' if self.requested_container_type != 'appx' else '' ] ) self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.target_dir = target_dir self.compressed = xml_state.build_type.get_compressed() self.xen_server = xml_state.is_xen_server() self.filesystem = FileSystemBuilder(xml_state, target_dir, root_dir + '/') self.system_setup = SystemSetup(xml_state=xml_state, root_dir=root_dir) self.boot_signing_keys = custom_args['signing_keys'] if custom_args \ and 'signing_keys' in custom_args else None self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.boot_image_task = BootImage(xml_state, target_dir, signing_keys=self.boot_signing_keys) self.image_name = ''.join([ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version() ]) self.archive_name = ''.join([self.image_name, '.tar.xz']) self.checksum_name = ''.join([self.image_name, '.md5']) self.kernel_filename = None self.hypervisor_filename = None self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def __init__(self, xml_state, target_dir, root_dir): self.label = None self.root_uuid = None self.root_dir = root_dir self.target_dir = target_dir self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type == 'pxe': self.requested_filesystem = xml_state.build_type.get_filesystem() else: self.requested_filesystem = self.requested_image_type if not self.requested_filesystem: raise KiwiFileSystemSetupError( 'No filesystem configured in %s type' % self.requested_image_type) self.filesystem_custom_parameters = { 'mount_options': xml_state.get_fs_mount_option_list(), 'create_options': xml_state.get_fs_create_option_list() } self.system_setup = SystemSetup(xml_state=xml_state, root_dir=self.root_dir) self.filename = ''.join([ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.', self.requested_filesystem ]) self.blocksize = xml_state.build_type.get_target_blocksize() self.filesystem_setup = FileSystemSetup(xml_state, root_dir) self.filesystems_no_device_node = ['squashfs'] self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
class TestRuntimeConfig(object): def setup(self): with patch.dict('os.environ', {'HOME': '../data'}): self.runtime_config = RuntimeConfig() @raises(KiwiRuntimeConfigFormatError) def test_invalid_yaml_format(self): self.runtime_config.config_data = {'xz': None} self.runtime_config.get_xz_options() def test_get_xz_options(self): assert self.runtime_config.get_xz_options() == ['-a', '-b', 'xxx'] def test_is_obs_public(self): assert self.runtime_config.is_obs_public() is True def test_is_obs_public_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.is_obs_public() is True def test_get_obs_download_server_url(self): assert self.runtime_config.get_obs_download_server_url() == \ 'http://example.com' def test_get_obs_download_server_url_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.get_obs_download_server_url() == \ Defaults.get_obs_download_server_url()
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.media_dir = None self.live_container_dir = None self.arch = Defaults.get_platform_name() self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.live_type = xml_state.build_type.get_flags() self.volume_id = xml_state.build_type.get_volid() or \ Defaults.get_volume_id() self.mbrid = SystemIdentifier() self.mbrid.calculate_id() self.publisher = xml_state.build_type.get_publisher() or \ Defaults.get_publisher() self.custom_args = custom_args if not self.live_type: self.live_type = Defaults.get_default_live_iso_type() self.boot_image = BootImageDracut(xml_state, target_dir, self.root_dir) self.firmware = FirmWare(xml_state) self.system_setup = SystemSetup(xml_state=xml_state, root_dir=self.root_dir) self.isoname = ''.join([ target_dir, '/', xml_state.xml_data.get_name(), '.' + Defaults.get_platform_name(), '-' + xml_state.get_image_version(), '.iso' ]) self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def __init__(self, should_perform_task_setup=True): from ..logger import log self.cli = Cli() # initialize runtime checker self.runtime_checker = None # initialize runtime configuration self.runtime_config = RuntimeConfig() # help requested self.cli.show_and_exit_on_help_request() # load/import task module self.task = self.cli.load_command() # get command specific args self.command_args = self.cli.get_command_args() # get global args self.global_args = self.cli.get_global_args() # initialize generic runtime check dicts self.checks_before_command_args = { 'check_minimal_required_preferences': [], 'check_efi_mode_for_disk_overlay_correctly_setup': [], 'check_boot_description_exists': [], 'check_consistent_kernel_in_boot_and_system_image': [], 'check_container_tool_chain_installed': [], 'check_volume_setup_defines_multiple_fullsize_volumes': [], 'check_volume_setup_has_no_root_definition': [], 'check_volume_label_used_with_lvm': [], 'check_xen_uniquely_setup_as_server_or_guest': [], 'check_mediacheck_installed': [], 'check_dracut_module_for_live_iso_in_package_list': [], 'check_dracut_module_for_disk_overlay_in_package_list': [], 'check_dracut_module_for_disk_oem_in_package_list': [], 'check_dracut_module_for_oem_install_in_package_list': [], 'check_architecture_supports_iso_firmware_setup': [] } self.checks_after_command_args = { 'check_repositories_configured': [], 'check_image_include_repos_publicly_resolvable': [] } if should_perform_task_setup: # set log level if self.global_args['--debug']: log.setLogLevel(logging.DEBUG) else: log.setLogLevel(logging.INFO) # set log file if self.global_args['--logfile']: log.set_logfile(self.global_args['--logfile']) if self.global_args['--color-output']: log.set_color_format()
def __new__(self, container_tag, container_dir=None): runtime_config = RuntimeConfig() tool_name = runtime_config.get_oci_archive_tool() if tool_name == 'umoci': return OCIUmoci(container_tag, container_dir) else: raise KiwiOCIArchiveToolError( 'No support for {0} tool available'.format(tool_name))
def test__get_attribute_raises_on_invalid_structure(self, mock_Cli): cli = Mock() cli.get_global_args.return_value = {} mock_Cli.return_value = cli with patch.dict('os.environ', {'HOME': '../data/kiwi_config/broken'}): runtime_config = RuntimeConfig(reread=True) with raises(KiwiRuntimeConfigFormatError): runtime_config._get_attribute('foo', 'bar')
def setup(self): with patch.dict('os.environ', {'HOME': '../data'}): self.runtime_config = RuntimeConfig() # pretend that none of the runtime config files exist, even if they do # (e.g. the system wide config file in /etc/kiwi.yml) # => this will give us the defaults with patch('os.path.exists', return_value=False): self.default_runtime_config = RuntimeConfig()
def test_config_sections_other_settings(self, mock_Cli): cli = Mock() cli.get_global_args.return_value = {} mock_Cli.return_value = cli with patch.dict('os.environ', {'HOME': '../data/kiwi_config/other'}): runtime_config = RuntimeConfig(reread=True) assert runtime_config.get_container_compression() == 'xz' assert runtime_config.get_package_changes() is True
def test_reading_system_wide_config_file(self, mock_yaml, mock_exists): exists_call_results = [True, False] def os_path_exists(config): return exists_call_results.pop() mock_exists.side_effect = os_path_exists with patch_open as mock_open: self.runtime_config = RuntimeConfig() mock_open.assert_called_once_with('/etc/kiwi.yml', 'r')
def __init__(self, uri: str, repo_type: str = 'rpm-md', source_type: str = ''): """ Manage kiwi source URIs and allow transformation into standard URLs :param str uri: URI, remote, local or metalink repository location The resource type as part of the URI can be set to one of: * http: * https: * ftp: * obs: * dir: * file: * obsrepositories: * this: The special this:// type resolve to the image description directory. The code to resolve this is not part of the Uri class because it has no state information about the image description directory. Therefore the resolving of the this:// path happens on construction of an XMLState object as part of the resolve_this_path() method. The method resolves the path into a native dir:// URI which can be properly handled here. :param str repo_type: repository type name, defaults to 'rpm-md' and is only effectively used when building inside of the open build service which maps local repositories to a specific environment :param str source_type: specify source type if the provided URI is a service. Currently only the metalink source type is handled """ self.runtime_config = RuntimeConfig() if source_type == 'metalink': uri = self._resolve_metalink_uri(uri) self.repo_type = repo_type self.uri = uri if not uri.startswith(os.sep) else ''.join( [Defaults.get_default_uri_type(), uri]) self.remote_uri_types = { 'http': True, 'https': True, 'ftp': True, 'obs': True } self.local_uri_type = { 'dir': True, 'file': True, 'obsrepositories': True }
def __init__(self, xml_state, root_dir): self.runtime_config = RuntimeConfig() self.arch = Defaults.get_platform_name() self.xml_state = xml_state self.description_dir = \ xml_state.xml_data.description_dir self.derived_description_dir = \ xml_state.xml_data.derived_description_dir self.root_dir = root_dir self._preferences_lookup() self._oemconfig_lookup()
def __init__(self, xml_state, root_dir, target_dir, custom_args=None): self.xml_state = xml_state self.root_dir = root_dir self.arch = platform.machine() self.target_dir = target_dir self.custom_args = {} self.temp_image_dir = None self.image_format = None self.diskname = self.get_target_file_path_for_format('raw') self.runtime_config = RuntimeConfig() self.post_init(custom_args)
def new(source_dir): name_map = {'xorriso': 'XorrIso', 'cdrtools': 'CdrTools'} runtime_config = RuntimeConfig() tool = runtime_config.get_iso_tool_category() try: iso_tool = importlib.import_module( 'kiwi.iso_tools.{}'.format(tool)) module_name = 'IsoTools{}'.format(name_map[tool]) return iso_tool.__dict__[module_name](source_dir) except Exception: raise KiwiIsoToolError( 'No support for {} tool available'.format(tool))
def new(): name_map = {'umoci': 'Umoci', 'buildah': 'Buildah'} runtime_config = RuntimeConfig() tool_name = runtime_config.get_oci_archive_tool() try: oci_tool = importlib.import_module( 'kiwi.oci_tools.{}'.format(tool_name)) module_name = 'OCI{}'.format(name_map[tool_name]) return oci_tool.__dict__[module_name]() except Exception: raise KiwiOCIArchiveToolError( 'No support for {0} tool available'.format(tool_name))
def test_config_sections_invalid(self): with patch.dict('os.environ', {'HOME': '../data/kiwi_config/invalid'}): runtime_config = RuntimeConfig(reread=True) with self._caplog.at_level(logging.WARNING): assert runtime_config.get_container_compression() is True assert 'Skipping invalid container compression: foo' in \ self._caplog.text with self._caplog.at_level(logging.WARNING): assert runtime_config.get_iso_tool_category() == 'xorriso' assert 'Skipping invalid iso tool category: foo' in \ self._caplog.text
def check_container_tool_chain_installed(self) -> None: """ When creating container images the specific tools are used in order to import and export OCI or Docker compatible images. This check searches for those tools to be installed in the build system and fails if it can't find them """ message_tool_not_found = dedent('''\n Required tool {name} not found in caller environment Creation of OCI or Docker images requires the tools {name} and skopeo to be installed on the build system. For SUSE based systems you can find the tools at: http://download.opensuse.org/repositories/Virtualization:/containers ''') message_version_unsupported = dedent('''\n {name} tool found with unknown version ''') message_unknown_tool = dedent('''\n Unknown tool: {0}. Please configure KIWI with an appropriate value (umoci or buildah). Consider this runtime configuration file syntax (/etc/kiwi.yml): oci: - archive_tool: umoci | buildah ''') expected_version = (0, 1, 0) if self.xml_state.get_build_type_name() in ['docker', 'oci']: runtime_config = RuntimeConfig() tool_name = runtime_config.get_oci_archive_tool() if tool_name == 'buildah': oci_tools = ['buildah', 'skopeo'] elif tool_name == 'umoci': oci_tools = ['umoci', 'skopeo'] else: raise KiwiRuntimeError(message_unknown_tool.format(tool_name)) for tool in oci_tools: if not Path.which(filename=tool, access_mode=os.X_OK): raise KiwiRuntimeError( message_tool_not_found.format(name=tool) ) elif not CommandCapabilities.check_version( tool, expected_version, raise_on_error=False ): raise KiwiRuntimeError( message_version_unsupported.format(name=tool) ) self._check_multitag_support()
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.requested_archive_type = xml_state.get_build_type_name() self.result = Result(xml_state) self.system_setup = SystemSetup(xml_state=xml_state, root_dir=self.root_dir) self.filename = self._target_file_for('tar.xz') self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.runtime_config = RuntimeConfig()
def __init__( self, xml_state: XMLState, root_dir: str, target_dir: str, custom_args: dict = None ): self.xml_state = xml_state self.root_dir = root_dir self.arch = Defaults.get_platform_name() self.target_dir = target_dir self.temp_image_dir: str = '' self.image_format: str = '' self.diskname = self.get_target_file_path_for_format('raw') self.runtime_config = RuntimeConfig() self.post_init(custom_args or {})
def test_config_sections_invalid(self, mock_Cli): cli = Mock() cli.get_global_args.return_value = {} mock_Cli.return_value = cli with patch.dict('os.environ', {'HOME': '../data/kiwi_config/invalid'}): runtime_config = RuntimeConfig(reread=True) with self._caplog.at_level(logging.WARNING): assert runtime_config.get_container_compression() == 'xz' assert 'Skipping invalid container compression: foo' in \ self._caplog.text with self._caplog.at_level(logging.WARNING): assert runtime_config.get_iso_tool_category() == 'xorriso' assert 'Skipping invalid iso tool category: foo' in \ self._caplog.text
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.target_dir = target_dir self.compressed = xml_state.build_type.get_compressed() self.xen_server = xml_state.is_xen_server() self.pxedeploy = xml_state.get_build_type_pxedeploy_section() self.filesystem = FileSystemBuilder( xml_state, target_dir, root_dir + '/' ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=root_dir ) self.boot_signing_keys = custom_args['signing_keys'] if custom_args \ and 'signing_keys' in custom_args else None self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.boot_image_task = BootImage( xml_state, target_dir, signing_keys=self.boot_signing_keys, custom_args=custom_args ) self.image_name = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version() ] ) self.archive_name = ''.join([self.image_name, '.tar.xz']) self.kernel_filename = None self.hypervisor_filename = None self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def sign_verification_metadata(self) -> None: """ Create an openssl based signature from the metadata block and attach it at the end of the block. This method requires access to a private key for signing. The path to the private key is read from the kiwi runtime config file from the following section: credentials: - verification_metadata_signing_key_file: /path/to/pkey """ if self.verification_metadata_file: runtime_config = RuntimeConfig() signing_key_file = runtime_config.\ get_credentials_verification_metadata_signing_key_file() if not signing_key_file: raise KiwiCredentialsError( '{0} not configured in runtime config'.format( 'verification_metadata_signing_key_file')) signature_file = Temporary().new_file() Command.run([ 'openssl', 'dgst', '-sha256', '-sigopt', 'rsa_padding_mode:pss', '-sigopt', 'rsa_pss_saltlen:-1', '-sigopt', 'rsa_mgf1_md:sha256', '-sign', signing_key_file, '-out', signature_file.name, self.verification_metadata_file.name ]) with open(signature_file.name, 'rb') as sig_fd: signature = sig_fd.read() with open(self.verification_metadata_file.name, 'ab') as meta: meta.write(signature)
def test_init_raises_custom_config_file_not_found(self, mock_Cli): cli = Mock() cli.get_global_args.return_value = {} mock_Cli.return_value = cli cli.get_global_args.return_value = {'--config': '../data/kiwi.yml'} with patch('os.path.isfile', return_value=False): with raises(KiwiRuntimeConfigFileError): RuntimeConfig(reread=True)
def __init__( self, xml_state: XMLState, target_dir: str, root_dir: str, custom_args: Dict = None ): self.target_dir = target_dir self.compressed = xml_state.build_type.get_compressed() self.xen_server = xml_state.is_xen_server() self.custom_cmdline = xml_state.build_type.get_kernelcmdline() self.filesystem = FileSystemBuilder( xml_state, target_dir, root_dir + '/' ) if xml_state.build_type.get_filesystem() else None self.system_setup = SystemSetup( xml_state=xml_state, root_dir=root_dir ) self.initrd_system = xml_state.get_initrd_system() self.boot_signing_keys = custom_args['signing_keys'] if custom_args \ and 'signing_keys' in custom_args else None self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.boot_image_task = BootImage.new( xml_state, target_dir, root_dir, signing_keys=self.boot_signing_keys ) self.bundle_format = xml_state.get_build_type_bundle_format() self.image_name = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + Defaults.get_platform_name(), '-' + xml_state.get_image_version() ] ) self.image: str = '' self.append_file = ''.join([self.image_name, '.append']) self.archive_name = ''.join([self.image_name, '.tar']) self.checksum_name = ''.join([self.image_name, '.md5']) self.kernel_filename: str = '' self.hypervisor_filename: str = '' self.result = Result(xml_state) self.runtime_config = RuntimeConfig() if not self.boot_image_task.has_initrd_support(): log.warning('Building without initrd support !')
def __init__( self, xml_state: XMLState, target_dir: str, root_dir: str, custom_args: Dict = None ): self.label = None self.root_uuid = '' self.root_dir = root_dir self.target_dir = target_dir self.bundle_format = xml_state.get_build_type_bundle_format() self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type in Defaults.get_kis_image_types(): self.requested_filesystem = xml_state.build_type.get_filesystem() else: self.requested_filesystem = self.requested_image_type if not self.requested_filesystem: raise KiwiFileSystemSetupError( 'No filesystem configured in %s type' % self.requested_image_type ) self.filesystem_custom_parameters = { 'mount_options': xml_state.get_fs_mount_option_list(), 'create_options': xml_state.get_fs_create_option_list() } if self.requested_filesystem == 'squashfs': self.filesystem_custom_parameters['compression'] = \ xml_state.build_type.get_squashfscompression() self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.filename = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + Defaults.get_platform_name(), '-' + xml_state.get_image_version(), '.', self.requested_filesystem ] ) self.blocksize = xml_state.build_type.get_target_blocksize() self.filesystem_setup = FileSystemSetup(xml_state, root_dir) self.filesystems_no_device_node = [ 'squashfs' ] self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def __init__(self, root_dir, transport, custom_args=None): self.root_dir = root_dir self.archive_transport = transport if custom_args: self.oci_config = custom_args else: self.oci_config = {} self.runtime_config = RuntimeConfig() # for builds inside the buildservice we include a reference to the # specific build. Thus disturl label only exists inside the # buildservice. if Defaults.is_buildservice_worker(): bs_label = 'org.openbuildservice.disturl' # Do not label anything if the build service label is # already present if 'labels' not in self.oci_config or \ bs_label not in self.oci_config['labels']: self._append_buildservice_disturl_label() if 'container_name' not in self.oci_config: log.info( 'No container configuration provided, ' 'using default container name "kiwi-container:latest"' ) self.oci_config['container_name'] = \ Defaults.get_default_container_name() self.oci_config['container_tag'] = \ Defaults.get_default_container_tag() if 'container_tag' not in self.oci_config: self.oci_config['container_tag'] = \ Defaults.get_default_container_tag() if 'entry_command' not in self.oci_config and \ 'entry_subcommand' not in self.oci_config: self.oci_config['entry_subcommand'] = \ Defaults.get_default_container_subcommand() if 'history' not in self.oci_config: self.oci_config['history'] = {} if 'created_by' not in self.oci_config['history']: self.oci_config['history']['created_by'] = \ Defaults.get_default_container_created_by()
def __init__(self, uri: str, repo_type: str = 'rpm-md'): self.runtime_config = RuntimeConfig() self.repo_type = repo_type self.uri = uri if not uri.startswith(os.sep) else ''.join( [Defaults.get_default_uri_type(), uri] ) self.remote_uri_types = { 'http': True, 'https': True, 'ftp': True, 'obs': True } self.local_uri_type = { 'dir': True, 'file': True, 'obsrepositories': True }
def __init__(self, uri, repo_type=None): self.runtime_config = RuntimeConfig() self.repo_type = repo_type self.uri = uri self.mount_stack = [] self.remote_uri_types = { 'http': True, 'https': True, 'ftp': True, 'obs': True } self.local_uri_type = { 'iso': True, 'dir': True, 'file': True, 'obsrepositories': True }
def __init__(self, uri: str, repo_type: Dict = None): self.runtime_config = RuntimeConfig() self.repo_type = repo_type self.uri = uri if not uri.startswith(os.sep) else ''.join( [Defaults.get_default_uri_type(), uri]) self.mount_stack: List[str] = [] self.remote_uri_types = { 'http': True, 'https': True, 'ftp': True, 'obs': True } self.local_uri_type = { 'iso': True, 'dir': True, 'file': True, 'obsrepositories': True }
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.media_dir = None self.live_container_dir = None self.arch = platform.machine() if self.arch == 'i686' or self.arch == 'i586': self.arch = 'ix86' self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.live_type = xml_state.build_type.get_flags() self.hybrid = xml_state.build_type.get_hybrid() self.volume_id = xml_state.build_type.get_volid() or \ Defaults.get_volume_id() self.mbrid = SystemIdentifier() self.mbrid.calculate_id() self.filesystem_custom_parameters = { 'mount_options': xml_state.get_fs_mount_option_list() } if not self.live_type: self.live_type = Defaults.get_default_live_iso_type() self.boot_image = BootImageDracut( xml_state, target_dir, self.root_dir ) self.firmware = FirmWare( xml_state ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.isoname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.iso' ] ) self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.requested_archive_type = xml_state.get_build_type_name() self.result = Result(xml_state) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.filename = self._target_file_for('tar.xz') self.checksum = self._target_file_for('md5') self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.runtime_config = RuntimeConfig()
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.root_dir = root_dir self.target_dir = target_dir self.container_config = xml_state.get_container_config() self.requested_container_type = xml_state.get_build_type_name() self.base_image = None self.base_image_md5 = None self.container_config['xz_options'] = custom_args['xz_options'] \ if custom_args and 'xz_options' in custom_args else None if xml_state.get_derived_from_image_uri(): # The base image is expected to be unpacked by the kiwi # prepare step and stored inside of the root_dir/image directory. # In addition a md5 file of the image is expected too self.base_image = Defaults.get_imported_root_image( self.root_dir ) self.base_image_md5 = ''.join([self.base_image, '.md5']) if not os.path.exists(self.base_image): raise KiwiContainerBuilderError( 'Unpacked Base image {0} not found'.format( self.base_image ) ) if not os.path.exists(self.base_image_md5): raise KiwiContainerBuilderError( 'Base image MD5 sum {0} not found at'.format( self.base_image_md5 ) ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.filename = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.', self.requested_container_type, '.tar' ] ) self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def __init__(self, uri, repo_type=None): self.runtime_config = RuntimeConfig() self.repo_type = repo_type self.uri = uri self.mount_stack = [] self.remote_uri_types = { 'http': True, 'https': True, 'ftp': True, 'obs': True } self.local_uri_type = { 'iso': True, 'dir': True, 'file': True, 'obsrepositories': True }
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.media_dir = None self.live_container_dir = None self.arch = platform.machine() if self.arch == 'i686' or self.arch == 'i586': self.arch = 'ix86' self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.live_type = xml_state.build_type.get_flags() self.volume_id = xml_state.build_type.get_volid() or \ Defaults.get_volume_id() self.mbrid = SystemIdentifier() self.mbrid.calculate_id() self.filesystem_custom_parameters = { 'mount_options': xml_state.get_fs_mount_option_list() } self.publisher = xml_state.build_type.get_publisher() or \ Defaults.get_publisher() if not self.live_type: self.live_type = Defaults.get_default_live_iso_type() self.boot_image = BootImageDracut( xml_state, target_dir, self.root_dir ) self.firmware = FirmWare( xml_state ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.isoname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.iso' ] ) self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def __init__(self, xml_state, target_dir, root_dir): self.label = None self.root_dir = root_dir self.target_dir = target_dir self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type == 'pxe': self.requested_filesystem = xml_state.build_type.get_filesystem() else: self.requested_filesystem = self.requested_image_type if not self.requested_filesystem: raise KiwiFileSystemSetupError( 'No filesystem configured in %s type' % self.requested_image_type ) self.filesystem_custom_parameters = { 'mount_options': xml_state.get_fs_mount_option_list() } self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.filename = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.', self.requested_filesystem ] ) self.blocksize = xml_state.build_type.get_target_blocksize() self.filesystem_setup = FileSystemSetup(xml_state, root_dir) self.filesystems_no_device_node = [ 'squashfs' ] self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
class LiveImageBuilder(object): """ **Live image builder** :param object xml_state: instance of :class:`XMLState` :param str target_dir: target directory path name :param str root_dir: root directory path name :param dict custom_args: Custom processing arguments defined as hash keys: * xz_options: string of XZ compression parameters """ def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.media_dir = None self.live_container_dir = None self.arch = platform.machine() if self.arch == 'i686' or self.arch == 'i586': self.arch = 'ix86' self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.live_type = xml_state.build_type.get_flags() self.volume_id = xml_state.build_type.get_volid() or \ Defaults.get_volume_id() self.mbrid = SystemIdentifier() self.mbrid.calculate_id() self.filesystem_custom_parameters = { 'mount_options': xml_state.get_fs_mount_option_list() } self.publisher = xml_state.build_type.get_publisher() or \ Defaults.get_publisher() if not self.live_type: self.live_type = Defaults.get_default_live_iso_type() self.boot_image = BootImageDracut( xml_state, target_dir, self.root_dir ) self.firmware = FirmWare( xml_state ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.isoname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.iso' ] ) self.result = Result(xml_state) self.runtime_config = RuntimeConfig() def create(self): """ Build a bootable hybrid live ISO image Image types which triggers this builder are: * image="iso" :raises KiwiLiveBootImageError: if no kernel or hipervisor is found in boot image tree :return: result :rtype: instance of :class:`Result` """ # media dir to store CD contents self.media_dir = mkdtemp( prefix='live-media.', dir=self.target_dir ) # unpack cdroot user files to media dir self.system_setup.import_cdroot_files(self.media_dir) rootsize = SystemSize(self.media_dir) # custom iso metadata log.info('Using following live ISO metadata:') log.info('--> Application id: {0}'.format(self.mbrid.get_id())) log.info('--> Publisher: {0}'.format(Defaults.get_publisher())) log.info('--> Volume id: {0}'.format(self.volume_id)) custom_iso_args = { 'meta_data': { 'publisher': self.publisher, 'preparer': Defaults.get_preparer(), 'volume_id': self.volume_id, 'mbr_id': self.mbrid.get_id(), 'efi_mode': self.firmware.efi_mode() } } # pack system into live boot structure as expected by dracut log.info( 'Packing system into dracut live ISO type: {0}'.format( self.live_type ) ) root_filesystem = Defaults.get_default_live_iso_root_filesystem() filesystem_custom_parameters = { 'mount_options': self.xml_state.get_fs_mount_option_list() } filesystem_setup = FileSystemSetup( self.xml_state, self.root_dir ) root_image = NamedTemporaryFile() loop_provider = LoopDevice( root_image.name, filesystem_setup.get_size_mbytes(root_filesystem), self.xml_state.build_type.get_target_blocksize() ) loop_provider.create() live_filesystem = FileSystem( name=root_filesystem, device_provider=loop_provider, root_dir=self.root_dir + os.sep, custom_args=filesystem_custom_parameters ) live_filesystem.create_on_device() log.info( '--> Syncing data to {0} root image'.format(root_filesystem) ) live_filesystem.sync_data( Defaults.get_exclude_list_for_root_data_sync() ) log.info('--> Creating squashfs container for root image') self.live_container_dir = mkdtemp( prefix='live-container.', dir=self.target_dir ) Path.create(self.live_container_dir + '/LiveOS') shutil.copy( root_image.name, self.live_container_dir + '/LiveOS/rootfs.img' ) live_container_image = FileSystem( name='squashfs', device_provider=None, root_dir=self.live_container_dir ) container_image = NamedTemporaryFile() live_container_image.create_on_file( container_image.name ) Path.create(self.media_dir + '/LiveOS') shutil.copy( container_image.name, self.media_dir + '/LiveOS/squashfs.img' ) # setup bootloader config to boot the ISO via isolinux log.info('Setting up isolinux bootloader configuration') bootloader_config_isolinux = BootLoaderConfig( 'isolinux', self.xml_state, self.media_dir ) bootloader_config_isolinux.setup_live_boot_images( mbrid=None, lookup_path=self.boot_image.boot_root_directory ) bootloader_config_isolinux.setup_live_image_config( mbrid=None ) bootloader_config_isolinux.write() # setup bootloader config to boot the ISO via EFI if self.firmware.efi_mode(): log.info('Setting up EFI grub bootloader configuration') bootloader_config_grub = BootLoaderConfig( 'grub2', self.xml_state, self.media_dir, { 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) } ) bootloader_config_grub.setup_live_boot_images( mbrid=self.mbrid, lookup_path=self.root_dir ) bootloader_config_grub.setup_live_image_config( mbrid=self.mbrid ) bootloader_config_grub.write() # call custom editbootconfig script if present self.system_setup.call_edit_boot_config_script( filesystem='iso:{0}'.format(self.media_dir), boot_part_id=1, working_directory=self.root_dir ) # prepare dracut initrd call self.boot_image.prepare() # create dracut initrd for live image log.info('Creating live ISO boot image') self._create_dracut_live_iso_config() self.boot_image.create_initrd(self.mbrid) # setup kernel file(s) and initrd in ISO boot layout log.info('Setting up kernel file(s) and boot image in ISO boot layout') self._setup_live_iso_kernel_and_initrd() # calculate size and decide if we need UDF if rootsize.accumulate_mbyte_file_sizes() > 4096: log.info('ISO exceeds 4G size, using UDF filesystem') custom_iso_args['meta_data']['udf'] = True # create iso filesystem from media_dir log.info('Creating live ISO image') iso_image = FileSystemIsoFs( device_provider=None, root_dir=self.media_dir, custom_args=custom_iso_args ) iso_image.create_on_file(self.isoname) # include metadata for checkmedia tool if self.xml_state.build_type.get_mediacheck() is True: Iso.set_media_tag(self.isoname) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.isoname ) self.result.add( key='live_image', filename=self.isoname, use_for_bundle=True, compress=False, shasum=True ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result def _create_dracut_live_iso_config(self): live_config_file = self.root_dir + '/etc/dracut.conf.d/02-livecd.conf' omit_modules = [ 'kiwi-dump', 'kiwi-overlay', 'kiwi-repart', 'kiwi-lib', 'multipath' ] live_config = [ 'add_dracutmodules+=" {0} pollcdrom "'.format( Defaults.get_live_dracut_module_from_flag(self.live_type) ), 'omit_dracutmodules+=" {0} "'.format(' '.join(omit_modules)), 'hostonly="no"', 'dracut_rescue_image="no"' ] with open(live_config_file, 'w') as config: for entry in live_config: config.write(entry + os.linesep) def _setup_live_iso_kernel_and_initrd(self): """ Copy kernel and initrd from the root tree into the iso boot structure """ boot_path = ''.join( [self.media_dir, '/boot/', self.arch, '/loader'] ) Path.create(boot_path) # Move kernel files to iso filesystem structure kernel = Kernel(self.boot_image.boot_root_directory) if kernel.get_kernel(): kernel.copy_kernel(boot_path, '/linux') else: raise KiwiLiveBootImageError( 'No kernel in boot image tree {0} found'.format( self.boot_image.boot_root_directory ) ) if self.xml_state.is_xen_server(): if kernel.get_xen_hypervisor(): kernel.copy_xen_hypervisor(boot_path, '/xen.gz') else: raise KiwiLiveBootImageError( 'No hypervisor in boot image tree {0} found'.format( self.boot_image.boot_root_directory ) ) # Move initrd to iso filesystem structure if os.path.exists(self.boot_image.initrd_filename): shutil.move( self.boot_image.initrd_filename, boot_path + '/initrd' ) else: raise KiwiLiveBootImageError( 'No boot image {0} in boot image tree {1} found'.format( self.boot_image.initrd_filename, self.boot_image.boot_root_directory ) ) def __del__(self): if self.media_dir or self.live_container_dir: log.info( 'Cleaning up {0} instance'.format(type(self).__name__) ) if self.media_dir: Path.wipe(self.media_dir) if self.live_container_dir: Path.wipe(self.live_container_dir)
def setup(self): with patch.dict('os.environ', {'HOME': '../data'}): self.runtime_config = RuntimeConfig()
def test_is_obs_public_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.is_obs_public() is True
class PxeBuilder(object): """ **Filesystem based PXE image builder.** :param object xml_state: instance of :class:`XMLState` :param str target_dir: target directory path name :param str root_dir: system image root directory :param dict custom_args: Custom processing arguments defined as hash keys: * signing_keys: list of package signing keys * xz_options: string of XZ compression parameters """ def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.target_dir = target_dir self.compressed = xml_state.build_type.get_compressed() self.xen_server = xml_state.is_xen_server() self.pxedeploy = xml_state.get_build_type_pxedeploy_section() self.filesystem = FileSystemBuilder( xml_state, target_dir, root_dir + '/' ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=root_dir ) self.boot_signing_keys = custom_args['signing_keys'] if custom_args \ and 'signing_keys' in custom_args else None self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.boot_image_task = BootImage( xml_state, target_dir, signing_keys=self.boot_signing_keys, custom_args=custom_args ) self.image_name = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version() ] ) self.archive_name = ''.join([self.image_name, '.tar.xz']) self.kernel_filename = None self.hypervisor_filename = None self.result = Result(xml_state) self.runtime_config = RuntimeConfig() def create(self): """ Build a pxe image set consisting out of a boot image(initrd) plus its appropriate kernel files and the root filesystem image with a checksum. The result can be used within the kiwi PXE boot infrastructure Image types which triggers this builder are: * image="pxe" :raises KiwiPxeBootImageError: if no kernel or hipervisor is found in boot image tree :return: result :rtype: instance of :class:`Result` """ log.info('Creating PXE root filesystem image') self.filesystem.create() os.rename( self.filesystem.filename, self.image_name ) self.image = self.image_name if self.compressed: log.info('xz compressing root filesystem image') compress = Compress(self.image) compress.xz(self.xz_options) self.image = compress.compressed_filename log.info('Creating PXE root filesystem MD5 checksum') self.filesystem_checksum = ''.join([self.image, '.md5']) checksum = Checksum(self.image) checksum.md5(self.filesystem_checksum) # prepare boot(initrd) root system log.info('Creating PXE boot image') self.boot_image_task.prepare() # export modprobe configuration to boot image self.system_setup.export_modprobe_setup( self.boot_image_task.boot_root_directory ) # extract kernel from boot(initrd) root system kernel = Kernel(self.boot_image_task.boot_root_directory) kernel_data = kernel.get_kernel() if kernel_data: self.kernel_filename = ''.join( [ os.path.basename(self.image_name), '-', kernel_data.version, '.kernel' ] ) kernel.copy_kernel( self.target_dir, self.kernel_filename ) else: raise KiwiPxeBootImageError( 'No kernel in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # extract hypervisor from boot(initrd) root system if self.xen_server: kernel_data = kernel.get_xen_hypervisor() if kernel_data: self.hypervisor_filename = ''.join( [os.path.basename(self.image_name), '-', kernel_data.name] ) kernel.copy_xen_hypervisor( self.target_dir, self.hypervisor_filename ) self.result.add( key='xen_hypervisor', filename=self.target_dir + '/' + self.hypervisor_filename, use_for_bundle=True, compress=False, shasum=True ) else: raise KiwiPxeBootImageError( 'No hypervisor in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # create initrd for pxe boot self.boot_image_task.create_initrd() # put results into a tarball if not self.xz_options: self.xz_options = Defaults.get_xz_compression_options() bash_command = [ 'tar', '-C', self.target_dir, '-c', '--to-stdout' ] + [ self.kernel_filename, os.path.basename(self.boot_image_task.initrd_filename), os.path.basename(self.image), os.path.basename(self.filesystem_checksum) ] + [ '|', 'xz', '-f' ] + self.xz_options + [ '>', self.archive_name ] Command.run(['bash', '-c', ' '.join(bash_command)]) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.archive_name ) # store results self.result.add( key='pxe_archive', filename=self.archive_name, use_for_bundle=True, compress=False, shasum=True ) # create image root metadata self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) if self.pxedeploy: log.warning( 'Creation of client config file from pxedeploy not implemented' ) return self.result
def test_get_container_compression_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.get_container_compression() == 'xz'
def test_get_iso_tool_category_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.get_iso_tool_category() == 'xorriso'
class TestRuntimeConfig(object): def setup(self): with patch.dict('os.environ', {'HOME': '../data'}): self.runtime_config = RuntimeConfig() @raises(KiwiRuntimeConfigFormatError) def test_invalid_yaml_format(self): self.runtime_config.config_data = {'xz': None} self.runtime_config.get_xz_options() def test_get_xz_options(self): assert self.runtime_config.get_xz_options() == ['-a', '-b', 'xxx'] def test_is_obs_public(self): assert self.runtime_config.is_obs_public() is True def test_is_obs_public_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.is_obs_public() is True def test_get_obs_download_server_url(self): assert self.runtime_config.get_obs_download_server_url() == \ 'http://example.com' def test_get_obs_download_server_url_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.get_obs_download_server_url() == \ Defaults.get_obs_download_server_url() def test_get_container_compression(self): assert self.runtime_config.get_container_compression() is None def test_get_container_compression_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.get_container_compression() == 'xz' @patch.object(RuntimeConfig, '_get_attribute') @patch('kiwi.logger.log.warning') def test_get_container_compression_invalid( self, mock_warning, mock_get_attribute ): mock_get_attribute.return_value = 'foo' assert self.runtime_config.get_container_compression() == 'xz' mock_warning.assert_called_once_with( 'Skipping invalid container compression: foo' ) @patch.object(RuntimeConfig, '_get_attribute') def test_get_container_compression_xz(self, mock_get_attribute): mock_get_attribute.return_value = 'xz' assert self.runtime_config.get_container_compression() == 'xz' def test_get_iso_tool_category(self): assert self.runtime_config.get_iso_tool_category() == 'cdrtools' def test_get_iso_tool_category_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.get_iso_tool_category() == 'xorriso' @patch.object(RuntimeConfig, '_get_attribute') @patch('kiwi.logger.log.warning') def test_get_iso_tool_category_invalid( self, mock_warning, mock_get_attribute ): mock_get_attribute.return_value = 'foo' assert self.runtime_config.get_iso_tool_category() == 'xorriso' mock_warning.assert_called_once_with( 'Skipping invalid iso tool category: foo' )
class Uri(object): """ **Normalize url types available in a kiwi configuration into standard mime types** :param str repo_type: repository type name. Only needed if the uri is not enough to determine the repository type e.g for yast2 vs. rpm-md obs repositories :param str uri: URI, repository location, file :param list mount_stack: list of mounted locations :param dict remote_uri_types: dictionary of remote uri type names :param dict local_uri_type: dictionary of local uri type names """ def __init__(self, uri, repo_type=None): self.runtime_config = RuntimeConfig() self.repo_type = repo_type self.uri = uri self.mount_stack = [] self.remote_uri_types = { 'http': True, 'https': True, 'ftp': True, 'obs': True } self.local_uri_type = { 'iso': True, 'dir': True, 'file': True, 'obsrepositories': True } def translate(self, check_build_environment=True): """ Translate repository location according to their URI type Depending on the URI type the provided location needs to be adapted e.g loop mounted in case of an ISO or updated by the service URL in case of an open buildservice project name :raises KiwiUriStyleUnknown: if the uri scheme can't be detected, is unknown or it is inconsistent with the build environment :param bool check_build_environment: specify if the uri translation should depend on the environment the build is called in. As of today this only effects the translation result if the image build happens inside of the Open Build Service :rtype: str """ uri = urlparse(self.uri) if not uri.scheme: raise KiwiUriStyleUnknown( 'URI scheme not detected {uri}'.format(uri=self.uri) ) elif uri.scheme == 'obs': if check_build_environment and Defaults.is_buildservice_worker(): return self._buildservice_path( name=''.join([uri.netloc, uri.path]).replace(':/', ':'), fragment=uri.fragment, urischeme=uri.scheme ) else: return self._obs_project_download_link( ''.join([uri.netloc, uri.path]).replace(':/', ':') ) elif uri.scheme == 'obsrepositories': if not Defaults.is_buildservice_worker(): raise KiwiUriStyleUnknown( 'Only the buildservice can use the {0} schema'.format( uri.scheme ) ) return self._buildservice_path( name=''.join([uri.netloc, uri.path]).replace(':/', ':'), fragment=uri.fragment, urischeme=uri.scheme ) elif uri.scheme == 'dir': return self._local_path(uri.path) elif uri.scheme == 'file': return self._local_path(uri.path) elif uri.scheme == 'iso': return self._iso_mount_path(uri.path) elif uri.scheme.startswith('http') or uri.scheme == 'ftp': return ''.join([uri.scheme, '://', uri.netloc, uri.path]) else: raise KiwiUriStyleUnknown( 'URI schema %s not supported' % self.uri ) def credentials_file_name(self): """ Filename to store repository credentials :return: credentials file name :rtype: str """ uri = urlparse(self.uri) # initialize query with default credentials file name. # The information will be overwritten if the uri contains # a parameter query with a credentials parameter query = {'credentials': 'kiwiRepoCredentials'} if uri.query: query = dict(params.split('=') for params in uri.query.split('&')) return query['credentials'] def alias(self): """ Create hexdigest from URI as alias If the repository definition from the XML description does not provide an alias, kiwi creates one for you. However it's better to assign a human readable alias in the XML configuration :return: alias name as hexdigest :rtype: str """ return hashlib.md5(self.uri.encode()).hexdigest() def is_remote(self): """ Check if URI is a remote or local location :return: True or False :rtype: bool """ uri = urlparse(self.uri) if not uri.scheme: raise KiwiUriStyleUnknown( 'URI scheme not detected %s' % self.uri ) if uri.scheme == 'obs' and Defaults.is_buildservice_worker(): return False elif uri.scheme in self.remote_uri_types: return True elif uri.scheme in self.local_uri_type: return False else: raise KiwiUriTypeUnknown( 'URI type %s unknown' % uri.scheme ) def is_public(self): """ Check if URI is considered to be publicly reachable :return: True or False :rtype: bool """ uri = urlparse(self.uri) if not uri.scheme: # unknown uri schema is considered not public return False elif uri.scheme == 'obs': # obs is public but only if the configured download_server is public return self.runtime_config.is_obs_public() elif uri.scheme in self.remote_uri_types: # listed in remote uri types, thus public return True else: # unknown uri type considered not public return False def get_fragment(self): """ Returns the fragment part of the URI. :return: fragment part of the URI if any, None otherwise :rtype: str, None """ uri = urlparse(self.uri) return uri.fragment def _iso_mount_path(self, path): # The prefix name 'kiwi_iso_mount' has a meaning here because the # zypper repository manager looks up iso mount paths by its repo # source name iso_mount_path = mkdtemp(prefix='kiwi_iso_mount.') iso_mount = MountManager( device=path, mountpoint=iso_mount_path ) self.mount_stack.append(iso_mount) iso_mount.mount() return iso_mount.mountpoint def _local_path(self, path): return os.path.normpath(path) def _obs_project_download_link(self, name): name_parts = name.split(os.sep) repository = name_parts.pop() project = os.sep.join(name_parts) try: download_link = os.sep.join( [ self.runtime_config.get_obs_download_server_url(), project.replace(':', ':/'), repository ] ) if not Defaults.is_buildservice_worker(): request = requests.get(download_link) request.raise_for_status() return request.url else: log.warning( 'Using {0} without location verification due to build ' 'in isolated environment'.format(download_link) ) return download_link except Exception as e: raise KiwiUriOpenError( '{0}: {1}'.format(type(e).__name__, format(e)) ) def _buildservice_path(self, name, urischeme, fragment=None): """ Special to openSUSE buildservice. If the buildservice builds the image it arranges the repos for each build in a special environment, the so called build worker. """ bs_source_dir = '/usr/src/packages/SOURCES' if self.repo_type == 'container': if urischeme == 'obsrepositories': local_path = os.sep.join( [bs_source_dir, 'containers/_obsrepositories', name] ) else: local_path = os.sep.join( [bs_source_dir, 'containers', name] ) if fragment: local_path = ''.join([local_path, '#', fragment]) else: local_path = os.sep.join( [bs_source_dir, 'repos', name] ) return self._local_path(local_path) def __del__(self): for mount in reversed(self.mount_stack): if mount.is_mounted(): if mount.umount(): Path.wipe(mount.mountpoint)
class ArchiveBuilder(object): """ Root archive image builder Attributes * :attr:`xml_state` Instance of XMLState * :attr:`target_dir` target directory path name * :attr:`root_dir` root directory path name * :attr:`custom_args` Custom processing arguments defined as hash keys: * xz_options: string of XZ compression parameters """ def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.requested_archive_type = xml_state.get_build_type_name() self.result = Result(xml_state) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.filename = self._target_file_for('tar.xz') self.checksum = self._target_file_for('md5') self.xz_options = custom_args['xz_options'] if custom_args \ and 'xz_options' in custom_args else None self.runtime_config = RuntimeConfig() def create(self): """ Create a root archive tarball Build a simple XZ compressed root tarball from the image root tree Image types which triggers this builder are: * image="tbz" """ supported_archives = Defaults.get_archive_image_types() if self.requested_archive_type not in supported_archives: raise KiwiArchiveSetupError( 'Unknown archive type: %s' % self.requested_archive_type ) if self.requested_archive_type == 'tbz': log.info('Creating XZ compressed tar archive') archive = ArchiveTar( self._target_file_for('tar') ) archive.create_xz_compressed( self.root_dir, xz_options=self.xz_options, exclude=Defaults.get_exclude_list_for_root_data_sync() ) checksum = Checksum(self.filename) log.info('--> Creating archive checksum') checksum.md5(self.checksum) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.filename ) self.result.add( key='root_archive', filename=self.filename, use_for_bundle=True, compress=False, shasum=True ) self.result.add( key='root_archive_md5', filename=self.checksum, use_for_bundle=False ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result def _target_file_for(self, suffix): return ''.join( [ self.target_dir, '/', self.xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + self.xml_state.get_image_version(), '.', suffix ] )
def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.arch = platform.machine() if self.arch == 'i686' or self.arch == 'i586': self.arch = 'ix86' self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.spare_part_mbsize = xml_state.get_build_type_spare_part_size() self.persistency_type = xml_state.build_type.get_devicepersistency() self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot() self.custom_root_mount_args = xml_state.get_fs_mount_option_list() self.build_type_name = xml_state.get_build_type_name() self.image_format = xml_state.build_type.get_format() self.install_iso = xml_state.build_type.get_installiso() self.install_stick = xml_state.build_type.get_installstick() self.install_pxe = xml_state.build_type.get_installpxe() self.blocksize = xml_state.build_type.get_target_blocksize() self.volume_manager_name = xml_state.get_volume_management() self.volumes = xml_state.get_volumes() self.volume_group_name = xml_state.get_volume_group_name() self.mdraid = xml_state.build_type.get_mdraid() self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr() self.force_mbr = xml_state.build_type.get_force_mbr() self.luks = xml_state.build_type.get_luks() self.luks_os = xml_state.build_type.get_luksOS() self.xen_server = xml_state.is_xen_server() self.requested_filesystem = xml_state.build_type.get_filesystem() self.requested_boot_filesystem = \ xml_state.build_type.get_bootfilesystem() self.bootloader = xml_state.build_type.get_bootloader() self.initrd_system = xml_state.get_initrd_system() self.target_removable = xml_state.build_type.get_target_removable() self.root_filesystem_is_multipath = \ xml_state.get_oemconfig_oem_multipath_scan() self.disk_setup = DiskSetup( xml_state, root_dir ) self.unpartitioned_bytes = \ xml_state.get_build_type_unpartitioned_bytes() self.custom_args = custom_args self.signing_keys = None if custom_args and 'signing_keys' in custom_args: self.signing_keys = custom_args['signing_keys'] self.boot_image = BootImage( xml_state, target_dir, root_dir, signing_keys=self.signing_keys, custom_args=self.custom_args ) self.firmware = FirmWare( xml_state ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.diskname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version(), '.raw' ] ) self.install_media = self._install_image_requested() self.generic_fstab_entries = [] # an instance of a class with the sync_data capability # representing the entire image system except for the boot/ area # which could live on another part of the disk self.system = None # an instance of a class with the sync_data capability # representing the boot/ area of the disk if not part of # self.system self.system_boot = None # an instance of a class with the sync_data capability # representing the boot/efi area of the disk self.system_efi = None # result store self.result = Result(xml_state) self.runtime_config = RuntimeConfig()
def test_get_obs_download_server_url_default(self): with patch.dict('os.environ', {'HOME': './'}): runtime_config = RuntimeConfig() assert runtime_config.get_obs_download_server_url() == \ Defaults.get_obs_download_server_url()
class DiskBuilder(object): """ **Disk image builder** :param object xml_state: Instance of :class:`XMLState` :param str target_dir: Target directory path name :param str root_dir: Root directory path name :param dict custom_args: Custom processing arguments defined as hash keys: * signing_keys: list of package signing keys * xz_options: string of XZ compression parameters """ def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.arch = platform.machine() if self.arch == 'i686' or self.arch == 'i586': self.arch = 'ix86' self.root_dir = root_dir self.target_dir = target_dir self.xml_state = xml_state self.spare_part_mbsize = xml_state.get_build_type_spare_part_size() self.persistency_type = xml_state.build_type.get_devicepersistency() self.root_filesystem_is_overlay = xml_state.build_type.get_overlayroot() self.custom_root_mount_args = xml_state.get_fs_mount_option_list() self.build_type_name = xml_state.get_build_type_name() self.image_format = xml_state.build_type.get_format() self.install_iso = xml_state.build_type.get_installiso() self.install_stick = xml_state.build_type.get_installstick() self.install_pxe = xml_state.build_type.get_installpxe() self.blocksize = xml_state.build_type.get_target_blocksize() self.volume_manager_name = xml_state.get_volume_management() self.volumes = xml_state.get_volumes() self.volume_group_name = xml_state.get_volume_group_name() self.mdraid = xml_state.build_type.get_mdraid() self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr() self.force_mbr = xml_state.build_type.get_force_mbr() self.luks = xml_state.build_type.get_luks() self.luks_os = xml_state.build_type.get_luksOS() self.xen_server = xml_state.is_xen_server() self.requested_filesystem = xml_state.build_type.get_filesystem() self.requested_boot_filesystem = \ xml_state.build_type.get_bootfilesystem() self.bootloader = xml_state.build_type.get_bootloader() self.initrd_system = xml_state.get_initrd_system() self.target_removable = xml_state.build_type.get_target_removable() self.root_filesystem_is_multipath = \ xml_state.get_oemconfig_oem_multipath_scan() self.disk_setup = DiskSetup( xml_state, root_dir ) self.unpartitioned_bytes = \ xml_state.get_build_type_unpartitioned_bytes() self.custom_args = custom_args self.signing_keys = None if custom_args and 'signing_keys' in custom_args: self.signing_keys = custom_args['signing_keys'] self.boot_image = BootImage( xml_state, target_dir, root_dir, signing_keys=self.signing_keys, custom_args=self.custom_args ) self.firmware = FirmWare( xml_state ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.diskname = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + self.arch, '-' + xml_state.get_image_version(), '.raw' ] ) self.install_media = self._install_image_requested() self.generic_fstab_entries = [] # an instance of a class with the sync_data capability # representing the entire image system except for the boot/ area # which could live on another part of the disk self.system = None # an instance of a class with the sync_data capability # representing the boot/ area of the disk if not part of # self.system self.system_boot = None # an instance of a class with the sync_data capability # representing the boot/efi area of the disk self.system_efi = None # result store self.result = Result(xml_state) self.runtime_config = RuntimeConfig() def create(self): """ Build a bootable disk image and optional installation image The installation image is a bootable hybrid ISO image which embeds the disk image and an image installer Image types which triggers this builder are: * image="oem" * image="vmx" :return: result :rtype: instance of :class:`Result` """ disk = DiskBuilder( self.xml_state, self.target_dir, self.root_dir, self.custom_args ) result = disk.create_disk() # cleanup disk resources taken prior to next steps del disk disk_installer = DiskBuilder( self.xml_state, self.target_dir, self.root_dir ) result = disk_installer.create_install_media(result) disk_format = DiskBuilder( self.xml_state, self.target_dir, self.root_dir ) disk_format.append_unpartitioned_space() result = disk_format.create_disk_format(result) return result def create_disk(self): """ Build a bootable raw disk image :raises KiwiInstallMediaError: if install media is required and image type is not oem :raises KiwiVolumeManagerSetupError: root overlay at the same time volumes are defined is not supported :return: result :rtype: instance of :class:`Result` """ if self.install_media and self.build_type_name != 'oem': raise KiwiInstallMediaError( 'Install media requires oem type setup, got %s' % self.build_type_name ) if self.root_filesystem_is_overlay and self.volume_manager_name: raise KiwiVolumeManagerSetupError( 'Volume management together with root overlay is not supported' ) # setup recovery archive, cleanup and create archive if requested self.system_setup.create_recovery_archive() # prepare boot(initrd) root system log.info('Preparing boot system') self.boot_image.prepare() # precalculate needed disk size disksize_mbytes = self.disk_setup.get_disksize_mbytes() # create the disk log.info('Creating raw disk image %s', self.diskname) loop_provider = LoopDevice( self.diskname, disksize_mbytes, self.blocksize ) loop_provider.create() self.disk = Disk( self.firmware.get_partition_table_type(), loop_provider, self.xml_state.get_disk_start_sector() ) # create the bootloader instance self.bootloader_config = BootLoaderConfig( self.bootloader, self.xml_state, self.root_dir, { 'targetbase': loop_provider.get_device(), 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) } ) # create disk partitions and instance device map device_map = self._build_and_map_disk_partitions() # create raid on current root device if requested if self.mdraid: self.raid_root = RaidDevice(device_map['root']) self.raid_root.create_degraded_raid(raid_level=self.mdraid) device_map['root'] = self.raid_root.get_device() self.disk.public_partition_id_map['kiwi_RaidPart'] = \ self.disk.public_partition_id_map['kiwi_RootPart'] self.disk.public_partition_id_map['kiwi_RaidDev'] = \ device_map['root'].get_device() # create luks on current root device if requested if self.luks: self.luks_root = LuksDevice(device_map['root']) self.luks_root.create_crypto_luks( passphrase=self.luks, os=self.luks_os ) device_map['root'] = self.luks_root.get_device() # create filesystems on boot partition(s) if any self._build_boot_filesystems(device_map) # create volumes and filesystems for root system if self.volume_manager_name: volume_manager_custom_parameters = { 'fs_mount_options': self.custom_root_mount_args, 'root_label': self.disk_setup.get_root_label(), 'root_is_snapshot': self.xml_state.build_type.get_btrfs_root_is_snapshot(), 'root_is_readonly_snapshot': self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot(), 'image_type': self.xml_state.get_build_type_name() } volume_manager = VolumeManager( self.volume_manager_name, device_map['root'], self.root_dir + '/', self.volumes, volume_manager_custom_parameters ) volume_manager.setup( self.volume_group_name ) volume_manager.create_volumes( self.requested_filesystem ) volume_manager.mount_volumes() self.generic_fstab_entries += volume_manager.get_fstab( self.persistency_type, self.requested_filesystem ) self.system = volume_manager device_map['root'] = volume_manager.get_device()['root'] else: log.info( 'Creating root(%s) filesystem on %s', self.requested_filesystem, device_map['root'].get_device() ) filesystem_custom_parameters = { 'mount_options': self.custom_root_mount_args } filesystem = FileSystem( self.requested_filesystem, device_map['root'], self.root_dir + '/', filesystem_custom_parameters ) filesystem.create_on_device( label=self.disk_setup.get_root_label() ) self.system = filesystem # create a random image identifier self.mbrid = SystemIdentifier() self.mbrid.calculate_id() # create first stage metadata to boot image self._write_partition_id_config_to_boot_image() self._write_recovery_metadata_to_boot_image() self._write_raid_config_to_boot_image() self._write_generic_fstab_to_boot_image(device_map) self.system_setup.export_modprobe_setup( self.boot_image.boot_root_directory ) # create first stage metadata to system image self._write_image_identifier_to_system_image() self._write_crypttab_to_system_image() self._write_generic_fstab_to_system_image(device_map) if self.initrd_system == 'dracut': self._create_dracut_config() # create initrd cpio archive self.boot_image.create_initrd(self.mbrid) # create dracut config omitting one time kiwi dracut modules if self.initrd_system == 'dracut': self._create_system_dracut_config() # create second stage metadata to system image self._copy_first_boot_files_to_system_image() self._write_bootloader_config_to_system_image(device_map) self.mbrid.write_to_disk( self.disk.storage_provider ) # set SELinux file security contexts if context exists self._setup_selinux_file_contexts() # syncing system data to disk image log.info('Syncing system to image') if self.system_efi: log.info('--> Syncing EFI boot data to EFI partition') self.system_efi.sync_data() if self.system_boot: log.info('--> Syncing boot data at extra partition') self.system_boot.sync_data( self._get_exclude_list_for_boot_data_sync() ) log.info('--> Syncing root filesystem data') if self.root_filesystem_is_overlay: squashed_root_file = NamedTemporaryFile() squashed_root = FileSystemSquashFs( device_provider=None, root_dir=self.root_dir ) squashed_root.create_on_file( filename=squashed_root_file.name, exclude=self._get_exclude_list_for_root_data_sync(device_map) ) Command.run( [ 'dd', 'if=%s' % squashed_root_file.name, 'of=%s' % device_map['readonly'].get_device() ] ) else: self.system.sync_data( self._get_exclude_list_for_root_data_sync(device_map) ) # install boot loader self._install_bootloader(device_map) # set root filesystem properties self._setup_property_root_is_readonly_snapshot() # prepare for install media if requested if self.install_media: log.info('Saving boot image instance to file') self.boot_image.dump( self.target_dir + '/boot_image.pickledump' ) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.diskname ) # store image file name in result self.result.add( key='disk_image', filename=self.diskname, use_for_bundle=True if not self.image_format else False, compress=True, shasum=True ) # create image root metadata self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result def create_disk_format(self, result_instance): """ Create a bootable disk format from a previously created raw disk image :param object result_instance: instance of :class:`Result` :return: updated result_instance :rtype: instance of :class:`Result` """ if self.image_format: log.info('Creating %s Disk Format', self.image_format) disk_format = DiskFormat( self.image_format, self.xml_state, self.root_dir, self.target_dir ) disk_format.create_image_format() disk_format.store_to_result(result_instance) return result_instance def append_unpartitioned_space(self): """ Extends the raw disk if an unpartitioned area is specified """ if self.unpartitioned_bytes: log.info( 'Expanding disk with %d bytes of unpartitioned space', self.unpartitioned_bytes ) disk_format = DiskFormat( 'raw', self.xml_state, self.root_dir, self.target_dir ) disk_format.resize_raw_disk(self.unpartitioned_bytes, append=True) firmware = FirmWare(self.xml_state) loop_provider = LoopDevice(disk_format.diskname) loop_provider.create(overwrite=False) partitioner = Partitioner( firmware.get_partition_table_type(), loop_provider ) partitioner.resize_table() def create_install_media(self, result_instance): """ Build an installation image. The installation image is a bootable hybrid ISO image which embeds the raw disk image and an image installer :param object result_instance: instance of :class:`Result` :return: updated result_instance with installation media :rtype: instance of :class:`Result` """ if self.install_media: install_image = InstallImageBuilder( self.xml_state, self.root_dir, self.target_dir, self._load_boot_image_instance(), self.custom_args ) if self.install_iso or self.install_stick: log.info('Creating hybrid ISO installation image') install_image.create_install_iso() result_instance.add( key='installation_image', filename=install_image.isoname, use_for_bundle=True, compress=False, shasum=True ) if self.install_pxe: log.info('Creating PXE installation archive') install_image.create_install_pxe_archive() result_instance.add( key='installation_pxe_archive', filename=install_image.pxename, use_for_bundle=True, compress=False, shasum=True ) return result_instance def _load_boot_image_instance(self): boot_image_dump_file = self.target_dir + '/boot_image.pickledump' if not os.path.exists(boot_image_dump_file): raise KiwiInstallMediaError( 'No boot image instance dump %s found' % boot_image_dump_file ) try: with open(boot_image_dump_file, 'rb') as boot_image_dump: boot_image = pickle.load(boot_image_dump) boot_image.enable_cleanup() Path.wipe(boot_image_dump_file) except Exception as e: raise KiwiInstallMediaError( 'Failed to load boot image dump: %s' % type(e).__name__ ) return boot_image def _setup_selinux_file_contexts(self): security_context = '/etc/selinux/targeted/contexts/files/file_contexts' if os.path.exists(self.root_dir + security_context): self.system_setup.set_selinux_file_contexts( security_context ) def _install_image_requested(self): if self.install_iso or self.install_stick or self.install_pxe: return True def _get_exclude_list_for_root_data_sync(self, device_map): exclude_list = Defaults.get_exclude_list_for_root_data_sync() if 'boot' in device_map and self.bootloader == 'grub2_s390x_emu': exclude_list.append('boot/zipl/*') exclude_list.append('boot/zipl/.*') elif 'boot' in device_map: exclude_list.append('boot/*') exclude_list.append('boot/.*') if 'efi' in device_map: exclude_list.append('boot/efi/*') exclude_list.append('boot/efi/.*') return exclude_list def _get_exclude_list_for_boot_data_sync(self): return ['efi/*'] def _build_boot_filesystems(self, device_map): if 'efi' in device_map: log.info( 'Creating EFI(fat16) filesystem on %s', device_map['efi'].get_device() ) filesystem = FileSystem( 'fat16', device_map['efi'], self.root_dir + '/boot/efi/' ) filesystem.create_on_device( label=self.disk_setup.get_efi_label() ) self.system_efi = filesystem if 'boot' in device_map: boot_filesystem = self.requested_boot_filesystem if not boot_filesystem: boot_filesystem = self.requested_filesystem boot_directory = self.root_dir + '/boot/' if self.bootloader == 'grub2_s390x_emu': boot_directory = self.root_dir + '/boot/zipl/' boot_filesystem = 'ext2' log.info( 'Creating boot(%s) filesystem on %s', boot_filesystem, device_map['boot'].get_device() ) filesystem = FileSystem( boot_filesystem, device_map['boot'], boot_directory ) filesystem.create_on_device( label=self.disk_setup.get_boot_label() ) self.system_boot = filesystem def _build_and_map_disk_partitions(self): # noqa: C901 self.disk.wipe() if self.firmware.legacy_bios_mode(): log.info('--> creating EFI CSM(legacy bios) partition') self.disk.create_efi_csm_partition( self.firmware.get_legacy_bios_partition_size() ) if self.firmware.efi_mode(): log.info('--> creating EFI partition') self.disk.create_efi_partition( self.firmware.get_efi_partition_size() ) if self.firmware.ofw_mode(): log.info('--> creating PReP partition') self.disk.create_prep_partition( self.firmware.get_prep_partition_size() ) if self.disk_setup.need_boot_partition(): log.info('--> creating boot partition') self.disk.create_boot_partition( self.disk_setup.boot_partition_size() ) if self.spare_part_mbsize: log.info('--> creating spare partition') self.disk.create_spare_partition( self.spare_part_mbsize ) if self.root_filesystem_is_overlay: log.info('--> creating readonly root partition') squashed_root_file = NamedTemporaryFile() squashed_root = FileSystemSquashFs( device_provider=None, root_dir=self.root_dir ) squashed_root.create_on_file( filename=squashed_root_file.name, exclude=[Defaults.get_shared_cache_location()] ) squashed_rootfs_mbsize = os.path.getsize( squashed_root_file.name ) / 1048576 self.disk.create_root_readonly_partition( int(squashed_rootfs_mbsize + 50) ) if self.volume_manager_name and self.volume_manager_name == 'lvm': log.info('--> creating LVM root partition') self.disk.create_root_lvm_partition('all_free') elif self.mdraid: log.info('--> creating mdraid root partition') self.disk.create_root_raid_partition('all_free') else: log.info('--> creating root partition') self.disk.create_root_partition('all_free') if self.firmware.bios_mode(): log.info('--> setting active flag to primary boot partition') self.disk.activate_boot_partition() if self.firmware.ofw_mode(): log.info('--> setting active flag to primary PReP partition') self.disk.activate_boot_partition() if self.firmware.efi_mode(): if self.force_mbr: log.info('--> converting partition table to MBR') self.disk.create_mbr() elif self.hybrid_mbr: log.info('--> converting partition table to hybrid GPT/MBR') self.disk.create_hybrid_mbr() self.disk.map_partitions() return self.disk.get_device() def _create_dracut_config(self): dracut_config = [ 'hostonly="no"', 'dracut_rescue_image="no"' ] dracut_modules = [] dracut_modules_omit = ['kiwi-live', 'kiwi-dump'] if self.root_filesystem_is_multipath is False: dracut_modules_omit.append('multipath') if self.root_filesystem_is_overlay: dracut_modules.append('kiwi-overlay') else: dracut_modules_omit.append('kiwi-overlay') if self.build_type_name == 'oem': dracut_modules.append('kiwi-lib') dracut_modules.append('kiwi-repart') self._write_dracut_config( config=dracut_config, modules=dracut_modules, omit_modules=dracut_modules_omit ) def _create_system_dracut_config(self): dracut_modules = [] dracut_modules_omit = ['kiwi-live', 'kiwi-dump', 'kiwi-repart'] if self.root_filesystem_is_overlay: dracut_modules.append('kiwi-overlay') else: dracut_modules_omit.append('kiwi-overlay') self._write_dracut_config( config=[], modules=dracut_modules, omit_modules=dracut_modules_omit ) def _write_dracut_config(self, config, modules, omit_modules): dracut_config_file = ''.join( [self.root_dir, Defaults.get_dracut_conf_name()] ) if modules: config.append( 'add_dracutmodules+=" {0} "'.format(' '.join(modules)) ) if omit_modules: config.append( 'omit_dracutmodules+=" {0} "'.format(' '.join(omit_modules)) ) with open(dracut_config_file, 'w') as dracut_config: for entry in config: dracut_config.write(entry + os.linesep) def _write_partition_id_config_to_boot_image(self): log.info('Creating config.partids in boot system') filename = ''.join( [self.boot_image.boot_root_directory, '/config.partids'] ) partition_id_map = self.disk.get_public_partition_id_map() with open(filename, 'w') as partids: for id_name, id_value in list(partition_id_map.items()): partids.write('{0}="{1}"{2}'.format( id_name, id_value, os.linesep) ) self.boot_image.include_file( os.sep + os.path.basename(filename) ) def _write_raid_config_to_boot_image(self): if self.mdraid: log.info('Creating etc/mdadm.conf in boot system') filename = ''.join( [self.boot_image.boot_root_directory, '/etc/mdadm.conf'] ) self.raid_root.create_raid_config(filename) self.boot_image.include_file( os.sep + os.sep.join(['etc', os.path.basename(filename)]) ) def _write_crypttab_to_system_image(self): if self.luks: log.info('Creating etc/crypttab') filename = ''.join( [self.root_dir, '/etc/crypttab'] ) self.luks_root.create_crypttab(filename) self.boot_image.include_file( os.sep + os.sep.join(['etc', os.path.basename(filename)]) ) def _write_generic_fstab_to_system_image(self, device_map): log.info('Creating generic system etc/fstab') self._write_generic_fstab(device_map, self.system_setup) def _write_generic_fstab_to_boot_image(self, device_map): if self.initrd_system == 'kiwi': log.info('Creating generic boot image etc/fstab') self._write_generic_fstab(device_map, self.boot_image.setup) def _write_generic_fstab(self, device_map, setup): root_is_snapshot = \ self.xml_state.build_type.get_btrfs_root_is_snapshot() root_is_readonly_snapshot = \ self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot() fs_check_interval = '1 1' custom_root_mount_args = list(self.custom_root_mount_args) if root_is_snapshot and root_is_readonly_snapshot: custom_root_mount_args += ['ro'] fs_check_interval = '0 0' self._add_generic_fstab_entry( device_map['root'].get_device(), '/', custom_root_mount_args, fs_check_interval ) if 'boot' in device_map: if self.bootloader == 'grub2_s390x_emu': boot_mount_point = '/boot/zipl' else: boot_mount_point = '/boot' self._add_generic_fstab_entry( device_map['boot'].get_device(), boot_mount_point ) if 'efi' in device_map: self._add_generic_fstab_entry( device_map['efi'].get_device(), '/boot/efi' ) setup.create_fstab( self.generic_fstab_entries ) def _add_generic_fstab_entry( self, device, mount_point, options=None, check='0 0' ): if not options: options = ['defaults'] block_operation = BlockID(device) blkid_type = 'LABEL' if self.persistency_type == 'by-label' else 'UUID' device_id = block_operation.get_blkid(blkid_type) fstab_entry = ' '.join( [ blkid_type + '=' + device_id, mount_point, block_operation.get_filesystem(), ','.join(options), check ] ) if fstab_entry not in self.generic_fstab_entries: self.generic_fstab_entries.append( fstab_entry ) def _write_image_identifier_to_system_image(self): log.info('Creating image identifier: %s', self.mbrid.get_id()) self.mbrid.write( self.root_dir + '/boot/mbrid' ) def _write_recovery_metadata_to_boot_image(self): if os.path.exists(self.root_dir + '/recovery.partition.size'): log.info('Copying recovery metadata to boot image') recovery_metadata = ''.join( [self.root_dir, '/recovery.partition.size'] ) Command.run( ['cp', recovery_metadata, self.boot_image.boot_root_directory] ) self.boot_image.include_file( os.sep + os.path.basename(recovery_metadata) ) def _write_bootloader_config_to_system_image(self, device_map): if self.bootloader is not 'custom': log.info('Creating %s bootloader configuration', self.bootloader) boot_options = [] if self.mdraid: boot_options.append('rd.auto') boot_names = self.boot_image.get_boot_names() boot_device = device_map['root'] if 'boot' in device_map: boot_device = device_map['boot'] root_uuid = self.disk.get_uuid( device_map['root'].get_device() ) boot_uuid = self.disk.get_uuid( boot_device.get_device() ) self.bootloader_config.setup_disk_boot_images(boot_uuid) self.bootloader_config.setup_disk_image_config( boot_uuid=boot_uuid, root_uuid=root_uuid, kernel=boot_names.kernel_name, initrd=boot_names.initrd_name, boot_options=' '.join(boot_options) ) self.bootloader_config.write() log.info('Creating config.bootoptions') filename = ''.join( [self.boot_image.boot_root_directory, '/config.bootoptions'] ) kexec_boot_options = ' '.join( [ self.bootloader_config.get_boot_cmdline(root_uuid) ] + boot_options ) with open(filename, 'w') as boot_options: boot_options.write( '{0}{1}'.format(kexec_boot_options, os.linesep) ) partition_id_map = self.disk.get_public_partition_id_map() boot_partition_id = partition_id_map['kiwi_RootPart'] if 'kiwi_BootPart' in partition_id_map: boot_partition_id = partition_id_map['kiwi_BootPart'] self.system_setup.call_edit_boot_config_script( self.requested_filesystem, boot_partition_id ) def _install_bootloader(self, device_map): root_device = device_map['root'] boot_device = root_device if 'boot' in device_map: boot_device = device_map['boot'] if 'readonly' in device_map: root_device = device_map['readonly'] custom_install_arguments = { 'boot_device': boot_device.get_device(), 'root_device': root_device.get_device(), 'firmware': self.firmware, 'target_removable': self.target_removable } if 'efi' in device_map: efi_device = device_map['efi'] custom_install_arguments.update( {'efi_device': efi_device.get_device()} ) if 'prep' in device_map: prep_device = device_map['prep'] custom_install_arguments.update( {'prep_device': prep_device.get_device()} ) if self.volume_manager_name: self.system.umount_volumes() custom_install_arguments.update( {'system_volumes': self.system.get_volumes()} ) if self.bootloader is not 'custom': log.debug( "custom arguments for bootloader installation %s", custom_install_arguments ) bootloader = BootLoaderInstall( self.bootloader, self.root_dir, self.disk.storage_provider, custom_install_arguments ) if bootloader.install_required(): bootloader.install() self.system_setup.call_edit_boot_install_script( self.diskname, boot_device.get_device() ) def _setup_property_root_is_readonly_snapshot(self): if self.volume_manager_name: root_is_snapshot = \ self.xml_state.build_type.get_btrfs_root_is_snapshot() root_is_readonly_snapshot = \ self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot() if root_is_snapshot and root_is_readonly_snapshot: log.info( 'Setting root filesystem into read-only mode' ) self.system.mount_volumes() self.system.set_property_readonly_root() self.system.umount_volumes() def _copy_first_boot_files_to_system_image(self): boot_names = self.boot_image.get_boot_names() if self.initrd_system == 'kiwi': log.info('Copy boot files to system image') kernel = Kernel(self.boot_image.boot_root_directory) log.info('--> boot image kernel as %s', boot_names.kernel_name) kernel.copy_kernel( self.root_dir, ''.join(['/boot/', boot_names.kernel_name]) ) if self.xen_server: if kernel.get_xen_hypervisor(): log.info('--> boot image Xen hypervisor as xen.gz') kernel.copy_xen_hypervisor( self.root_dir, '/boot/xen.gz' ) else: raise KiwiDiskBootImageError( 'No hypervisor in boot image tree %s found' % self.boot_image.boot_root_directory ) log.info('--> initrd archive as %s', boot_names.initrd_name) Command.run( [ 'mv', self.boot_image.initrd_filename, self.root_dir + ''.join(['/boot/', boot_names.initrd_name]) ] )
class ContainerBuilder(object): """ **Container image builder** :param object xml_state: Instance of :class:`XMLState` :param str target_dir: target directory path name :param str root_dir: root directory path name :param dict custom_args: Custom processing arguments defined as hash keys: * xz_options: string of XZ compression parameters """ def __init__(self, xml_state, target_dir, root_dir, custom_args=None): self.root_dir = root_dir self.target_dir = target_dir self.container_config = xml_state.get_container_config() self.requested_container_type = xml_state.get_build_type_name() self.base_image = None self.base_image_md5 = None self.container_config['xz_options'] = custom_args['xz_options'] \ if custom_args and 'xz_options' in custom_args else None if xml_state.get_derived_from_image_uri(): # The base image is expected to be unpacked by the kiwi # prepare step and stored inside of the root_dir/image directory. # In addition a md5 file of the image is expected too self.base_image = Defaults.get_imported_root_image( self.root_dir ) self.base_image_md5 = ''.join([self.base_image, '.md5']) if not os.path.exists(self.base_image): raise KiwiContainerBuilderError( 'Unpacked Base image {0} not found'.format( self.base_image ) ) if not os.path.exists(self.base_image_md5): raise KiwiContainerBuilderError( 'Base image MD5 sum {0} not found at'.format( self.base_image_md5 ) ) self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.filename = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.', self.requested_container_type, '.tar' ] ) self.result = Result(xml_state) self.runtime_config = RuntimeConfig() def create(self): """ Builds a container image which is usually a tarball including container specific metadata. Image types which triggers this builder are: * image="docker" :return: result :rtype: instance of :class:`Result` """ if not self.base_image: log.info( 'Setting up %s container', self.requested_container_type ) container_setup = ContainerSetup( self.requested_container_type, self.root_dir, self.container_config ) container_setup.setup() else: checksum = Checksum(self.base_image) if not checksum.matches(checksum.md5(), self.base_image_md5): raise KiwiContainerBuilderError( 'base image file {0} checksum validation failed'.format( self.base_image ) ) log.info( '--> Creating container image' ) container_image = ContainerImage( self.requested_container_type, self.root_dir, self.container_config ) self.filename = container_image.create( self.filename, self.base_image ) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.filename ) self.result.add( key='container', filename=self.filename, use_for_bundle=True, compress=False, shasum=True ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result
class ContainerImageOCI(object): """ Create oci container from a root directory :param string root_dir: root directory path name :param dict custom_args: Custom processing arguments defined as hash keys: Example .. code:: python { 'container_name': 'name', 'container_tag': '1.0', 'additional_tags': ['current', 'foobar'], 'entry_command': [ '--config.entrypoint=/bin/bash', '--config.entrypoint=-x' ], 'entry_subcommand': [ '--config.cmd=ls', '--config.cmd=-l' ], 'maintainer': ['--author=tux'], 'user': ['--config.user=root'], 'workingdir': ['--config.workingdir=/root'], 'expose_ports': [ '--config.exposedports=80', '--config.exposedports=42' ], 'volumes': [ '--config.volume=/var/log', '--config.volume=/tmp' ], 'environment': ['--config.env=PATH=/bin'], 'labels': ['--config.label=name=value'] } """ def __init__(self, root_dir, custom_args=None): # noqa: C901 self.root_dir = root_dir self.oci_dir = None self.oci_root_dir = None self.container_name = 'kiwi-container' self.container_tag = 'latest' self.additional_tags = [] self.entry_command = [] self.entry_subcommand = [] self.maintainer = [] self.user = [] self.workingdir = [] self.expose_ports = [] self.volumes = [] self.environment = [] self.labels = [] self.runtime_config = RuntimeConfig() if custom_args: if 'container_name' in custom_args: self.container_name = custom_args['container_name'] if 'container_tag' in custom_args: self.container_tag = custom_args['container_tag'] if 'additional_tags' in custom_args: self.additional_tags = custom_args['additional_tags'] if 'entry_command' in custom_args: self.entry_command = custom_args['entry_command'] if 'entry_subcommand' in custom_args: self.entry_subcommand = custom_args['entry_subcommand'] if 'maintainer' in custom_args: self.maintainer = custom_args['maintainer'] if 'user' in custom_args: self.user = custom_args['user'] if 'workingdir' in custom_args: self.workingdir = custom_args['workingdir'] if 'expose_ports' in custom_args: self.expose_ports = custom_args['expose_ports'] if 'volumes' in custom_args: self.volumes = custom_args['volumes'] if 'environment' in custom_args: self.environment = custom_args['environment'] if 'labels' in custom_args: self.labels = custom_args['labels'] # for builds inside the buildservice we include a reference to the # specific build. Thus disturl label only exists inside the # buildservice. if Defaults.is_buildservice_worker(): self._append_buildservice_disturl_label() if not custom_args or 'container_name' not in custom_args: log.info( 'No container configuration provided, ' 'using default container name "kiwi-container:latest"' ) if not self.entry_command and not self.entry_subcommand: self.entry_subcommand = ['--config.cmd=/bin/bash'] def create(self, filename, base_image): """ Create compressed oci system container tar archive :param string filename: archive file name :param string base_image: archive used as a base image """ exclude_list = Defaults.get_exclude_list_for_root_data_sync() exclude_list.append('boot') exclude_list.append('dev') exclude_list.append('sys') exclude_list.append('proc') self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.') self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.') container_dir = os.sep.join( [self.oci_dir, 'umoci_layout'] ) container_name = ':'.join( [container_dir, self.container_tag] ) if base_image: Path.create(container_dir) image_tar = ArchiveTar(base_image) image_tar.extract(container_dir) Command.run([ 'umoci', 'config', '--image', '{0}:base_layer'.format(container_dir), '--tag', self.container_tag ]) else: Command.run( ['umoci', 'init', '--layout', container_dir] ) Command.run( ['umoci', 'new', '--image', container_name] ) Command.run( ['umoci', 'unpack', '--image', container_name, self.oci_root_dir] ) oci_root = DataSync( ''.join([self.root_dir, os.sep]), os.sep.join([self.oci_root_dir, 'rootfs']) ) oci_root.sync_data( options=['-a', '-H', '-X', '-A', '--delete'], exclude=exclude_list ) Command.run( ['umoci', 'repack', '--image', container_name, self.oci_root_dir] ) for tag in self.additional_tags: Command.run( ['umoci', 'config', '--image', container_name, '--tag', tag] ) Command.run( [ 'umoci', 'config' ] + self.maintainer + self.user + self.workingdir + self.entry_command + self.entry_subcommand + self.expose_ports + self.volumes + self.environment + self.labels + [ '--image', container_name, '--created', datetime.utcnow().strftime( '%Y-%m-%dT%H:%M:%S+00:00' ) ] ) Command.run( ['umoci', 'gc', '--layout', container_dir] ) return self.pack_image_to_file(filename) def pack_image_to_file(self, filename): """ Packs the oci image into the given filename. :param string filename: file name of the resulting packed image """ image_dir = os.sep.join([self.oci_dir, 'umoci_layout']) oci_tarfile = ArchiveTar(filename) container_compressor = self.runtime_config.get_container_compression() if container_compressor: return oci_tarfile.create_xz_compressed( image_dir, xz_options=self.runtime_config.get_xz_options() ) else: return oci_tarfile.create(image_dir) def _append_buildservice_disturl_label(self): with open(os.sep + Defaults.get_buildservice_env_name()) as env: for line in env: if line.startswith('BUILD_DISTURL') and '=' in line: disturl = line.split('=')[1].strip() if disturl: self.labels.append( ''.join([ '--config.label=' 'org.openbuildservice.disturl=', line.split('=')[1].strip() ]) ) log.warning('Could not find BUILD_DISTURL inside .buildenv') def __del__(self): if self.oci_dir: Path.wipe(self.oci_dir) if self.oci_root_dir: Path.wipe(self.oci_root_dir)
class FileSystemBuilder(object): """ **Filesystem image builder** :param str label: filesystem label :param str root_dir: root directory path name :param str target_dir: target directory path name :param str requested_image_type: configured image type :param str requested_filesystem: requested filesystem name :param obejct system_setup: instance of :class:`SystemSetup` :param str filename: file name of the filesystem image :param int blocksize: configured disk blocksize :param object filesystem_setup: instance of :class:`FileSystemSetup` :param object filesystems_no_device_node: List of filesystems which are created from a data tree and do not require a block device e.g loop :param dict filesystem_custom_parameters: Configured custom filesystem mount and creation arguments :param object result: instance of :class:`Result` """ def __init__(self, xml_state, target_dir, root_dir): self.label = None self.root_dir = root_dir self.target_dir = target_dir self.requested_image_type = xml_state.get_build_type_name() if self.requested_image_type == 'pxe': self.requested_filesystem = xml_state.build_type.get_filesystem() else: self.requested_filesystem = self.requested_image_type if not self.requested_filesystem: raise KiwiFileSystemSetupError( 'No filesystem configured in %s type' % self.requested_image_type ) self.filesystem_custom_parameters = { 'mount_options': xml_state.get_fs_mount_option_list() } self.system_setup = SystemSetup( xml_state=xml_state, root_dir=self.root_dir ) self.filename = ''.join( [ target_dir, '/', xml_state.xml_data.get_name(), '.' + platform.machine(), '-' + xml_state.get_image_version(), '.', self.requested_filesystem ] ) self.blocksize = xml_state.build_type.get_target_blocksize() self.filesystem_setup = FileSystemSetup(xml_state, root_dir) self.filesystems_no_device_node = [ 'squashfs' ] self.result = Result(xml_state) self.runtime_config = RuntimeConfig() def create(self): """ Build a mountable filesystem image Image types which triggers this builder are: * image="ext2" * image="ext3" * image="ext4" * image="btrfs" * image="xfs" :return: result :rtype: instance of :class:`Result` """ log.info( 'Creating %s filesystem', self.requested_filesystem ) supported_filesystems = Defaults.get_filesystem_image_types() if self.requested_filesystem not in supported_filesystems: raise KiwiFileSystemSetupError( 'Unknown filesystem: %s' % self.requested_filesystem ) if self.requested_filesystem not in self.filesystems_no_device_node: self._operate_on_loop() else: self._operate_on_file() self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.filename ) self.result.add( key='filesystem_image', filename=self.filename, use_for_bundle=True, compress=True, shasum=True ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result def _operate_on_loop(self): filesystem = None loop_provider = LoopDevice( self.filename, self.filesystem_setup.get_size_mbytes(), self.blocksize ) loop_provider.create() filesystem = FileSystem( self.requested_filesystem, loop_provider, self.root_dir, self.filesystem_custom_parameters ) filesystem.create_on_device(self.label) log.info( '--> Syncing data to filesystem on %s', loop_provider.get_device() ) exclude_list = [ 'image', '.profile', '.kconfig', Defaults.get_shared_cache_location() ] filesystem.sync_data(exclude_list) def _operate_on_file(self): default_provider = DeviceProvider() filesystem = FileSystem( self.requested_filesystem, default_provider, self.root_dir, self.filesystem_custom_parameters ) filesystem.create_on_file( self.filename, self.label )
def __init__(self, root_dir, custom_args=None): # noqa: C901 self.root_dir = root_dir self.oci_dir = None self.oci_root_dir = None self.container_name = 'kiwi-container' self.container_tag = 'latest' self.additional_tags = [] self.entry_command = [] self.entry_subcommand = [] self.maintainer = [] self.user = [] self.workingdir = [] self.expose_ports = [] self.volumes = [] self.environment = [] self.labels = [] self.runtime_config = RuntimeConfig() if custom_args: if 'container_name' in custom_args: self.container_name = custom_args['container_name'] if 'container_tag' in custom_args: self.container_tag = custom_args['container_tag'] if 'additional_tags' in custom_args: self.additional_tags = custom_args['additional_tags'] if 'entry_command' in custom_args: self.entry_command = custom_args['entry_command'] if 'entry_subcommand' in custom_args: self.entry_subcommand = custom_args['entry_subcommand'] if 'maintainer' in custom_args: self.maintainer = custom_args['maintainer'] if 'user' in custom_args: self.user = custom_args['user'] if 'workingdir' in custom_args: self.workingdir = custom_args['workingdir'] if 'expose_ports' in custom_args: self.expose_ports = custom_args['expose_ports'] if 'volumes' in custom_args: self.volumes = custom_args['volumes'] if 'environment' in custom_args: self.environment = custom_args['environment'] if 'labels' in custom_args: self.labels = custom_args['labels'] # for builds inside the buildservice we include a reference to the # specific build. Thus disturl label only exists inside the # buildservice. if Defaults.is_buildservice_worker(): self._append_buildservice_disturl_label() if not custom_args or 'container_name' not in custom_args: log.info( 'No container configuration provided, ' 'using default container name "kiwi-container:latest"' ) if not self.entry_command and not self.entry_subcommand: self.entry_subcommand = ['--config.cmd=/bin/bash']
def __new__(self, source_dir): runtime_config = RuntimeConfig() if runtime_config.get_iso_tool_category() == 'cdrtools': return IsoToolsCdrTools(source_dir) else: return IsoToolsXorrIso(source_dir)