def _obs_project_download_link(self, name):
name_parts = name.split(os.sep)
repository = name_parts.pop()
project = os.sep.join(name_parts)
try:
download_link = os.sep.join(
[
self.runtime_config.get_obs_download_server_url(),
project.replace(':', ':/'), repository
]
)
if not Defaults.is_buildservice_worker():
request = requests.get(download_link)
request.raise_for_status()
return request.url
else:
log.warning(
'Using {0} without location verification due to build '
'in isolated environment'.format(download_link)
)
return download_link
except Exception as e:
raise KiwiUriOpenError(
'{0}: {1}'.format(type(e).__name__, format(e))
)
def set_flag(self, partition_id, flag_name):
"""
Set msdos partition flag
:param int partition_id: partition number
:param string flag_name: name from flag map
"""
if flag_name not in self.flag_map:
raise KiwiPartitionerMsDosFlagError(
'Unknown partition flag %s' % flag_name
)
if self.flag_map[flag_name]:
if flag_name == 'f.active':
Command.run(
[
'parted', self.disk_device,
'set', format(partition_id), 'boot', 'on'
]
)
else:
Command.run(
[
'sfdisk', '-c', self.disk_device,
format(partition_id), self.flag_map[flag_name]
]
)
else:
log.warning('Flag %s ignored on msdos', flag_name)
def setup_locale(self):
"""
Setup UTF8 system wide locale
"""
if 'locale' in self.preferences:
if 'POSIX' in self.preferences['locale'].split(','):
locale = 'POSIX'
else:
locale = '{0}.UTF-8'.format(
self.preferences['locale'].split(',')[0]
)
log.info('Setting up locale: %s', self.preferences['locale'])
if CommandCapabilities.has_option_in_help(
'systemd-firstboot', '--locale',
root=self.root_dir, raise_on_error=False
):
Path.wipe(self.root_dir + '/etc/locale.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--locale=' + locale
])
elif os.path.exists(self.root_dir + '/etc/sysconfig/language'):
Shell.run_common_function(
'baseUpdateSysConfig', [
self.root_dir + '/etc/sysconfig/language',
'RC_LANG', locale
]
)
else:
log.warning(
'locale setup skipped no capable '
'systemd-firstboot or etc/sysconfig/language not found'
)
def _setup_secure_boot_efi_image(self, lookup_path):
"""
Provide the shim loader and the shim signed grub2 loader
in the required boot path. Normally this task is done by
the shim-install tool. However, shim-install does not exist
on all distributions and the script does not operate well
in e.g CD environments from which we generate live and/or
install media. Thus shim-install is used if possible at
install time of the bootloader because it requires access
to the target block device. In any other case this setup
code should act as the fallback solution
"""
log.warning(
'--> Running fallback setup for shim secure boot efi image'
)
if not lookup_path:
lookup_path = self.root_dir
shim_image = Defaults.get_shim_loader(lookup_path)
if not shim_image:
raise KiwiBootLoaderGrubSecureBootError(
'Microsoft signed shim loader not found'
)
grub_image = Defaults.get_signed_grub_loader(lookup_path)
if not grub_image:
raise KiwiBootLoaderGrubSecureBootError(
'Shim signed grub2 efi loader not found'
)
Command.run(
['cp', shim_image, self._get_efi_image_name()]
)
Command.run(
['cp', grub_image, self.efi_boot_path]
)
def _get_root_cmdline_parameter(self, uuid):
firmware = self.xml_state.build_type.get_firmware()
initrd_system = self.xml_state.get_initrd_system()
cmdline = self.xml_state.build_type.get_kernelcmdline()
if cmdline and 'root=' in cmdline:
log.info(
'Kernel root device explicitly set via kernelcmdline'
)
return None
want_root_cmdline_parameter = False
if firmware and 'ec2' in firmware:
# EC2 requires to specifiy the root device in the bootloader
# configuration. This is because the used pvgrub or hvmloader
# reads this information and passes it to the guest configuration
# which has an impact on the devices attached to the guest.
want_root_cmdline_parameter = True
if initrd_system == 'dracut':
# When using a dracut initrd we have to specify the location
# of the root device
want_root_cmdline_parameter = True
if want_root_cmdline_parameter:
if uuid and self.xml_state.build_type.get_overlayroot():
return 'root=overlay:UUID={0}'.format(uuid)
elif uuid:
return 'root=UUID={0} rw'.format(uuid)
else:
log.warning(
'root=UUID=<uuid> setup requested, but uuid is not provided'
)
def __init__(self, root_dir, image_uri):
self.unknown_uri = None
self.root_dir = root_dir
try:
if image_uri.is_remote():
raise KiwiRootImportError(
'Only local imports are supported'
)
self.image_file = image_uri.translate()
if not os.path.exists(self.image_file):
raise KiwiRootImportError(
'Could not stat base image file: {0}'.format(
self.image_file
)
)
except KiwiUriTypeUnknown:
# Let specialized class handle unknown uri schemes
log.warning(
'Unkown URI type for the base image: %s', image_uri.uri
)
self.unknown_uri = image_uri.uri
finally:
self.post_init(image_uri)
def _get_dracut_output_file_format(self):
"""
Unfortunately the dracut initrd output file format varies between
the different Linux distributions. Tools like lsinitrd, and also
grub2 rely on the initrd output file to be in that format.
Thus when kiwi uses dracut the same file format should be used
all over the place in order to stay compatible with what the
distribution does
"""
default_outfile_format = 'initramfs-{kernel_version}.img'
dracut_search_env = {
'PATH': os.sep.join([self.boot_root_directory, 'usr', 'bin'])
}
dracut_tool = Path.which(
'dracut', custom_env=dracut_search_env, access_mode=os.X_OK
)
if dracut_tool:
outfile_expression = r'outfile="/boot/(init.*\$kernel.*)"'
with open(dracut_tool) as dracut:
outfile = re.findall(outfile_expression, dracut.read())[0]
if outfile:
return outfile.replace('$kernel', '{kernel_version}')
log.warning('Could not detect dracut output file format')
log.warning('Using default initrd file name format {0}'.format(
default_outfile_format
))
return default_outfile_format
def setup_keyboard_map(self):
"""
Setup console keyboard
"""
if 'keytable' in self.preferences:
log.info(
'Setting up keytable: %s', self.preferences['keytable']
)
if CommandCapabilities.has_option_in_help(
'systemd-firstboot', '--keymap',
root=self.root_dir, raise_on_error=False
):
Path.wipe(self.root_dir + '/etc/vconsole.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--keymap=' + self.preferences['keytable']
])
elif os.path.exists(self.root_dir + '/etc/sysconfig/keyboard'):
Shell.run_common_function(
'baseUpdateSysConfig', [
self.root_dir + '/etc/sysconfig/keyboard', 'KEYTABLE',
'"' + self.preferences['keytable'] + '"'
]
)
else:
log.warning(
'keyboard setup skipped no capable '
'systemd-firstboot or etc/sysconfig/keyboard found'
)
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured'
)
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution
)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
dev_mount = MountManager(
device='/dev', mountpoint=self.root_dir + '/dev'
)
dev_mount.umount()
if self.repository.unauthenticated == 'false':
log.warning(
'KIWI does not support signature checks for apt-get '
'package manager during the bootstrap procedure, any '
'provided key will only be used inside the chroot '
'environment'
)
Command.run(
[
'debootstrap', '--no-check-gpg', self.distribution,
bootstrap_dir, self.distribution_path
], self.command_env
)
data = DataSync(
bootstrap_dir + '/', self.root_dir
)
data.sync_data(
options=['-a', '-H', '-X', '-A']
)
for key in self.repository.signing_keys:
Command.run([
'chroot', self.root_dir, 'apt-key', 'add', key
], self.command_env)
except Exception as e:
raise KiwiDebootstrapError(
'%s: %s' % (type(e).__name__, format(e))
)
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def install_bootstrap(self, manager):
"""
Install system software using the package manager
from the host, also known as bootstrapping
:param object manager: instance of a :class:`PackageManager` subclass
:raises KiwiBootStrapPhaseFailed: if the bootstrapping process fails
either installing packages or including bootstrap archives
"""
if not self.xml_state.get_bootstrap_packages_sections():
log.warning('No <packages> sections marked as "bootstrap" found')
log.info('Processing of bootstrap stage skipped')
return
log.info('Installing bootstrap packages')
bootstrap_packages = self.xml_state.get_bootstrap_packages()
collection_type = self.xml_state.get_bootstrap_collection_type()
log.info('--> collection type: %s', collection_type)
bootstrap_collections = self.xml_state.get_bootstrap_collections()
bootstrap_products = self.xml_state.get_bootstrap_products()
bootstrap_archives = self.xml_state.get_bootstrap_archives()
# process package installations
if collection_type == 'onlyRequired':
manager.process_only_required()
else:
manager.process_plus_recommended()
all_install_items = self._setup_requests(
manager,
bootstrap_packages,
bootstrap_collections,
bootstrap_products
)
process = CommandProcess(
command=manager.process_install_requests_bootstrap(),
log_topic='bootstrap'
)
try:
process.poll_show_progress(
items_to_complete=all_install_items,
match_method=process.create_match_method(
manager.match_package_installed
)
)
except Exception as e:
raise KiwiBootStrapPhaseFailed(
'Bootstrap package installation failed: %s' % format(e)
)
manager.dump_reload_package_database()
# process archive installations
if bootstrap_archives:
try:
self._install_archives(bootstrap_archives)
except Exception as e:
raise KiwiBootStrapPhaseFailed(
'Bootstrap archive installation failed: %s' % format(e)
)
def __del__(self):
if self.node_name:
log.info('Cleaning up %s instance', type(self).__name__)
try:
Command.run(['losetup', '-d', self.node_name])
except Exception:
log.warning(
'loop device %s still busy', self.node_name
)
def _cleanup_dir_stack(self):
for location in reversed(self.dir_stack):
try:
Path.remove_hierarchy(self.root_dir + location)
except Exception as e:
log.warning(
'Failed to remove directory %s: %s', location, format(e)
)
del self.dir_stack[:]
def _accumulate_volume_size(self, root_mbytes):
"""
Calculate number of mbytes to add to the disk to allow
the creaton of the volumes with their configured size
"""
disk_volume_mbytes = 0
data_volume_mbytes = self._calculate_volume_mbytes()
root_volume = self._get_root_volume_configuration()
# For oem types we only add the default min volume size
# because their target size request is handled on first boot
# of the disk image in oemboot/repart
if self.build_type_name == 'oem':
for volume in self.volumes:
disk_volume_mbytes += Defaults.get_min_volume_mbytes()
return disk_volume_mbytes
# For vmx types we need to add the configured volume
# sizes because the image is used directly as it is without
# being deployed and resized on a target disk
for volume in self.volumes:
if volume.realpath and not volume.realpath == '/' and volume.size:
[size_type, req_size] = volume.size.split(':')
disk_add_mbytes = 0
if size_type == 'freespace':
disk_add_mbytes += int(req_size)
else:
disk_add_mbytes += int(req_size) - \
data_volume_mbytes.volume[volume.realpath]
if disk_add_mbytes > 0:
disk_volume_mbytes += disk_add_mbytes + \
Defaults.get_min_volume_mbytes()
else:
log.warning(
'volume size of %s MB for %s is too small, skipped',
int(req_size), volume.realpath
)
if root_volume:
if root_volume.size_type == 'freespace':
disk_add_mbytes = root_volume.req_size
else:
disk_add_mbytes = root_volume.req_size - \
root_mbytes + data_volume_mbytes.total
if disk_add_mbytes > 0:
disk_volume_mbytes += disk_add_mbytes + \
Defaults.get_min_volume_mbytes()
else:
log.warning(
'root volume size of %s MB is too small, skipped',
root_volume.req_size
)
return disk_volume_mbytes
def _json(self):
if self.style == 'color':
if self.color_json:
self._color_json()
else:
log.warning('pjson for color output not installed')
log.warning('run: pip install pjson')
self._standard_json()
else:
self._standard_json()
def get_install_image_boot_default(self, loader=None):
"""
Provide the default boot menu entry identifier for install images
The install image can be configured to provide more than
one boot menu entry. Menu entries configured are:
* [0] Boot From Hard Disk
* [1] Install
* [2] Failsafe Install
The installboot attribute controlls which of these are used
by default. If not specified the boot from hard disk entry
will be the default. Depending on the specified loader type
either an entry number or name will be returned.
:param string loader: bootloader name
:return: menu name or id
:rtype: str
"""
menu_entry_title = self.get_menu_entry_title(plain=True)
menu_type = namedtuple(
'menu_type', ['name', 'menu_id']
)
menu_list = [
menu_type(
name='Boot_from_Hard_Disk', menu_id='0'
),
menu_type(
name='Install_' + menu_entry_title, menu_id='1'
),
menu_type(
name='Failsafe_--_Install_' + menu_entry_title, menu_id='2'
)
]
boot_id = 0
install_boot_name = self.xml_state.build_type.get_installboot()
if install_boot_name == 'failsafe-install':
boot_id = 2
elif install_boot_name == 'install':
boot_id = 1
if not self.failsafe_boot_entry_requested() and boot_id == 2:
log.warning(
'Failsafe install requested but failsafe menu entry is disabled'
)
log.warning('Switching to standard install')
boot_id = 1
if loader and loader == 'isolinux':
return menu_list[boot_id].name
else:
return menu_list[boot_id].menu_id
def request_package_exclusion(self, name):
"""
Queue a package exclusion(skip) request
Package exclusion for apt package manager not yet implemented
:param string name: unused
"""
log.warning(
'Package exclusion for (%s) not supported for apt-get', name
)
def __del__(self):
if self.storage_provider.is_loop() and self.is_mapped:
log.info('Cleaning up %s instance', type(self).__name__)
try:
for device_node in self.partition_map.values():
Command.run(['dmsetup', 'remove', device_node])
except Exception:
log.warning(
'cleanup of partition device maps failed, %s still busy',
self.storage_provider.get_device()
)
def request_collection(self, name):
"""
Queue a collection request
There is no collection definition in the deb repo data
:param string name: unused
"""
log.warning(
'Collection(%s) handling not supported for apt-get', name
)
def __del__(self):
if self.volume_group:
log.info('Cleaning up %s instance', type(self).__name__)
if self.umount_volumes():
Path.wipe(self.mountpoint)
try:
Command.run(['vgchange', '-an', self.volume_group])
except Exception:
log.warning(
'volume group %s still busy', self.volume_group
)
def request_product(self, name):
"""
Queue a product request
There is no product definition in the deb repo data
:param string name: unused
"""
log.warning(
'Product(%s) handling not supported for apt-get', name
)
def __del__(self):
if self.luks_device:
log.info('Cleaning up %s instance', type(self).__name__)
try:
Command.run(
['cryptsetup', 'luksClose', self.luks_name]
)
except Exception:
log.warning(
'Shutdown of luks map %s failed, %s still busy',
self.luks_name, self.luks_device
)
def _check_boot_theme_exists(self):
if self.theme:
theme_dir = os.sep.join(
[
self.root_dir, 'boot', self.boot_directory_name,
'themes', self.theme
]
)
if not os.path.exists(theme_dir):
log.warning('Theme %s not found', theme_dir)
log.warning('Set bootloader terminal to console mode')
self.terminal = 'console'
def __del__(self):
if self.raid_device:
log.info('Cleaning up %s instance', type(self).__name__)
try:
Command.run(
['mdadm', '--stop', self.raid_device]
)
except Exception:
log.warning(
'Shutdown of raid device failed, %s still busy',
self.raid_device
)
def _json(self):
"""
Show data in json output format and selected style
"""
if self.style == 'color':
if self.color_json:
self._color_json()
else:
log.warning('pjson for color output not installed')
log.warning('run: pip install pjson')
self._standard_json()
else:
self._standard_json()
def _append_buildservice_disturl_label(self):
with open(os.sep + Defaults.get_buildservice_env_name()) as env:
for line in env:
if line.startswith('BUILD_DISTURL') and '=' in line:
disturl = line.split('=')[1].strip()
if disturl:
self.labels.append(
''.join([
'--config.label='
'org.openbuildservice.disturl=',
line.split('=')[1].strip()
])
)
log.warning('Could not find BUILD_DISTURL inside .buildenv')
def __init__(self, xml_state, root_dir):
self.configured_size = xml_state.get_build_type_size(
include_unpartitioned=True
)
if xml_state.get_build_type_unpartitioned_bytes() > 0:
log.warning(
'Unpartitoned size attribute is ignored for filesystem images'
)
self.size = SystemSize(root_dir)
self.requested_image_type = xml_state.get_build_type_name()
if self.requested_image_type in Defaults.get_filesystem_image_types():
self.requested_filesystem = self.requested_image_type
else:
self.requested_filesystem = xml_state.build_type.get_filesystem()
def _cleanup_mount_stack(self):
for mount in reversed(self.mount_stack):
if mount.is_mounted():
try:
mount.umount_lazy()
except Exception as e:
log.warning(
'Image root directory %s not cleanly umounted: %s',
self.root_dir, format(e)
)
else:
log.warning('Path %s not a mountpoint', mount.mountpoint)
del self.mount_stack[:]
def get_install_image_boot_default(self, loader=None):
"""
Provide the default boot menu entry identifier for install images
The install image can be configured to provide more than
one boot menu entry. Menu entries configured are:
* [0] Boot From Hard Disk
* [1] Install
* [2] Failsafe Install
The installboot attribute controlls which of these are used
by default. If not specified the boot from hard disk entry
will be the default. Depending on the specified loader type
either an entry number or name will be returned.
:param string loader: bootloader name
:return: menu name or id
:rtype: str
"""
menu_entry_title = self.get_menu_entry_title(plain=True)
menu_type = namedtuple('menu_type', ['name', 'menu_id'])
menu_list = [
menu_type(name='Boot_from_Hard_Disk', menu_id='0'),
menu_type(name='Install_' + menu_entry_title, menu_id='1'),
menu_type(name='Failsafe_--_Install_' + menu_entry_title,
menu_id='2')
]
boot_id = 0
install_boot_name = self.xml_state.build_type.get_installboot()
if install_boot_name == 'failsafe-install':
boot_id = 2
elif install_boot_name == 'install':
boot_id = 1
if not self.failsafe_boot_entry_requested() and boot_id == 2:
log.warning(
'Failsafe install requested but failsafe menu entry is disabled'
)
log.warning('Switching to standard install')
boot_id = 1
if loader and loader == 'isolinux':
return menu_list[boot_id].name
else:
return menu_list[boot_id].menu_id
def sync_data(self, options=None, exclude=None):
"""
Sync data from source to target using rsync
:param list options: rsync options
:param list exclude: file patterns to exclude
"""
target_entry_permissions = None
exclude_options = []
rsync_options = []
if options:
rsync_options = options
if not self.target_supports_extended_attributes():
warn_me = False
if '-X' in rsync_options:
rsync_options.remove('-X')
warn_me = True
if '-A' in rsync_options:
rsync_options.remove('-A')
warn_me = True
if warn_me:
log.warning(
'Extended attributes not supported for target: %s',
self.target_dir
)
if exclude:
for item in exclude:
exclude_options.append('--exclude')
exclude_options.append(
'/' + item
)
if os.path.exists(self.target_dir):
target_entry_permissions = os.stat(self.target_dir)[ST_MODE]
Command.run(
['rsync'] + rsync_options + exclude_options + [
self.source_dir, self.target_dir
]
)
if target_entry_permissions:
# rsync applies the permissions of the source directory
# also to the target directory which is unwanted because
# only permissions of the files and directories from the
# source directory and its contents should be transfered
# but not from the source directory itself. Therefore
# the permission bits of the target directory before the
# sync are applied back after sync to ensure they have
# not changed
os.chmod(self.target_dir, target_entry_permissions)
def sync_data(self, options=None, exclude=None):
"""
Sync data from source to target using rsync
:param list options: rsync options
:param list exclude: file patterns to exclude
"""
target_entry_permissions = None
exclude_options = []
rsync_options = []
if options:
rsync_options = options
if not self.target_supports_extended_attributes():
warn_me = False
if '-X' in rsync_options:
rsync_options.remove('-X')
warn_me = True
if '-A' in rsync_options:
rsync_options.remove('-A')
warn_me = True
if warn_me:
log.warning(
'Extended attributes not supported for target: %s',
self.target_dir
)
if exclude:
for item in exclude:
exclude_options.append('--exclude')
exclude_options.append(
'/' + item
)
if os.path.exists(self.target_dir):
target_entry_permissions = os.stat(self.target_dir)[ST_MODE]
Command.run(
['rsync'] + rsync_options + exclude_options + [
self.source_dir, self.target_dir
]
)
if target_entry_permissions:
# rsync applies the permissions of the source directory
# also to the target directory which is unwanted because
# only permissions of the files and directories from the
# source directory and its contents should be transfered
# but not from the source directory itself. Therefore
# the permission bits of the target directory before the
# sync are applied back after sync to ensure they have
# not changed
os.chmod(self.target_dir, target_entry_permissions)
def setup_keyboard_map(self):
"""
Setup etc/sysconfig/keyboard console keyboard
"""
if 'keytable' in self.preferences:
keyboard_config = self.root_dir + '/etc/sysconfig/keyboard'
if os.path.exists(keyboard_config):
log.info('Setting up keytable: %s',
self.preferences['keytable'])
Shell.run_common_function('baseUpdateSysConfig', [
keyboard_config, 'KEYTABLE',
'"' + self.preferences['keytable'] + '"'
])
else:
log.warning(
'keyboard setup skipped etc/sysconfig/keyboard not found')
def create_on_file(self, filename, label=None, exclude=None):
"""
Create iso filesystem from data tree
There is no label which could be set for iso filesystem
thus this parameter is not used
:param string filename: result file path name
:param string label: unused
:param string exclude: unused
"""
meta_data = self.custom_args['meta_data']
efi_mode = meta_data.get('efi_mode')
iso_tool = IsoTools(self.root_dir)
iso = Iso(self.root_dir)
if not efi_mode:
iso.setup_isolinux_boot_path()
if not iso_tool.has_iso_hybrid_capability():
iso.create_header_end_marker()
iso_tool.init_iso_creation_parameters(meta_data)
iso_tool.add_efi_loader_parameters()
iso_tool.create_iso(filename)
if not iso_tool.has_iso_hybrid_capability():
if not efi_mode:
hybrid_offset = iso.create_header_end_block(filename)
iso_tool.create_iso(filename,
hidden_files=[iso.header_end_name])
iso.relocate_boot_catalog(filename)
iso.fix_boot_catalog(filename)
mbr_id = meta_data['mbr_id'] if 'mbr_id' in meta_data else \
'0xffffffff'
iso.create_hybrid(hybrid_offset, mbr_id, filename)
else:
message = dedent('''
Can't create hybrid ISO in EFI mode with cdrtools
isohybrid requires isolinux as loader. In EFI mode
the configured bootloader e.g grub is used and no
isolinux signature exists.
''').strip() + os.linesep
log.warning(message)
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured')
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
dev_mount = MountManager(device='/dev',
mountpoint=self.root_dir + '/dev')
dev_mount.umount()
if self.repository.unauthenticated == 'false':
log.warning(
'KIWI does not support signature checks for apt-get '
'package manager during the bootstrap procedure, any '
'provided key will only be used inside the chroot '
'environment')
Command.run([
'debootstrap', '--no-check-gpg', self.distribution,
bootstrap_dir, self.distribution_path
], self.command_env)
data = DataSync(bootstrap_dir + '/', self.root_dir)
data.sync_data(options=['-a', '-H', '-X', '-A'])
for key in self.repository.signing_keys:
Command.run(['chroot', self.root_dir, 'apt-key', 'add', key],
self.command_env)
except Exception as e:
raise KiwiDebootstrapError('%s: %s' %
(type(e).__name__, format(e)))
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def check_version(self,
call,
version_waterline,
version_flags=None,
root=None,
raise_on_error=True):
"""
Checks if the given command version is equal or higher than
the given version tuple.
:param str call: the command the check
:param tuple version_waterline: minimum desired version of the command
:param list version_flags: a list with the required command arguments.
:param str root: root directory of the env to validate
:param bool raise_on_error: control error behavior
:raises KiwiCommandCapabilitiesError: if raise_on_error is True and
command execution fails or version can't be parsed.
:return: True if the current command version is equal or higher to
version_waterline
:rtype: bool
"""
version_args = version_flags or ['--version']
if root:
arguments = ['chroot', root, call] + version_args
else:
arguments = [call] + version_args
version_info = None
try:
command = Command.run(arguments)
for line in command.output.splitlines():
match = re.search('[0-9]+(\.[0-9]+)*', line)
if match:
version_info = tuple(
int(elt) for elt in match.group(0).split('.'))
break
if version_info is None:
raise Exception
except Exception:
message = 'Could not parse {0} version'.format(call)
if raise_on_error:
raise KiwiCommandCapabilitiesError(message)
log.warning(message)
return False
return version_info >= version_waterline
def _update_vmdk_descriptor(self):
"""
Update the VMDK descriptor with the VMware tools version
and type information. This is done to let VMware's virtualization
infrastructure know about the installed VMware tools at boot
time of the image within a VMware virtual environment
e.g VCloud Air. It's required to have vmtoolsd installed as
part of the image. If not found a warning is provided to the
user and the VMDK descriptor stays untouched
"""
vmdk_vmtoolsd = self.root_dir + '/usr/bin/vmtoolsd'
if not os.path.exists(vmdk_vmtoolsd):
log.warning(
'Could not find vmtoolsd in image root %s', self.root_dir
)
log.warning(
'Update of VMDK metadata skipped'
)
return
log.info('Updating VMDK metadata')
vmdk_tools_install_type = 4
vmdk_tools_version = self._get_vmdk_tools_version()
vmdk_image_name = self.get_target_name_for_format('vmdk')
log.info(
'--> Setting tools version: %d', vmdk_tools_version
)
log.info(
'--> Setting tools install type: %d', vmdk_tools_install_type
)
vmdk_descriptor_call = Command.run(
['dd', 'if=' + vmdk_image_name, 'bs=1', 'count=1024', 'skip=512']
)
vmdk_descriptor_lines = \
vmdk_descriptor_call.output.strip('\0').split('\n')
if (vmdk_descriptor_lines[0] != 'encoding="UTF-8"'):
vmdk_descriptor_lines.insert(0, 'encoding="UTF-8"')
vmdk_descriptor_lines.append(
'ddb.toolsInstallType = "%s"' % vmdk_tools_install_type
)
vmdk_descriptor_lines.append(
'ddb.toolsVersion = "%s"' % vmdk_tools_version
)
with open(vmdk_image_name, 'r+b') as vmdk:
vmdk.seek(512, 0)
vmdk.write(bytes('\n'.join(vmdk_descriptor_lines), 'utf-8'))
vmdk.seek(0, 2)
def extract_oci_image(self):
"""
Extract and converts to OCI the image from the provided
image file to a temporary location to KIWI can work with it.
"""
if not self.unknown_uri:
compressor = Compress(self.image_file)
compressor.uncompress(True)
self.uncompressed_image = compressor.uncompressed_filename
skopeo_uri = 'docker-archive:{0}'.format(self.uncompressed_image)
else:
log.warning('Bypassing base image URI to skopeo tool')
skopeo_uri = self.unknown_uri
Command.run([
'skopeo', 'copy', skopeo_uri, 'oci:{0}'.format(self.oci_layout_dir)
])
def get_size_mbytes(self, filesystem=None):
"""
Precalculate the requires size in mbytes to store all data
from the root directory in the requested filesystem. Return
the configured value if present, if not return the calculated
result
:param string filesystem: name
:return: mbytes
:rtype: int
"""
root_dir_mbytes = self.size.accumulate_mbyte_file_sizes()
filesystem_mbytes = self.size.customize(
root_dir_mbytes, self.requested_filesystem or filesystem
)
if not self.configured_size:
log.info(
'Using calculated size: %d MB',
filesystem_mbytes
)
return filesystem_mbytes
elif self.configured_size.additive:
result_filesystem_mbytes = \
self.configured_size.mbytes + filesystem_mbytes
log.info(
'Using configured size: %d MB + %d MB calculated = %d MB',
self.configured_size.mbytes,
filesystem_mbytes,
result_filesystem_mbytes
)
return result_filesystem_mbytes
else:
log.info(
'Using configured size: %d MB',
self.configured_size.mbytes
)
if self.configured_size.mbytes < filesystem_mbytes:
log.warning(
'--> Configured size smaller than calculated size: %d MB',
filesystem_mbytes
)
return self.configured_size.mbytes
def install_bootstrap(self, manager):
"""
Install system software using the package manager
from the host, also known as bootstrapping
"""
if not self.xml_state.get_bootstrap_packages_sections():
log.warning('No <packages> sections marked as "bootstrap" found')
log.info('Processing of bootstrap stage skipped')
return
log.info('Installing bootstrap packages')
bootstrap_packages = self.xml_state.get_bootstrap_packages()
bootstrap_packages.append(self.xml_state.get_package_manager())
collection_type = self.xml_state.get_bootstrap_collection_type()
log.info('--> collection type: %s', collection_type)
bootstrap_collections = self.xml_state.get_bootstrap_collections()
bootstrap_products = self.xml_state.get_bootstrap_products()
bootstrap_archives = self.xml_state.get_bootstrap_archives()
# process package installations
if collection_type == 'onlyRequired':
manager.process_only_required()
else:
manager.process_plus_recommended()
all_install_items = self._setup_requests(manager, bootstrap_packages,
bootstrap_collections,
bootstrap_products)
process = CommandProcess(
command=manager.process_install_requests_bootstrap(),
log_topic='bootstrap')
try:
process.poll_show_progress(
items_to_complete=all_install_items,
match_method=process.create_match_method(
manager.match_package_installed))
except Exception as e:
raise KiwiBootStrapPhaseFailed(
'Bootstrap package installation failed: %s' % format(e))
manager.dump_reload_package_database()
# process archive installations
if bootstrap_archives:
try:
self._install_archives(bootstrap_archives)
except Exception as e:
raise KiwiBootStrapPhaseFailed(
'Bootstrap archive installation failed: %s' % format(e))
def get_size_mbytes(self, filesystem=None):
"""
Precalculate the requires size in mbytes to store all data
from the root directory in the requested filesystem. Return
the configured value if present, if not return the calculated
result
:param string filesystem: name
:return: mbytes
:rtype: int
"""
root_dir_mbytes = self.size.accumulate_mbyte_file_sizes()
filesystem_mbytes = self.size.customize(
root_dir_mbytes, self.requested_filesystem or filesystem
)
if not self.configured_size:
log.info(
'Using calculated size: %d MB',
filesystem_mbytes
)
return filesystem_mbytes
elif self.configured_size.additive:
result_filesystem_mbytes = \
self.configured_size.mbytes + filesystem_mbytes
log.info(
'Using configured size: %d MB + %d MB calculated = %d MB',
self.configured_size.mbytes,
filesystem_mbytes,
result_filesystem_mbytes
)
return result_filesystem_mbytes
else:
log.info(
'Using configured size: %d MB',
self.configured_size.mbytes
)
if self.configured_size.mbytes < filesystem_mbytes:
log.warning(
'--> Configured size smaller than calculated size: %d MB',
filesystem_mbytes
)
return self.configured_size.mbytes
def set_flag(self, partition_id, flag_name):
"""
Set GPT partition flag
:param int partition_id: partition number
:param string flag_name: name from flag map
"""
if flag_name not in self.flag_map:
raise KiwiPartitionerGptFlagError('Unknown partition flag %s' %
flag_name)
if self.flag_map[flag_name]:
Command.run([
'sgdisk', '-t',
':'.join([format(partition_id),
self.flag_map[flag_name]]), self.disk_device
])
else:
log.warning('Flag %s ignored on GPT', flag_name)
def extract_oci_image(self):
"""
Extract and converts to OCI the image from the provided
image file to a temporary location to KIWI can work with it.
"""
if not self.unknown_uri:
compressor = Compress(self.image_file)
compressor.uncompress(True)
self.uncompressed_image = compressor.uncompressed_filename
skopeo_uri = 'docker-archive:{0}'.format(self.uncompressed_image)
else:
log.warning('Bypassing base image URI to skopeo tool')
skopeo_uri = self.unknown_uri
Command.run([
'skopeo', 'copy', skopeo_uri,
'oci:{0}:base_layer'.format(self.oci_layout_dir)
])
def decode(literal):
"""
Decodes the given literal with the default charset. In case of
failure attemps to decode using utf-8 charset.
:param bytes literal: literal to decode
:return: decoded string
:rtype: str
"""
try:
return Codec._wrapped_decode(literal)
except Exception:
log.warning("Failed decoding literal. Forcing UTF-8 decoding")
try:
return Codec._wrapped_decode(literal, 'utf_8')
except Exception:
raise KiwiDecodingError('Locale setup is not utf-8 compatible')
def _accumulate_volume_size(self, root_mbytes):
"""
Calculate number of mbytes to add to the disk to allow
the creaton of the volumes with their configured size
"""
disk_volume_mbytes = 0
data_volume_mbytes = self._calculate_volume_mbytes()
root_volume = self._get_root_volume_configuration()
for volume in self.volumes:
if volume.realpath and not volume.realpath == '/' and volume.size:
[size_type, req_size] = volume.size.split(':')
disk_add_mbytes = 0
if size_type == 'freespace':
disk_add_mbytes += int(req_size)
else:
disk_add_mbytes += int(req_size) - \
data_volume_mbytes.volume[volume.realpath]
if disk_add_mbytes > 0:
disk_volume_mbytes += disk_add_mbytes + \
Defaults.get_min_volume_mbytes()
else:
log.warning(
'volume size of %s MB for %s is too small, skipped',
int(req_size), volume.realpath)
if root_volume:
if root_volume.size_type == 'freespace':
disk_add_mbytes = root_volume.req_size
else:
disk_add_mbytes = root_volume.req_size - \
root_mbytes + data_volume_mbytes.total
if disk_add_mbytes > 0:
disk_volume_mbytes += disk_add_mbytes + \
Defaults.get_min_volume_mbytes()
else:
log.warning('root volume size of %s MB is too small, skipped',
root_volume.req_size)
return disk_volume_mbytes
def __init__(self, root_dir, image_uri, custom_args=None):
self.unknown_uri = None
self.root_dir = root_dir
try:
if image_uri.is_remote():
raise KiwiRootImportError('Only local imports are supported')
self.image_file = image_uri.translate()
if not os.path.exists(self.image_file):
raise KiwiRootImportError(
'Could not stat base image file: {0}'.format(
self.image_file))
except KiwiUriTypeUnknown:
# Let specialized class handle unknown uri schemes
log.warning('Unkown URI type for the base image: %s',
image_uri.uri)
self.unknown_uri = image_uri.uri
finally:
self.post_init(custom_args)
def _setup_secure_boot_efi_image(self, lookup_path):
"""
Provide the shim loader and the shim signed grub2 loader
in the required boot path. Normally this task is done by
the shim-install tool. However, shim-install does not exist
on all distributions and the script does not operate well
in e.g CD environments from which we generate live and/or
install media. Thus shim-install is used if possible at
install time of the bootloader because it requires access
to the target block device. In any other case this setup
code should act as the fallback solution
"""
log.warning(
'--> Running fallback setup for shim secure boot efi image'
)
if not lookup_path:
lookup_path = self.boot_dir
grub_image = Defaults.get_signed_grub_loader(lookup_path)
if not grub_image:
raise KiwiBootLoaderGrubSecureBootError(
'Signed grub2 efi loader not found'
)
shim_image = Defaults.get_shim_loader(lookup_path)
if shim_image:
# The shim concept is based on a two step system including a
# grub image(shim) that got signed by Microsoft followed by
# a grub image that got signed by the shim. The shim image
# is the one that gets loaded by the firmware which itself
# loads the second stage grub image
Command.run(
['cp', shim_image, self._get_efi_image_name()]
)
Command.run(
['cp', grub_image, self.efi_boot_path]
)
else:
# Without shim a self signed grub image is used that
# gets loaded by the firmware
Command.run(
['cp', grub_image, self._get_efi_image_name()]
)
def extract_oci_image(self):
"""
Extract the image from the provided image file to a temporary
location to KIWI can work with it.
"""
if not self.unknown_uri:
tar = ArchiveTar(self.image_file)
self.uncompressed_image = mkdtemp(prefix='kiwi_uncompressed.')
tar.extract(self.uncompressed_image)
if self.tag:
skopeo_uri = 'oci:{0}:{1}'.format(self.uncompressed_image,
self.tag)
else:
skopeo_uri = 'oci:{0}'.format(self.uncompressed_image)
else:
log.warning('Bypassing base image URI to skopeo tool')
skopeo_uri = self.unknown_uri
Command.run([
'skopeo', 'copy', skopeo_uri, 'oci:{0}'.format(self.oci_layout_dir)
])
def _obs_project_download_link(self, name):
name_parts = name.split(os.sep)
repository = name_parts.pop()
project = os.sep.join(name_parts)
try:
download_link = os.sep.join([
self.runtime_config.get_obs_download_server_url(),
project.replace(':', ':/'), repository
])
if not Defaults.is_buildservice_worker():
request = requests.get(download_link)
request.raise_for_status()
return request.url
else:
log.warning(
'Using {0} without location verification due to build '
'in isolated environment'.format(download_link))
return download_link
except Exception as e:
raise KiwiUriOpenError('{0}: {1}'.format(
type(e).__name__, format(e)))
def __new__(self,
table_type,
storage_provider,
start_sector=None): # noqa: C901
host_architecture = platform.machine()
if host_architecture == 'x86_64':
if table_type == 'gpt':
return PartitionerGpt(storage_provider, start_sector)
elif table_type == 'msdos':
return PartitionerMsDos(storage_provider, start_sector)
elif host_architecture == 'i686' or host_architecture == 'i586':
if table_type == 'msdos':
return PartitionerMsDos(storage_provider, start_sector)
elif 'ppc64' in host_architecture:
if table_type == 'gpt':
return PartitionerGpt(storage_provider, start_sector)
elif table_type == 'msdos':
return PartitionerMsDos(storage_provider, start_sector)
elif 's390' in host_architecture:
if table_type == 'dasd':
if start_sector:
log.warning('disk_start_sector value is ignored '
'for dasd partitions')
return PartitionerDasd(storage_provider)
elif table_type == 'msdos':
return PartitionerMsDos(storage_provider, start_sector)
elif 'arm' in host_architecture or host_architecture == 'aarch64':
if table_type == 'gpt':
return PartitionerGpt(storage_provider, start_sector)
elif table_type == 'msdos':
return PartitionerMsDos(storage_provider, start_sector)
raise KiwiPartitionerSetupError(
'Support for partitioner on %s architecture not implemented' %
host_architecture)
def has_option_in_help(self,
call,
flag,
help_flags=None,
root=None,
raise_on_error=True):
"""
Checks if the given flag is present in the help output
of the given command.
:param str call: the command the check
:param str flag: the flag or substring to find in stdout
:param list help_flags: a list with the required command arguments.
:param str root: root directory of the env to validate
:raises KiwiCommandCapabilitiesError: if command execution fails
:return: True if the flag is found, False in any other case
:rtype: bool
"""
help_args = help_flags or ['--help']
if root:
arguments = ['chroot', root, call] + help_args
else:
arguments = [call] + help_args
try:
command = Command.run(arguments)
for line in command.output.splitlines():
if flag in line:
return True
for line in command.error.splitlines():
if flag in line:
return True
except Exception:
message = 'Could not parse {} output'.format(call)
if raise_on_error:
raise KiwiCommandCapabilitiesError(message)
log.warning(message)
return False
def _get_boot_image_output_file_format(self):
"""
The initrd output file format varies between
the different Linux distributions. Tools like lsinitrd, and also
grub2 rely on the initrd output file to be in that format. Thus
kiwi should use the same file format to stay compatible
with the distributions. The format is determined by the
outfile format used in the dracut initrd tool which is the
standard on all major linux distributions.
"""
if self.xml_state.get_initrd_system() == 'kiwi':
# The custom kiwi initrd system is used only on SUSE systems.
# The initrd environment does not provide dracut and thus the
# outfile format cannot be determined. on SUSE systems the
# initrd format is different than on upstream and therefore
# it can be explicitly specified. Note that the custom initrd
# system will become obsolete in the future.
default_outfile_format = 'initrd-{kernel_version}'
else:
default_outfile_format = 'initramfs-{kernel_version}.img'
dracut_search_env = {
'PATH': os.sep.join([self.boot_root_directory, 'usr', 'bin'])
}
dracut_tool = Path.which(
'dracut', custom_env=dracut_search_env, access_mode=os.X_OK
)
if dracut_tool:
outfile_expression = r'outfile="/boot/(init.*\$kernel.*)"'
with open(dracut_tool) as dracut:
matches = re.findall(outfile_expression, dracut.read())
if matches:
return matches[0].replace('$kernel', '{kernel_version}')
log.warning('Could not detect dracut output file format')
log.warning('Using default initrd file name format {0}'.format(
default_outfile_format
))
return default_outfile_format
def set_flag(self, partition_id, flag_name):
"""
Set msdos partition flag
:param int partition_id: partition number
:param string flag_name: name from flag map
"""
if flag_name not in self.flag_map:
raise KiwiPartitionerMsDosFlagError('Unknown partition flag %s' %
flag_name)
if self.flag_map[flag_name]:
if flag_name == 'f.active':
Command.run([
'parted', self.disk_device, 'set',
format(partition_id), 'boot', 'on'
])
else:
Command.run([
'sfdisk', '-c', self.disk_device,
format(partition_id), self.flag_map[flag_name]
])
else:
log.warning('Flag %s ignored on msdos', flag_name)
def _cleanup_intermediate_config(self):
# delete kiwi copied config files
config_files_to_delete = []
for config in self.cleanup_files:
config_files_to_delete.append(self.root_dir + config)
del self.cleanup_files[:]
# delete stale symlinks if there are any. normally the package
# installation process should have replaced the symlinks with
# real files from the packages
for config in self.config_files:
if os.path.islink(self.root_dir + config):
config_files_to_delete.append(self.root_dir + config)
try:
Command.run(['rm', '-f'] + config_files_to_delete)
except Exception as e:
log.warning('Failed to remove intermediate config files: %s',
format(e))
self._restore_intermediate_config_rpmnew_variants()
def setup_keyboard_map(self):
"""
Setup console keyboard
"""
if 'keytable' in self.preferences:
log.info('Setting up keytable: %s', self.preferences['keytable'])
if CommandCapabilities.has_option_in_help('systemd-firstboot',
'--keymap',
root=self.root_dir,
raise_on_error=False):
Path.wipe(self.root_dir + '/etc/vconsole.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--keymap=' + self.preferences['keytable']
])
elif os.path.exists(self.root_dir + '/etc/sysconfig/keyboard'):
Shell.run_common_function('baseUpdateSysConfig', [
self.root_dir + '/etc/sysconfig/keyboard', 'KEYTABLE',
'"' + self.preferences['keytable'] + '"'
])
else:
log.warning(
'keyboard setup skipped no capable '
'systemd-firstboot or etc/sysconfig/keyboard found')
def process(self): # noqa: C901
"""
Prepare and install a new system for chroot access
"""
self.manual = Help()
if self._help():
return
Privileges.check_for_root_permissions()
self.load_xml_description(self.command_args['--description'])
abs_root_path = os.path.abspath(self.command_args['--root'])
prepare_checks = self.checks_before_command_args
prepare_checks.update(
{'check_target_directory_not_in_shared_cache': [abs_root_path]})
self.run_checks(prepare_checks)
if self.command_args['--ignore-repos']:
self.xml_state.delete_repository_sections()
elif self.command_args['--ignore-repos-used-for-build']:
self.xml_state.delete_repository_sections_used_for_build()
if self.command_args['--set-repo']:
self.xml_state.set_repository(
*self.sextuple_token(self.command_args['--set-repo']))
if self.command_args['--add-repo']:
for add_repo in self.command_args['--add-repo']:
self.xml_state.add_repository(*self.sextuple_token(add_repo))
if self.command_args['--set-container-tag']:
self.xml_state.set_container_config_tag(
self.command_args['--set-container-tag'])
if self.command_args['--add-container-label']:
for add_label in self.command_args['--add-container-label']:
try:
(name, value) = add_label.split('=', 1)
self.xml_state.add_container_config_label(name, value)
except Exception:
log.warning('Container label {0} ignored. Invalid format: '
'expected labelname=value'.format(add_label))
if self.command_args['--set-container-derived-from']:
self.xml_state.set_derived_from_image_uri(
self.command_args['--set-container-derived-from'])
self.run_checks(self.checks_after_command_args)
log.info('Preparing system')
system = SystemPrepare(self.xml_state, abs_root_path,
self.command_args['--allow-existing-root'])
manager = system.setup_repositories(self.command_args['--clear-cache'],
self.command_args['--signing-key'])
system.install_bootstrap(manager,
self.command_args['--add-bootstrap-package'])
system.install_system(manager)
if self.command_args['--add-package']:
system.install_packages(manager,
self.command_args['--add-package'])
if self.command_args['--delete-package']:
system.delete_packages(manager,
self.command_args['--delete-package'])
profile = Profile(self.xml_state)
defaults = Defaults()
defaults.to_profile(profile)
setup = SystemSetup(self.xml_state, abs_root_path)
setup.import_shell_environment(profile)
setup.import_description()
setup.import_overlay_files()
setup.import_image_identifier()
setup.setup_groups()
setup.setup_users()
setup.setup_keyboard_map()
setup.setup_locale()
setup.setup_plymouth_splash()
setup.setup_timezone()
setup.setup_permissions()
# make sure manager instance is cleaned up now
del manager
# setup permanent image repositories after cleanup
setup.import_repositories_marked_as_imageinclude()
setup.call_config_script()
# handle uninstall package requests, gracefully uninstall
# with dependency cleanup
system.pinch_system(force=False)
# handle delete package requests, forced uninstall without
# any dependency resolution
system.pinch_system(force=True)
# delete any custom rpm macros created
Rpm(abs_root_path,
Defaults.get_custom_rpm_image_macro_name()).wipe_config()
# make sure system instance is cleaned up now
del system
def setup_repositories(self, clear_cache=False, signing_keys=None):
"""
Set up repositories for software installation and return a
package manager for performing software installation tasks
:param bool clear_cache: flag the clear cache before configure
anything
:param list signing_keys: keys imported to the package manager
:return: instance of :class:`PackageManager`
:rtype: PackageManager
"""
repository_options = []
repository_sections = \
self.xml_state.get_repository_sections_used_for_build()
package_manager = self.xml_state.get_package_manager()
rpm_locale_list = self.xml_state.get_rpm_locale()
if self.xml_state.get_rpm_check_signatures():
repository_options.append('check_signatures')
if self.xml_state.get_rpm_excludedocs():
repository_options.append('exclude_docs')
if rpm_locale_list:
repository_options.append('_install_langs%{0}'.format(
':'.join(rpm_locale_list)))
repo = Repository(self.root_bind, package_manager, repository_options)
repo.setup_package_database_configuration()
if signing_keys:
repo.import_trusted_keys(signing_keys)
for xml_repo in repository_sections:
repo_type = xml_repo.get_type()
repo_source = xml_repo.get_source().get_path()
repo_user = xml_repo.get_username()
repo_secret = xml_repo.get_password()
repo_alias = xml_repo.get_alias()
repo_priority = xml_repo.get_priority()
repo_dist = xml_repo.get_distribution()
repo_components = xml_repo.get_components()
repo_repository_gpgcheck = xml_repo.get_repository_gpgcheck()
repo_package_gpgcheck = xml_repo.get_package_gpgcheck()
repo_sourcetype = xml_repo.get_sourcetype()
log.info('Setting up repository %s', repo_source)
log.info('--> Type: {0}'.format(repo_type))
if repo_sourcetype:
log.info('--> SourceType: {0}'.format(repo_sourcetype))
if repo_priority:
log.info('--> Priority: {0}'.format(repo_priority))
uri = Uri(repo_source, repo_type)
repo_source_translated = uri.translate()
log.info('--> Translated: {0}'.format(repo_source_translated))
if not repo_alias:
repo_alias = uri.alias()
log.info('--> Alias: {0}'.format(repo_alias))
if not uri.is_remote() and not os.path.exists(
repo_source_translated):
log.warning('repository %s does not exist and will be skipped',
repo_source)
continue
if not uri.is_remote():
self.root_bind.mount_shared_directory(repo_source_translated)
repo.add_repo(repo_alias, repo_source_translated, repo_type,
repo_priority, repo_dist, repo_components, repo_user,
repo_secret, uri.credentials_file_name(),
repo_repository_gpgcheck, repo_package_gpgcheck,
repo_sourcetype)
if clear_cache:
repo.delete_repo_cache(repo_alias)
self.uri_list.append(uri)
repo.cleanup_unused_repos()
return PackageManager(repo, package_manager)
def get_disksize_mbytes(self):
"""
Precalculate disk size requirements in mbytes
:return: disk size mbytes
:rtype: int
"""
log.info('Precalculating required disk size')
calculated_disk_mbytes = 0
root_filesystem_mbytes = self.rootsize.customize(
self.rootsize.accumulate_mbyte_file_sizes(), self.filesystem
)
calculated_disk_mbytes += root_filesystem_mbytes
log.info(
'--> system data with filesystem overhead needs %s MB',
root_filesystem_mbytes
)
if self.volume_manager and self.volume_manager == 'lvm':
lvm_overhead_mbytes = Defaults.get_lvm_overhead_mbytes()
log.info(
'--> LVM overhead adding %s MB', lvm_overhead_mbytes
)
calculated_disk_mbytes += lvm_overhead_mbytes
volume_mbytes = self._accumulate_volume_size(
root_filesystem_mbytes
)
if volume_mbytes:
calculated_disk_mbytes += volume_mbytes
log.info(
'--> volume(s) size setup adding %s MB', volume_mbytes
)
legacy_bios_mbytes = self.firmware.get_legacy_bios_partition_size()
if legacy_bios_mbytes:
calculated_disk_mbytes += legacy_bios_mbytes
log.info(
'--> legacy bios boot partition adding %s MB',
legacy_bios_mbytes
)
boot_mbytes = self.boot_partition_size()
if boot_mbytes:
calculated_disk_mbytes += boot_mbytes
log.info(
'--> boot partition adding %s MB', boot_mbytes
)
if self.spare_part_mbytes:
calculated_disk_mbytes += self.spare_part_mbytes
log.info(
'--> spare partition adding %s MB', self.spare_part_mbytes
)
efi_mbytes = self.firmware.get_efi_partition_size()
if efi_mbytes:
calculated_disk_mbytes += efi_mbytes
log.info(
'--> EFI partition adding %s MB', efi_mbytes
)
prep_mbytes = self.firmware.get_prep_partition_size()
if prep_mbytes:
calculated_disk_mbytes += prep_mbytes
log.info(
'--> PReP partition adding %s MB', prep_mbytes
)
recovery_mbytes = self._inplace_recovery_partition_size()
if recovery_mbytes:
calculated_disk_mbytes += recovery_mbytes
log.info(
'--> In-place recovery partition adding: %s MB',
recovery_mbytes
)
if not self.configured_size:
log.info(
'Using calculated disk size: %d MB',
calculated_disk_mbytes
)
return calculated_disk_mbytes
elif self.configured_size.additive:
result_disk_mbytes = \
self.configured_size.mbytes + calculated_disk_mbytes
log.info(
'Using configured disk size: %d MB + %d MB calculated = %d MB',
self.configured_size.mbytes,
calculated_disk_mbytes,
result_disk_mbytes
)
return result_disk_mbytes
else:
log.info(
'Using configured disk size: %d MB',
self.configured_size.mbytes
)
if self.configured_size.mbytes < calculated_disk_mbytes:
log.warning(
'--> Configured size smaller than calculated size: %d MB',
calculated_disk_mbytes
)
return self.configured_size.mbytes
def get_disksize_mbytes(self):
"""
Precalculate disk size requirements in mbytes
:return: disk size mbytes
:rtype: int
"""
log.info('Precalculating required disk size')
calculated_disk_mbytes = 0
root_filesystem_mbytes = self.rootsize.customize(
self.rootsize.accumulate_mbyte_file_sizes(), self.filesystem)
calculated_disk_mbytes += root_filesystem_mbytes
log.info('--> system data with filesystem overhead needs %s MB',
root_filesystem_mbytes)
if self.volume_manager and self.volume_manager == 'lvm':
if self.build_type_name == 'vmx':
# only for vmx types we need to add the configured volume
# sizes. oem disks are self expandable and will resize to
# the configured sizes on first boot of the disk image
volume_mbytes = self._accumulate_volume_size(
root_filesystem_mbytes)
if volume_mbytes:
calculated_disk_mbytes += volume_mbytes
log.info('--> volume(s) size setup adding %s MB',
volume_mbytes)
legacy_bios_mbytes = self.firmware.get_legacy_bios_partition_size()
if legacy_bios_mbytes:
calculated_disk_mbytes += legacy_bios_mbytes
log.info('--> legacy bios boot partition adding %s MB',
legacy_bios_mbytes)
boot_mbytes = self.boot_partition_size()
if boot_mbytes:
calculated_disk_mbytes += boot_mbytes
log.info('--> boot partition adding %s MB', boot_mbytes)
if self.spare_part_mbytes:
calculated_disk_mbytes += self.spare_part_mbytes
log.info('--> spare partition adding %s MB',
self.spare_part_mbytes)
efi_mbytes = self.firmware.get_efi_partition_size()
if efi_mbytes:
calculated_disk_mbytes += efi_mbytes
log.info('--> EFI partition adding %s MB', efi_mbytes)
prep_mbytes = self.firmware.get_prep_partition_size()
if prep_mbytes:
calculated_disk_mbytes += prep_mbytes
log.info('--> PReP partition adding %s MB', prep_mbytes)
recovery_mbytes = self._inplace_recovery_partition_size()
if recovery_mbytes:
calculated_disk_mbytes += recovery_mbytes
log.info('--> In-place recovery partition adding: %s MB',
recovery_mbytes)
if not self.configured_size:
log.info('Using calculated disk size: %d MB',
calculated_disk_mbytes)
return calculated_disk_mbytes
elif self.configured_size.additive:
result_disk_mbytes = \
self.configured_size.mbytes + calculated_disk_mbytes
log.info(
'Using configured disk size: %d MB + %d MB calculated = %d MB',
self.configured_size.mbytes, calculated_disk_mbytes,
result_disk_mbytes)
return result_disk_mbytes
else:
log.info('Using configured disk size: %d MB',
self.configured_size.mbytes)
if self.configured_size.mbytes < calculated_disk_mbytes:
log.warning(
'--> Configured size smaller than calculated size: %d MB',
calculated_disk_mbytes)
return self.configured_size.mbytes
def create(self):
"""
Build a pxe image set consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum. The result can be used within the kiwi
PXE boot infrastructure
Image types which triggers this builder are:
* image="pxe"
"""
log.info('Creating PXE root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
self.filesystem_checksum = ''.join([self.image, '.md5'])
checksum = Checksum(self.image)
checksum.md5(self.filesystem_checksum)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[os.path.basename(self.image_name), '-', kernel_data.name]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
# put results into a tarball
Command.run(
[
'tar', '-C', self.target_dir, '-cJf', self.archive_name,
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.image),
os.path.basename(self.filesystem_checksum)
]
)
# store results
self.result.add(
key='pxe_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=False,
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
if self.pxedeploy:
log.warning(
'Creation of client config file from pxedeploy not implemented'
)
return self.result
def post_init(self, custom_args=None):
"""
Post initialization method
Store custom yum arguments and create runtime configuration
and environment
Attributes
* :attr:`shared_yum_dir`
shared directory between image root and build system root
* :attr:`runtime_yum_config_file`
yum runtime config file name
* :attr:`command_env`
customized os.environ for yum
* :attr:`runtime_yum_config`
Instance of ConfigParser
:param list custom_args: yum arguments
"""
self.custom_args = custom_args
if not custom_args:
self.custom_args = []
# extract custom arguments not used in yum call
if 'exclude_docs' in self.custom_args:
self.custom_args.remove('exclude_docs')
log.warning('rpm-excludedocs not supported for yum: ignoring')
if 'check_signatures' in self.custom_args:
self.custom_args.remove('check_signatures')
self.gpg_check = '1'
else:
self.gpg_check = '0'
self.repo_names = []
# yum support is based on creating repo files which contains
# path names to the repo and its cache. In order to allow a
# persistent use of the files in and outside of a chroot call
# an active bind mount from RootBind::mount_shared_directory
# is expected and required
manager_base = self.shared_location + '/yum'
self.shared_yum_dir = {
'reposd-dir': manager_base + '/repos',
'cache-dir': manager_base + '/cache',
'pluginconf-dir': manager_base + '/pluginconf'
}
self.runtime_yum_config_file = NamedTemporaryFile(
dir=self.root_dir
)
self.yum_args = [
'-c', self.runtime_yum_config_file.name, '-y'
] + self.custom_args
self.command_env = self._create_yum_runtime_environment()
# config file parameters for yum tool
self._create_runtime_config_parser()
self._create_runtime_plugin_config_parser()
self._write_runtime_config()
def process(self):
"""
Create result bundle from the image build results in the
specified target directory. Each result image will contain
the specified bundle identifier as part of its filename.
Uncompressed image files will also become xz compressed
and a sha sum will be created from every result image
"""
self.manual = Help()
if self._help():
return
# load serialized result object from target directory
result_directory = os.path.abspath(self.command_args['--target-dir'])
bundle_directory = os.path.abspath(self.command_args['--bundle-dir'])
if result_directory == bundle_directory:
raise KiwiBundleError(
'Bundle directory must be different from target directory')
log.info('Bundle build results from %s', result_directory)
result = Result.load(result_directory + '/kiwi.result')
image_version = result.xml_state.get_image_version()
image_name = result.xml_state.xml_data.get_name()
ordered_results = OrderedDict(sorted(result.get_results().items()))
# hard link bundle files, compress and build checksum
if not os.path.exists(bundle_directory):
Path.create(bundle_directory)
for result_file in list(ordered_results.values()):
if result_file.use_for_bundle:
bundle_file_basename = os.path.basename(result_file.filename)
# The bundle id is only taken into account for image results
# which contains the image version appended in its file name
part_name = list(bundle_file_basename.partition(image_name))
bundle_file_basename = ''.join([
part_name[0], part_name[1], part_name[2].replace(
image_version,
image_version + '-' + self.command_args['--id'])
])
log.info('Creating %s', bundle_file_basename)
bundle_file = ''.join(
[bundle_directory, '/', bundle_file_basename])
Command.run(['cp', result_file.filename, bundle_file])
if result_file.compress:
log.info('--> XZ compressing')
compress = Compress(bundle_file)
compress.xz(self.runtime_config.get_xz_options())
bundle_file = compress.compressed_filename
if self.command_args['--zsync-source'] and result_file.shasum:
# Files with a checksum are considered to be image files
# and are therefore eligible to be provided via the
# requested Partial/differential file download based on
# zsync
zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK)
if zsyncmake:
log.info('--> Creating zsync control file')
Command.run([
zsyncmake, '-e', '-u',
os.sep.join([
self.command_args['--zsync-source'],
os.path.basename(bundle_file)
]), '-o', bundle_file + '.zsync', bundle_file
])
else:
log.warning(
'--> zsyncmake missing, zsync setup skipped')
if result_file.shasum:
log.info('--> Creating SHA 256 sum')
checksum = Checksum(bundle_file)
with open(bundle_file + '.sha256', 'w') as shasum:
shasum.write('{0} {1}'.format(
checksum.sha256(), os.path.basename(bundle_file)))
