def _setup_secure_boot_efi_image(self, lookup_path):
"""
Provide the shim loader and the shim signed grub2 loader
in the required boot path. Normally this task is done by
the shim-install tool. However, shim-install does not exist
on all distributions and the script does not operate well
in e.g CD environments from which we generate live and/or
install media. Thus shim-install is used if possible at
install time of the bootloader because it requires access
to the target block device. In any other case this setup
code should act as the fallback solution
"""
log.warning(
'--> Running fallback setup for shim secure boot efi image'
)
if not lookup_path:
lookup_path = self.root_dir
shim_image = Defaults.get_shim_loader(lookup_path)
if not shim_image:
raise KiwiBootLoaderGrubSecureBootError(
'Microsoft signed shim loader not found'
)
grub_image = Defaults.get_signed_grub_loader(lookup_path)
if not grub_image:
raise KiwiBootLoaderGrubSecureBootError(
'Shim signed grub2 efi loader not found'
)
Command.run(
['cp', shim_image, self._get_efi_image_name()]
)
Command.run(
['cp', grub_image, self.efi_boot_path]
)
def _get_target_offset(self):
if self.target_table_type == 'dasd':
blocks = self._read_dasd_disk_geometry('blocks per track')
bash_command = [
'fdasd', '-f', '-s', '-p', self.target_device,
'|', 'head', '-n', '1', '|', 'tr', '-s', '" "'
]
fdasd_call = Command.run(
['bash', '-c', ' '.join(bash_command)]
)
fdasd_output = fdasd_call.output
try:
start_track = int(fdasd_output.split(' ')[2].lstrip())
except Exception:
raise KiwiDiskGeometryError(
'unknown partition format: %s' % fdasd_output
)
return start_track * blocks
else:
blocks = self._read_msdos_disk_geometry('blocks per track')
parted_call = Command.run(
['parted', '-m', self.target_device, 'unit', 's', 'print']
)
parted_output = parted_call.output.lstrip()
first_partition_format = re.search('1:(.*?)s', parted_output)
if not first_partition_format:
raise KiwiDiskGeometryError(
'unknown partition format: %s' % parted_output
)
start_track = int(first_partition_format.group(1))
return start_track * blocks
def create(self, name, mbsize, type_name, flags=None):
"""
Create DASD partition
:param string name: partition name
:param int mbsize: partition size
:param string type_name: unused
:param list flags: unused
"""
self.partition_id += 1
fdasd_input = NamedTemporaryFile()
with open(fdasd_input.name, 'w') as partition:
log.debug(
'%s: fdasd: n p cur_position +%sM w q',
name, format(mbsize)
)
if mbsize == 'all_free':
partition.write('n\np\n\n\nw\nq\n')
else:
partition.write('n\np\n\n+%dM\nw\nq\n' % mbsize)
bash_command = ' '.join(
['cat', fdasd_input.name, '|', 'fdasd', '-f', self.disk_device]
)
try:
Command.run(
['bash', '-c', bash_command]
)
except Exception:
# unfortunately fdasd reports that it can't read in the partition
# table which I consider a bug in fdasd. However the table was
# correctly created and therefore we continue. Problem is that we
# are not able to detect real errors with the fdasd operation at
# that point.
log.debug('potential fdasd errors were ignored')
def create_degraded_raid(self, raid_level):
"""
Create a raid array in degraded mode with one device missing.
This only works in the raid levels 0(striping) and 1(mirroring)
:param string raid_level: raid level name
"""
if raid_level not in self.raid_level_map:
raise KiwiRaidSetupError(
'Only raid levels 0(striping) and 1(mirroring) are supported'
)
raid_device = None
for raid_id in range(9):
raid_device = '/dev/md' + format(raid_id)
if os.path.exists(raid_device):
raid_device = None
else:
break
if not raid_device:
raise KiwiRaidSetupError(
'Could not find free raid device in range md0-8'
)
log.info(
'Creating raid array in %s mode as %s',
raid_level, raid_device
)
Command.run(
[
'mdadm', '--create', '--run', raid_device,
'--level', self.raid_level_map[raid_level],
'--raid-disks', '2',
self.storage_provider.get_device(), 'missing'
]
)
self.raid_device = raid_device
def process_delete_requests(self, force=False):
"""
Process package delete requests (chroot)
:param bool force: force deletion: true|false
"""
delete_items = []
for delete_item in self._delete_items():
try:
Command.run(['chroot', self.root_dir, 'rpm', '-q', delete_item])
delete_items.append(delete_item)
except Exception:
# ignore packages which are not installed
pass
if not delete_items:
raise KiwiRequestError(
'None of the requested packages to delete are installed'
)
if force:
force_options = ['--nodeps', '--allmatches', '--noscripts']
return Command.call(
[
'chroot', self.root_dir, 'rpm', '-e'
] + force_options + delete_items,
self.chroot_command_env
)
else:
return Command.call(
[
'chroot', self.root_dir, 'zypper'
] + self.chroot_zypper_args + [
'remove', '-u', '--force-resolution'
] + delete_items,
self.chroot_command_env
)
def resize_raw_disk(self, size_bytes):
"""
Resize raw disk image to specified size. If the request
would actually shrink the disk an exception is raised.
If the disk got changed the method returns True, if
the new size is the same as the current size nothing
gets resized and the method returns False
:param int size: new size in bytes
:rtype: bool
"""
current_byte_size = os.path.getsize(self.diskname)
size_bytes = int(size_bytes)
if size_bytes < current_byte_size:
raise KiwiResizeRawDiskError(
'shrinking {0} disk to {1} bytes corrupts the image'.format(
self.diskname, size_bytes
)
)
elif size_bytes == current_byte_size:
return False
Command.run(
['qemu-img', 'resize', self.diskname, format(size_bytes)]
)
return True
def setup_intermediate_config(self):
"""
Create intermediate config files
Some config files e.g etc/hosts needs to be temporarly copied
from the buildsystem host to the image root system in order to
allow e.g DNS resolution in the way as it is configured on the
buildsystem host. These config files only exists during the image
build process and are not part of the final image
:raises KiwiSetupIntermediateConfigError: if the management of
intermediate configuration files fails
"""
try:
for config in self.config_files:
if os.path.exists(config):
self.cleanup_files.append(config + '.kiwi')
Command.run(
['cp', config, self.root_dir + config + '.kiwi']
)
link_target = os.path.basename(config) + '.kiwi'
Command.run(
['ln', '-s', '-f', link_target, self.root_dir + config]
)
except Exception as e:
self.cleanup()
raise KiwiSetupIntermediateConfigError(
'%s: %s' % (type(e).__name__, format(e))
)
def create(self, name, mbsize, type_name, flags=None):
"""
Create GPT partition
:param string name: partition name
:param int mbsize: partition size
:param string type_name: partition type
:param list flags: additional flags
"""
self.partition_id += 1
if mbsize == 'all_free':
partition_end = '0'
else:
partition_end = '+' + format(mbsize) + 'M'
if self.partition_id > 1 or not self.start_sector:
# A start sector value of 0 specifies the default value
# defined in sgdisk
self.start_sector = 0
Command.run(
[
'sgdisk', '-n', ':'.join(
[
format(self.partition_id),
format(self.start_sector),
partition_end
]
), '-c', ':'.join([format(self.partition_id), name]),
self.disk_device
]
)
self.set_flag(self.partition_id, type_name)
if flags:
for flag_name in flags:
self.set_flag(self.partition_id, flag_name)
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image',
'{0}:base_layer'.format(self.oci_layout_dir), self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep])
)
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def create(self, source_dir, exclude=None):
"""
Create cpio archive
:param string source_dir: data source directory
:param list exclude: list of excluded items
"""
find_excludes = []
find_command = ['cd', source_dir, '&&', 'find', '.']
cpio_command = [
'cpio', '--quiet', '-o', '-H', 'newc', '>', self.filename
]
if exclude:
for path in exclude:
if find_excludes:
find_excludes.append('-or')
find_excludes.append('-path')
find_excludes.append('.' + path)
find_excludes.append('-prune')
find_excludes.append('-o')
find_excludes.append('-print')
find_command += find_excludes
bash_command = find_command + ['|'] + cpio_command
Command.run(
['bash', '-c', ' '.join(bash_command)]
)
def test_run_raises_error(self, mock_popen, mock_which):
mock_which.return_value = "command"
mock_process = mock.Mock()
mock_process.communicate = mock.Mock(return_value=[str.encode("stdout"), str.encode("stderr")])
mock_process.returncode = 1
mock_popen.return_value = mock_process
Command.run(["command", "args"])
def setup_locale(self):
"""
Setup UTF8 system wide locale
"""
if 'locale' in self.preferences:
if 'POSIX' in self.preferences['locale'].split(','):
locale = 'POSIX'
else:
locale = '{0}.UTF-8'.format(
self.preferences['locale'].split(',')[0]
)
log.info('Setting up locale: %s', self.preferences['locale'])
if CommandCapabilities.has_option_in_help(
'systemd-firstboot', '--locale',
root=self.root_dir, raise_on_error=False
):
Path.wipe(self.root_dir + '/etc/locale.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--locale=' + locale
])
elif os.path.exists(self.root_dir + '/etc/sysconfig/language'):
Shell.run_common_function(
'baseUpdateSysConfig', [
self.root_dir + '/etc/sysconfig/language',
'RC_LANG', locale
]
)
else:
log.warning(
'locale setup skipped no capable '
'systemd-firstboot or etc/sysconfig/language not found'
)
def uncompress(self, temporary=False):
"""
Uncompress with format autodetection
By default the original source file will be changed into
the uncompressed variant. If temporary is set to True
a temporary file is created instead
:param bool temporary: uncompress to a temporary file
"""
zipper = self.get_format()
if not zipper:
raise KiwiCompressionFormatUnknown(
'could not detect compression format for %s' %
self.source_filename
)
if not temporary:
Command.run([zipper, '-d', self.source_filename])
self.uncompressed_filename = self.source_filename
else:
self.temp_file = NamedTemporaryFile()
bash_command = [
zipper, '-c', '-d', self.source_filename,
'>', self.temp_file.name
]
Command.run(['bash', '-c', ' '.join(bash_command)])
self.uncompressed_filename = self.temp_file.name
return self.uncompressed_filename
def wipe(self):
"""
Zap (destroy) any GPT and MBR data structures if present
For DASD disks create a new VTOC table
"""
if 'dasd' in self.table_type:
log.debug('Initialize DASD disk with new VTOC table')
fdasd_input = NamedTemporaryFile()
with open(fdasd_input.name, 'w') as vtoc:
vtoc.write('y\n\nw\nq\n')
bash_command = ' '.join(
[
'cat', fdasd_input.name, '|',
'fdasd', '-f', self.storage_provider.get_device()
]
)
try:
Command.run(
['bash', '-c', bash_command]
)
except Exception:
# unfortunately fdasd reports that it can't read in the
# partition table which I consider a bug in fdasd. However
# the table was correctly created and therefore we continue.
# Problem is that we are not able to detect real errors
# with the fdasd operation at that point.
log.debug('potential fdasd errors were ignored')
else:
log.debug('Initialize %s disk', self.table_type)
Command.run(
[
'sgdisk', '--zap-all', self.storage_provider.get_device()
]
)
def setup_keyboard_map(self):
"""
Setup console keyboard
"""
if 'keytable' in self.preferences:
log.info(
'Setting up keytable: %s', self.preferences['keytable']
)
if CommandCapabilities.has_option_in_help(
'systemd-firstboot', '--keymap',
root=self.root_dir, raise_on_error=False
):
Path.wipe(self.root_dir + '/etc/vconsole.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--keymap=' + self.preferences['keytable']
])
elif os.path.exists(self.root_dir + '/etc/sysconfig/keyboard'):
Shell.run_common_function(
'baseUpdateSysConfig', [
self.root_dir + '/etc/sysconfig/keyboard', 'KEYTABLE',
'"' + self.preferences['keytable'] + '"'
]
)
else:
log.warning(
'keyboard setup skipped no capable '
'systemd-firstboot or etc/sysconfig/keyboard found'
)
def set_flag(self, partition_id, flag_name):
"""
Set msdos partition flag
:param int partition_id: partition number
:param string flag_name: name from flag map
"""
if flag_name not in self.flag_map:
raise KiwiPartitionerMsDosFlagError(
'Unknown partition flag %s' % flag_name
)
if self.flag_map[flag_name]:
if flag_name == 'f.active':
Command.run(
[
'parted', self.disk_device,
'set', format(partition_id), 'boot', 'on'
]
)
else:
Command.run(
[
'sfdisk', '-c', self.disk_device,
format(partition_id), self.flag_map[flag_name]
]
)
else:
log.warning('Flag %s ignored on msdos', flag_name)
def _copy_first_boot_files_to_system_image(self):
boot_names = self.boot_image.get_boot_names()
if self.initrd_system == 'kiwi':
log.info('Copy boot files to system image')
kernel = Kernel(self.boot_image.boot_root_directory)
log.info('--> boot image kernel as %s', boot_names.kernel_name)
kernel.copy_kernel(
self.root_dir, ''.join(['/boot/', boot_names.kernel_name])
)
if self.xen_server:
if kernel.get_xen_hypervisor():
log.info('--> boot image Xen hypervisor as xen.gz')
kernel.copy_xen_hypervisor(
self.root_dir, '/boot/xen.gz'
)
else:
raise KiwiDiskBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image.boot_root_directory
)
log.info('--> initrd archive as %s', boot_names.initrd_name)
Command.run(
[
'mv', self.boot_image.initrd_filename,
self.root_dir + ''.join(['/boot/', boot_names.initrd_name])
]
)
def _create_iso_install_kernel_and_initrd(self):
boot_path = self.media_dir + '/boot/' + self.arch + '/loader'
Path.create(boot_path)
kernel = Kernel(self.boot_image_task.boot_root_directory)
if kernel.get_kernel():
kernel.copy_kernel(boot_path, '/linux')
else:
raise KiwiInstallBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
if self.xml_state.is_xen_server():
if kernel.get_xen_hypervisor():
kernel.copy_xen_hypervisor(boot_path, '/xen.gz')
else:
raise KiwiInstallBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
if self.initrd_system == 'dracut':
self._create_dracut_install_config()
self._add_system_image_boot_options_to_boot_image()
self.boot_image_task.create_initrd(self.mbrid, 'initrd_kiwi_install')
Command.run(
[
'mv', self.boot_image_task.initrd_filename,
boot_path + '/initrd'
]
)
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
:return: process results in command type
:rtype: namedtuple
"""
yum = self._get_yum_binary_name()
Command.run(
[yum] + self.yum_args + ['makecache']
)
bash_command = [
yum
] + self.yum_args + [
'--installroot', self.root_dir
] + self.custom_args + ['install'] + self.package_requests
if self.collection_requests:
bash_command += [
'&&', yum
] + self.yum_args + [
'--installroot', self.root_dir
] + self.custom_args + ['groupinstall'] + self.collection_requests
self.cleanup_requests()
return Command.call(
['bash', '-c', ' '.join(bash_command)], self.command_env
)
def pack_image_to_file(self, filename):
"""
Packs the given oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
oci_image = os.sep.join([
self.oci_dir, ':'.join(['umoci_layout', self.container_tag])
])
additional_tags = []
for tag in self.additional_tags:
additional_tags.extend([
'--additional-tag', '{0}:{1}'.format(self.container_name, tag)
])
# make sure the target tar file does not exist
# skopeo doesn't support force overwrite
Path.wipe(filename)
Command.run(
[
'skopeo', 'copy', 'oci:{0}'.format(
oci_image
),
'docker-archive:{0}:{1}:{2}'.format(
filename, self.container_name, self.container_tag
)
] + additional_tags
)
container_compressor = self.runtime_config.get_container_compression()
if container_compressor:
compressor = Compress(filename)
return compressor.xz(self.runtime_config.get_xz_options())
else:
return filename
def create(self, name, mbsize, type_name, flags=None):
"""
Create GPT partition
:param string name: partition name
:param int mbsize: partition size
:param string type_name: partition type
:param list flags: additional flags
"""
self.partition_id += 1
if mbsize == 'all_free':
partition_end = '0'
else:
partition_end = '+' + format(mbsize) + 'M'
Command.run(
[
'sgdisk', '-n',
':'.join([format(self.partition_id), '0', partition_end]),
'-c', ':'.join([format(self.partition_id), name]),
self.disk_device
]
)
self.set_flag(self.partition_id, type_name)
if flags:
for flag_name in flags:
self.set_flag(self.partition_id, flag_name)
def process_install_requests(self):
"""
Process package install requests for image phase (chroot)
"""
yum = self._get_yum_binary_name(root=self.root_dir)
if self.exclude_requests:
# For Yum, excluding a package means removing it from
# the solver operation. This is done by adding --exclude
# to the command line. This means that if the package is
# hard required by another package, it will break the transaction.
for package in self.exclude_requests:
self.custom_args.append('--exclude=' + package)
Command.run(
['chroot', self.root_dir, 'rpm', '--rebuilddb']
)
chroot_yum_args = self.root_bind.move_to_root(
self.yum_args
)
bash_command = [
'chroot', self.root_dir, yum
] + chroot_yum_args + self.custom_args + [
'install'
] + self.package_requests
if self.collection_requests:
bash_command += [
'&&', 'chroot', self.root_dir, yum
] + chroot_yum_args + self.custom_args + [
'groupinstall'
] + self.collection_requests
self.cleanup_requests()
return Command.call(
['bash', '-c', ' '.join(bash_command)], self.command_env
)
def process_delete_requests(self, force=False):
"""
Process package delete requests (chroot)
:param bool force: force deletion: true|false
"""
delete_items = []
for delete_item in self.package_requests:
try:
Command.run(['chroot', self.root_dir, 'rpm', '-q', delete_item])
delete_items.append(delete_item)
except Exception:
# ignore packages which are not installed
pass
if not delete_items:
raise KiwiRequestError(
'None of the requested packages to delete are installed'
)
delete_options = ['--nodeps', '--allmatches', '--noscripts']
self.cleanup_requests()
return Command.call(
[
'chroot', self.root_dir, 'rpm', '-e'
] + delete_options + delete_items,
self.command_env
)
def process_install_requests(self):
"""
Process package install requests for image phase (chroot)
:return: process results in command type
:rtype: namedtuple
"""
if self.exclude_requests:
# For zypper excluding a package means, removing it from
# the solver operation. This is done by adding a package
# lock. This means that if the package is hard required
# by another package, it will break the transaction.
metadata_dir = ''.join([self.root_dir, '/etc/zypp'])
if not os.path.exists(metadata_dir):
Path.create(metadata_dir)
for package in self.exclude_requests:
Command.run(
['chroot', self.root_dir, 'zypper'] +
self.chroot_zypper_args + ['al'] + [package],
self.chroot_command_env
)
return Command.call(
['chroot', self.root_dir, 'zypper'] + self.chroot_zypper_args + [
'install', '--auto-agree-with-licenses'
] + self.custom_args + self._install_items(),
self.chroot_command_env
)
def create_xz_compressed(
self, source_dir, exclude=None, options=None, xz_options=None
):
"""
Create XZ compressed tar archive
:param string source_dir: data source directory
:param list exclude: list of excluded items
:param list options: custom tar creation options
:param list xz_options: custom xz compression options
"""
if not options:
options = []
if not xz_options:
xz_options = Defaults.get_xz_compression_options()
bash_command = [
'tar', '-C', source_dir
] + options + self.xattrs_options + [
'-c', '--to-stdout'
] + self._get_archive_items(source_dir, exclude) + [
'|', 'xz', '-f'
] + xz_options + [
'>', self.filename + '.xz'
]
Command.run(['bash', '-c', ' '.join(bash_command)])
return self.filename + '.xz'
def test_run_raises_error(self, mock_popen):
mock_process = mock.Mock()
mock_process.communicate = mock.Mock(
return_value=[str.encode('stdout'), str.encode('stderr')]
)
mock_process.returncode = 1
mock_popen.return_value = mock_process
Command.run(['command', 'args'])
def import_trusted_keys(self, signing_keys):
"""
Imports trusted keys into the image
:param list signing_keys: list of the key files to import
"""
for key in signing_keys:
Command.run(['rpm', '--root', self.root_dir, '--import', key])
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured'
)
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution
)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
dev_mount = MountManager(
device='/dev', mountpoint=self.root_dir + '/dev'
)
dev_mount.umount()
if self.repository.unauthenticated == 'false':
log.warning(
'KIWI does not support signature checks for apt-get '
'package manager during the bootstrap procedure, any '
'provided key will only be used inside the chroot '
'environment'
)
Command.run(
[
'debootstrap', '--no-check-gpg', self.distribution,
bootstrap_dir, self.distribution_path
], self.command_env
)
data = DataSync(
bootstrap_dir + '/', self.root_dir
)
data.sync_data(
options=['-a', '-H', '-X', '-A']
)
for key in self.repository.signing_keys:
Command.run([
'chroot', self.root_dir, 'apt-key', 'add', key
], self.command_env)
except Exception as e:
raise KiwiDebootstrapError(
'%s: %s' % (type(e).__name__, format(e))
)
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def __del__(self):
if self.node_name:
log.info('Cleaning up %s instance', type(self).__name__)
try:
Command.run(['losetup', '-d', self.node_name])
except Exception:
log.warning(
'loop device %s still busy', self.node_name
)
def _search_for(self, name, in_file):
search = '^' + name + ':'
try:
Command.run(
['chroot', self.root_dir, 'grep', '-q', search, in_file]
)
except Exception:
return False
return True
def _import_custom_scripts(self):
"""
Import custom scripts
"""
# custom_scripts defines a dictionary with all script hooks
# for each script name a the filepath and additional flags
# are defined. the filepath could be either a relative or
# absolute information. If filepath is set to None this indicates
# the script hook is not used. The raise_if_not_exists flag
# causes kiwi to raise an exception if a specified script
# filepath does not exist
script_type = namedtuple('script_type',
['filepath', 'raise_if_not_exists'])
custom_scripts = {
defaults.POST_BOOTSTRAP_SCRIPT:
script_type(filepath=defaults.POST_BOOTSTRAP_SCRIPT,
raise_if_not_exists=False),
defaults.POST_PREPARE_SCRIPT:
script_type(filepath=defaults.POST_PREPARE_SCRIPT,
raise_if_not_exists=False),
defaults.PRE_CREATE_SCRIPT:
script_type(filepath=defaults.PRE_CREATE_SCRIPT,
raise_if_not_exists=False),
defaults.POST_DISK_SYNC_SCRIPT:
script_type(filepath=defaults.POST_DISK_SYNC_SCRIPT,
raise_if_not_exists=False),
defaults.EDIT_BOOT_CONFIG_SCRIPT:
script_type(
filepath=self.xml_state.build_type.get_editbootconfig(),
raise_if_not_exists=True),
defaults.EDIT_BOOT_INSTALL_SCRIPT:
script_type(
filepath=self.xml_state.build_type.get_editbootinstall(),
raise_if_not_exists=True)
}
sorted_custom_scripts = OrderedDict(sorted(custom_scripts.items()))
script_target_dir = os.path.join(self.root_dir,
defaults.IMAGE_METADATA_DIR)
need_script_helper_functions = False
for name, script in list(sorted_custom_scripts.items()):
if script.filepath:
if script.filepath.startswith('/'):
script_file = script.filepath
else:
script_file = os.path.join(self.description_dir,
script.filepath)
if os.path.exists(script_file):
script_target_file = os.path.join(script_target_dir, name)
log.info('--> Importing {0} script to {1}'.format(
script.filepath, script_target_file))
Command.run(['cp', script_file, script_target_file])
need_script_helper_functions = True
elif script.raise_if_not_exists:
raise KiwiImportDescriptionError(
'Specified script {0} does not exist'.format(
script_file))
if need_script_helper_functions:
log.info('--> Importing script helper functions')
Command.run([
'cp',
Defaults.get_common_functions_file(),
self.root_dir + '/.kconfig'
])
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains:
* The raw system image xz compressed
* The raw system image checksum metadata file
* The append file template for the boot server
* The system image initrd for kexec
* The install initrd
* The kernel
Image types which triggers this builder are:
* installpxe="true|false"
"""
self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.',
dir=self.target_dir)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join([self.pxe_dir, '/', self.pxename, '.xz'])
compress = Compress(source_filename=self.diskname,
keep_source_on_compress=True)
compress.xz(self.xz_options)
Command.run(['mv', compress.compressed_filename, pxe_image_filename])
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join([self.pxe_dir, '/', self.pxename, '.md5'])
checksum = Checksum(self.diskname)
checksum.md5(pxe_md5_filename)
# the install image name is stored in a config file
if self.initrd_system == 'kiwi':
self._write_install_image_info_to_boot_image()
# the kexec required system image initrd is stored for dracut kiwi-dump
if self.initrd_system == 'dracut':
boot_names = self.boot_image_task.get_boot_names()
system_image_initrd = os.sep.join(
[self.root_dir, 'boot', boot_names.initrd_name])
target_initrd_name = '{0}/{1}.initrd'.format(
self.pxe_dir, self.pxename)
shutil.copy(system_image_initrd, target_initrd_name)
os.chmod(target_initrd_name, 420)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join([self.pxe_dir, '/', self.pxename, '.append'])
if self.initrd_system == 'kiwi':
cmdline = 'pxe=1'
else:
cmdline = ' '.join([
'rd.kiwi.install.pxe',
'rd.kiwi.install.image=http://example.com/image.xz'
])
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self._create_pxe_install_kernel_and_initrd()
# create pxe image bound boot config file, contents can be
# changed but presence is required.
log.info('Creating pxe install boot options file')
configname = '{0}.config.bootoptions'.format(self.pxename)
shutil.copy(os.sep.join([self.root_dir, 'config.bootoptions']),
os.sep.join([self.pxe_dir, configname]))
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(self.pxetarball)
archive.create(self.pxe_dir)
def install(self): # noqa: C901
"""
Install bootloader on disk device
"""
log.info('Installing grub2 on disk %s', self.device)
if self.target_removable:
self.install_arguments.append('--removable')
if Defaults.is_x86_arch(self.arch):
self.target = 'i386-pc'
self.install_device = self.device
self.modules = ' '.join(
Defaults.get_grub_bios_modules(multiboot=True)
)
self.install_arguments.append('--skip-fs-probe')
elif self.arch.startswith('ppc64'):
if not self.custom_args or 'prep_device' not in self.custom_args:
raise KiwiBootLoaderGrubInstallError(
'prep device name required for grub2 installation on ppc'
)
self.target = 'powerpc-ieee1275'
self.install_device = self.custom_args['prep_device']
self.modules = ' '.join(Defaults.get_grub_ofw_modules())
self.install_arguments.append('--skip-fs-probe')
self.install_arguments.append('--no-nvram')
elif self.arch.startswith('s390'):
self.target = 's390x-emu'
self.install_device = self.device
self.modules = ' '.join(Defaults.get_grub_s390_modules())
self.install_arguments.append('--skip-fs-probe')
self.install_arguments.append('--no-nvram')
else:
raise KiwiBootLoaderGrubPlatformError(
'host architecture %s not supported for grub2 installation' %
self.arch
)
self.root_mount = MountManager(
device=self.custom_args['root_device']
)
if 's390' in self.arch:
self.boot_mount = MountManager(
device=self.custom_args['boot_device'],
mountpoint=self.root_mount.mountpoint + '/boot/zipl'
)
else:
self.boot_mount = MountManager(
device=self.custom_args['boot_device'],
mountpoint=self.root_mount.mountpoint + '/boot'
)
if self.custom_args.get('efi_device'):
self.efi_mount = MountManager(
device=self.custom_args['efi_device'],
mountpoint=self.root_mount.mountpoint + '/boot/efi'
)
self.root_mount.mount()
if not self.root_mount.device == self.boot_mount.device:
self.boot_mount.mount()
if self.efi_mount:
self.efi_mount.mount()
if self.volumes:
for volume_path in Path.sort_by_hierarchy(
sorted(self.volumes.keys())
):
volume_mount = MountManager(
device=self.volumes[volume_path]['volume_device'],
mountpoint=self.root_mount.mountpoint + '/' + volume_path
)
self.volumes_mount.append(volume_mount)
volume_mount.mount(
options=[self.volumes[volume_path]['volume_options']]
)
self.device_mount = MountManager(
device='/dev',
mountpoint=self.root_mount.mountpoint + '/dev'
)
self.proc_mount = MountManager(
device='/proc',
mountpoint=self.root_mount.mountpoint + '/proc'
)
self.sysfs_mount = MountManager(
device='/sys',
mountpoint=self.root_mount.mountpoint + '/sys'
)
self.device_mount.bind_mount()
self.proc_mount.bind_mount()
self.sysfs_mount.bind_mount()
# check if a grub installation could be found in the image system
module_directory = Defaults.get_grub_path(
self.root_mount.mountpoint, self.target, raise_on_error=False
)
if not module_directory:
raise KiwiBootLoaderGrubDataError(
'No grub2 installation found in {0} for target {1}'.format(
self.root_mount.mountpoint, self.target
)
)
module_directory = module_directory.replace(
self.root_mount.mountpoint, ''
)
boot_directory = '/boot'
# wipe existing grubenv to allow the grub installer to create a new one
grubenv_glob = os.sep.join(
[self.root_mount.mountpoint, 'boot', '*', 'grubenv']
)
for grubenv in glob.glob(grubenv_glob):
Path.wipe(grubenv)
# install grub2 boot code
if self.firmware.get_partition_table_type() == 'dasd':
# On s390 and in CDL mode (4k DASD) the call of grub2-install
# does not work because grub2-install is not able to identify
# a 4k fdasd partitioned device as a grub supported device
# and fails. As grub2-install is only used to invoke
# grub2-zipl-setup and has no other job to do we can
# circumvent this problem by directly calling grub2-zipl-setup
# instead.
Command.run(
[
'chroot', self.root_mount.mountpoint,
'grub2-zipl-setup', '--keep'
]
)
zipl_config_file = ''.join(
[
self.root_mount.mountpoint, '/boot/zipl/config'
]
)
zipl2grub_config_file_orig = ''.join(
[
self.root_mount.mountpoint,
'/etc/default/zipl2grub.conf.in.orig'
]
)
if os.path.exists(zipl2grub_config_file_orig):
Command.run(
[
'mv', zipl2grub_config_file_orig,
zipl2grub_config_file_orig.replace('.orig', '')
]
)
if os.path.exists(zipl_config_file):
Command.run(
['mv', zipl_config_file, zipl_config_file + '.kiwi']
)
else:
Command.run(
[
'chroot', self.root_mount.mountpoint,
self._get_grub2_install_tool_name(
self.root_mount.mountpoint
)
] + self.install_arguments + [
'--directory', module_directory,
'--boot-directory', boot_directory,
'--target', self.target,
'--modules', self.modules,
self.install_device
]
)
if self.firmware and self.firmware.efi_mode() == 'uefi':
shim_install = self._get_shim_install_tool_name(
self.root_mount.mountpoint
)
# if shim-install does _not_ exist the fallback mechanism
# has applied at the bootloader/config level and we expect
# no further tool calls to be required
if shim_install:
# Before we call shim-install, the grub installer binary is
# replaced by a noop. Actually there is no reason for
# shim-install to call the grub installer because it should
# only setup the system for EFI secure boot which does not
# require any bootloader code in the master boot record.
# In addition kiwi has called the grub installer right
# before
self._disable_grub2_install(self.root_mount.mountpoint)
Command.run(
[
'chroot', self.root_mount.mountpoint,
'shim-install', '--removable',
self.install_device
]
)
# restore the grub installer noop
self._enable_grub2_install(self.root_mount.mountpoint)
def rebuild_database(self):
"""
Rebuild image rpm database taking current macro setup into account
"""
Command.run(['chroot', self.root_dir, 'rpmdb', '--rebuilddb'])
def _volume_group_in_use_on_host_system(self, volume_group_name):
vgs_call = Command.run(['vgs', '--noheadings', '-o', 'vg_name'])
for host_volume_group_name in vgs_call.output.split('\n'):
if volume_group_name in host_volume_group_name:
return True
return False
def test_call_command_does_not_exist(self):
Command.call(['does-not-exist'], os.environ)
def process(self):
"""
Prepare and install a new system for chroot access
"""
self.manual = Help()
if self._help():
return
Privileges.check_for_root_permissions()
self.load_xml_description(self.command_args['--description'],
self.global_args['--kiwi-file'])
abs_root_path = os.path.abspath(self.command_args['--root'])
prepare_checks = self.checks_before_command_args
prepare_checks.update(
{'check_target_directory_not_in_shared_cache': [abs_root_path]})
self.run_checks(prepare_checks)
if self.command_args['--ignore-repos']:
self.xml_state.delete_repository_sections()
elif self.command_args['--ignore-repos-used-for-build']:
self.xml_state.delete_repository_sections_used_for_build()
if self.command_args['--set-repo']:
self.xml_state.set_repository(
*self.sextuple_token(self.command_args['--set-repo']))
if self.command_args['--add-repo']:
for add_repo in self.command_args['--add-repo']:
self.xml_state.add_repository(*self.sextuple_token(add_repo))
if self.command_args['--set-container-tag']:
self.xml_state.set_container_config_tag(
self.command_args['--set-container-tag'])
if self.command_args['--add-container-label']:
for add_label in self.command_args['--add-container-label']:
try:
(name, value) = add_label.split('=', 1)
self.xml_state.add_container_config_label(name, value)
except Exception:
log.warning('Container label {0} ignored. Invalid format: '
'expected labelname=value'.format(add_label))
if self.command_args['--set-container-derived-from']:
self.xml_state.set_derived_from_image_uri(
self.command_args['--set-container-derived-from'])
self.run_checks(self.checks_after_command_args)
log.info('Preparing system')
system = SystemPrepare(self.xml_state, abs_root_path,
self.command_args['--allow-existing-root'])
manager = system.setup_repositories(
self.command_args['--clear-cache'],
self.command_args['--signing-key'] +
self.xml_state.get_repositories_signing_keys(),
self.global_args['--target-arch'])
run_bootstrap = True
if self.xml_state.get_package_manager() == 'apt' and \
self.command_args['--allow-existing-root']:
# try to call apt-get inside of the existing root.
# If the call succeeds we skip calling debootstrap again
# and assume the root to be ok to proceed with apt-get
# if it fails, treat the root as dirty and give the
# bootstrap a try
try:
Command.run(['chroot', abs_root_path, 'apt-get', '--version'])
run_bootstrap = False
log.warning('debootstrap will only be called once, skipped')
except Exception:
run_bootstrap = True
if run_bootstrap:
system.install_bootstrap(
manager, self.command_args['--add-bootstrap-package'])
setup = SystemSetup(self.xml_state, abs_root_path)
setup.import_description()
# call post_bootstrap.sh script if present
setup.call_post_bootstrap_script()
system.install_system(manager)
if self.command_args['--add-package']:
system.install_packages(manager,
self.command_args['--add-package'])
if self.command_args['--delete-package']:
system.delete_packages(manager,
self.command_args['--delete-package'])
profile = Profile(self.xml_state)
defaults = Defaults()
defaults.to_profile(profile)
profile.create(Defaults.get_profile_file(abs_root_path))
setup.import_overlay_files()
setup.import_image_identifier()
setup.setup_groups()
setup.setup_users()
setup.setup_keyboard_map()
setup.setup_locale()
setup.setup_plymouth_splash()
setup.setup_timezone()
setup.setup_permissions()
# make sure manager instance is cleaned up now
del manager
# setup permanent image repositories after cleanup
setup.import_repositories_marked_as_imageinclude()
# call config.sh script if present
setup.call_config_script()
# handle uninstall package requests, gracefully uninstall
# with dependency cleanup
system.pinch_system(force=False)
# handle delete package requests, forced uninstall without
# any dependency resolution
system.pinch_system(force=True)
# delete any custom rpm macros created
system.clean_package_manager_leftovers()
# make sure system instance is cleaned up now
del system
def test_run_does_not_raise_error_if_command_not_found(self, mock_which):
mock_which.return_value = None
result = Command.run(['command', 'args'], os.environ, False)
assert result.error is None
assert result.output is None
assert result.returncode == -1
def test_run_command_does_not_exist(self):
Command.run(['does-not-exist'])
def test_run_invalid_environment(self):
Command.run(['command', 'args'], {'HOME': '/root'})
def test_run_failure(self, mock_popen, mock_which):
mock_which.return_value = 'command'
mock_popen.side_effect = KiwiCommandError('Run failure')
Command.run(['command', 'args'])
def test_call_failure(self, mock_popen, mock_which):
mock_which.return_value = 'command'
mock_popen.side_effect = KiwiCommandError('Call failure')
Command.call(['command', 'args'])
def _copy_theme_data_to_boot_directory(self, lookup_path, target):
if not lookup_path:
lookup_path = self.boot_dir
font_name = 'unicode.pf2'
efi_font_dir = Defaults.get_grub_efi_font_directory(
lookup_path
)
boot_fonts_dir = os.path.normpath(
os.sep.join(
[
self.boot_dir,
self.get_boot_path(target),
self.boot_directory_name,
'fonts'
]
)
)
try:
unicode_font = Defaults.get_grub_path(
lookup_path, font_name
)
if not os.path.exists(os.sep.join([boot_fonts_dir, font_name])):
Path.create(boot_fonts_dir)
Command.run(
['cp', unicode_font, boot_fonts_dir]
)
if efi_font_dir:
Command.run(
['cp', unicode_font, efi_font_dir]
)
except Exception as issue:
raise KiwiBootLoaderGrubFontError(
'Setting up unicode font failed with {0}'.format(issue)
)
boot_theme_dir = os.sep.join(
[self.boot_dir, 'boot', self.boot_directory_name, 'themes']
)
Path.create(boot_theme_dir)
if self.theme:
theme_dir = Defaults.get_grub_path(
lookup_path, 'themes/' + self.theme, raise_on_error=False
)
boot_theme_background_file = self._find_theme_background_file(
lookup_path
)
if theme_dir and os.path.exists(theme_dir):
if boot_theme_background_file:
# A background file was found. Preserve a copy of the
# file which was created at install time of the theme
# package by the activate-theme script
boot_theme_background_backup_file = os.sep.join(
[self.boot_dir, 'background.png']
)
Command.run(
[
'cp', boot_theme_background_file,
boot_theme_background_backup_file
]
)
# sync theme data from install path to boot path
data = DataSync(
theme_dir, boot_theme_dir
)
data.sync_data(
options=['-a']
)
if boot_theme_background_file:
# Install preserved background file to the theme
Command.run(
[
'mv', boot_theme_background_backup_file,
os.sep.join([boot_theme_dir, self.theme])
]
)
elif boot_theme_background_file:
# assume all theme data is in the directory of the
# background file and just sync that directory to the
# boot path
data = DataSync(
os.path.dirname(boot_theme_background_file), boot_theme_dir
)
data.sync_data(
options=['-a']
)
self._check_boot_theme_exists()
def _get_modules(self) -> List[str]:
cmd = Command.run([
'chroot', self.boot_root_directory, 'dracut', '--list-modules',
'--no-kernel'
])
return cmd.output.splitlines()
def add_repo(self,
name,
uri,
repo_type='rpm-md',
prio=None,
dist=None,
components=None,
user=None,
secret=None,
credentials_file=None,
repo_gpgcheck=None,
pkg_gpgcheck=None):
"""
Add zypper repository
:param str name: repository name
:param str uri: repository URI
:param repo_type: repostory type name
:param int prio: zypper repostory priority
:param dist: unused
:param components: unused
:param user: credentials username
:param secret: credentials password
:param credentials_file: zypper credentials file
:param bool repo_gpgcheck: enable repository signature validation
:param bool pkg_gpgcheck: enable package signature validation
"""
if credentials_file:
repo_secret = os.sep.join(
[self.shared_zypper_dir['credentials-dir'], credentials_file])
if os.path.exists(repo_secret):
Path.wipe(repo_secret)
if user and secret:
uri = ''.join([uri, '?credentials=', credentials_file])
with open(repo_secret, 'w') as credentials:
credentials.write('username={0}{1}'.format(
user, os.linesep))
credentials.write('password={0}{1}'.format(
secret, os.linesep))
repo_file = ''.join(
[self.shared_zypper_dir['reposd-dir'], '/', name, '.repo'])
self.repo_names.append(''.join([name, '.repo']))
if os.path.exists(repo_file):
Path.wipe(repo_file)
self._backup_package_cache()
zypper_addrepo_command = ['zypper'] + self.zypper_args + [
'--root', self.root_dir, 'addrepo', '--refresh', '--keep-packages'
if Uri(uri).is_remote() else '--no-keep-packages', '--no-check',
uri, name
]
try:
Command.run(zypper_addrepo_command, self.command_env)
except Exception:
# for whatever reason zypper sometimes failes with
# a 'failed to cache rpm database' error. I could not
# find any reason why and a simple recall of the exact
# same command in the exact same environment works.
# Thus the stupid but simple workaround to this problem
# is try one recall before really failing
Command.run(zypper_addrepo_command, self.command_env)
if prio or repo_gpgcheck is not None or pkg_gpgcheck is not None:
repo_config = ConfigParser()
repo_config.read(repo_file)
if repo_gpgcheck is not None:
repo_config.set(name, 'repo_gpgcheck',
'1' if repo_gpgcheck else '0')
if pkg_gpgcheck is not None:
repo_config.set(name, 'pkg_gpgcheck',
'1' if pkg_gpgcheck else '0')
if prio:
repo_config.set(name, 'priority', format(prio))
with open(repo_file, 'w') as repo:
repo_config.write(repo)
self._restore_package_cache()
def create_install_iso(self):
"""
Create an install ISO from the disk_image as hybrid ISO
bootable via legacy BIOS, EFI and as disk from Stick
Image types which triggers this builder are:
* installiso="true|false"
* installstick="true|false"
"""
self.media_dir = mkdtemp(prefix='kiwi_install_media.',
dir=self.target_dir)
# unpack cdroot user files to media dir
self.setup.import_cdroot_files(self.media_dir)
# custom iso metadata
self.custom_iso_args = {
'meta_data': {
'volume_id': self.iso_volume_id,
'mbr_id': self.mbrid.get_id(),
'efi_mode': self.firmware.efi_mode(),
'ofw_mode': self.firmware.ofw_mode()
}
}
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
self.squashed_contents = mkdtemp(prefix='kiwi_install_squashfs.',
dir=self.target_dir)
checksum = Checksum(self.diskname)
checksum.md5(self.squashed_contents + '/' + self.md5name)
# the system image name is stored in a config file
self._write_install_image_info_to_iso_image()
if self.initrd_system == 'kiwi':
self._write_install_image_info_to_boot_image()
# the system image is stored as squashfs embedded file
log.info('Creating squashfs embedded disk image')
Command.run([
'cp', '-l', self.diskname,
self.squashed_contents + '/' + self.squashed_diskname
])
squashed_image_file = ''.join(
[self.target_dir, '/', self.squashed_diskname, '.squashfs'])
squashed_image = FileSystemSquashFs(
device_provider=None,
root_dir=self.squashed_contents,
custom_args={
'compression':
self.xml_state.build_type.get_squashfscompression()
})
squashed_image.create_on_file(squashed_image_file)
Command.run(['mv', squashed_image_file, self.media_dir])
log.info('Setting up install image bootloader configuration')
if self.firmware.efi_mode():
# setup bootloader config to boot the ISO via EFI
# This also embedds an MBR and the respective BIOS modules
# for compat boot. The complete bootloader setup will be
# based on grub
bootloader_config = BootLoaderConfig(
'grub2',
self.xml_state,
root_dir=self.root_dir,
boot_dir=self.media_dir,
custom_args={
'grub_directory_name':
Defaults.get_grub_boot_directory_name(self.root_dir)
})
bootloader_config.setup_install_boot_images(
mbrid=self.mbrid,
lookup_path=self.boot_image_task.boot_root_directory)
else:
# setup bootloader config to boot the ISO via isolinux.
# This allows for booting on x86 platforms in BIOS mode
# only.
bootloader_config = BootLoaderConfig('isolinux',
self.xml_state,
root_dir=self.root_dir,
boot_dir=self.media_dir)
IsoToolsBase.setup_media_loader_directory(
self.boot_image_task.boot_root_directory, self.media_dir,
bootloader_config.get_boot_theme())
bootloader_config.write_meta_data()
bootloader_config.setup_install_image_config(mbrid=self.mbrid)
bootloader_config.write()
# create initrd for install image
log.info('Creating install image boot image')
self._create_iso_install_kernel_and_initrd()
# the system image initrd is stored to allow kexec
self._copy_system_image_initrd_to_iso_image()
# create iso filesystem from media_dir
log.info('Creating ISO filesystem')
iso_image = FileSystemIsoFs(device_provider=None,
root_dir=self.media_dir,
custom_args=self.custom_iso_args)
iso_image.create_on_file(self.isoname)
def init_database(self):
Command.run(['rpm', '--root', self.root_dir, '--initdb'])
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
:raises KiwiDebootstrapError: if no main distribution repository
is configured, if the debootstrap script is not found or if the
debootstrap script execution fails
:return: process results in command type
:rtype: namedtuple
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured'
)
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution
)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
cmd = ['debootstrap']
if self.repository.unauthenticated == 'false' and \
os.path.exists(self.repository.keyring):
cmd.append('--keyring={}'.format(self.repository.keyring))
else:
cmd.append('--no-check-gpg')
if self.deboostrap_minbase:
cmd.append('--variant=minbase')
if self.repository.components:
cmd.append(
'--components={0}'.format(
','.join(self.repository.components)
)
)
cmd.extend([
self.distribution, bootstrap_dir, self.distribution_path
])
Command.run(cmd, self.command_env)
data = DataSync(
bootstrap_dir + '/', self.root_dir
)
data.sync_data(
options=['-a', '-H', '-X', '-A'],
exclude=['proc', 'sys', 'dev']
)
except Exception as e:
raise KiwiDebootstrapError(
'%s: %s' % (type(e).__name__, format(e))
)
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def add_repo(self,
name,
uri,
repo_type='rpm-md',
prio=None,
dist=None,
components=None,
user=None,
secret=None,
credentials_file=None,
repo_gpgcheck=None,
pkg_gpgcheck=None):
"""
Add zypper repository
:param str name: repository name
:param str uri: repository URI
:param repo_type: repostory type name
:param int prio: zypper repostory priority
:param dist: unused
:param components: unused
:param user: credentials username
:param secret: credentials password
:param credentials_file: zypper credentials file
:param bool repo_gpgcheck: enable repository signature validation
:param bool pkg_gpgcheck: enable package signature validation
"""
if credentials_file:
repo_secret = os.sep.join(
[self.shared_zypper_dir['credentials-dir'], credentials_file])
if os.path.exists(repo_secret):
Path.wipe(repo_secret)
if user and secret:
uri = ''.join([uri, '?credentials=', credentials_file])
with open(repo_secret, 'w') as credentials:
credentials.write('username={0}{1}'.format(
user, os.linesep))
credentials.write('password={0}{1}'.format(
secret, os.linesep))
repo_file = ''.join(
[self.shared_zypper_dir['reposd-dir'], '/', name, '.repo'])
self.repo_names.append(''.join([name, '.repo']))
if os.path.exists(repo_file):
Path.wipe(repo_file)
self._backup_package_cache()
Command.run(['zypper'] + self.zypper_args + [
'--root', self.root_dir, 'addrepo', '--refresh', '--type',
self._translate_repo_type(repo_type), '--keep-packages'
if Uri(uri).is_remote() else '--no-keep-packages', '-C', uri, name
], self.command_env)
if prio or repo_gpgcheck is not None or pkg_gpgcheck is not None:
repo_config = ConfigParser()
repo_config.read(repo_file)
if repo_gpgcheck is not None:
repo_config.set(name, 'repo_gpgcheck',
'1' if repo_gpgcheck else '0')
if pkg_gpgcheck is not None:
repo_config.set(name, 'pkg_gpgcheck',
'1' if pkg_gpgcheck else '0')
if prio:
repo_config.set(name, 'priority', format(prio))
with open(repo_file, 'w') as repo:
repo_config.write(repo)
self._restore_package_cache()
def _setup_default_grub(self): # noqa: C901
"""
Create or update etc/default/grub by parameters required
according to the root filesystem setup
* GRUB_TIMEOUT
* SUSE_BTRFS_SNAPSHOT_BOOTING
* GRUB_BACKGROUND
* GRUB_THEME
* GRUB_USE_LINUXEFI
* GRUB_USE_INITRDEFI
* GRUB_SERIAL_COMMAND
* GRUB_CMDLINE_LINUX_DEFAULT
* GRUB_GFXMODE
* GRUB_TERMINAL
* GRUB_DISTRIBUTOR
"""
grub_default_entries = {
'GRUB_TIMEOUT': self.timeout,
'GRUB_GFXMODE': self.gfxmode,
'GRUB_TERMINAL': '"{0}"'.format(self.terminal),
'GRUB_DISTRIBUTOR': '"{0}"'.format(self.get_menu_entry_title())
}
if self.cmdline:
grub_default_entries[
'GRUB_CMDLINE_LINUX_DEFAULT'] = '"{0}"'.format(
re.sub(r'root=.* |root=.*$', '', self.cmdline).strip())
if self.terminal and 'serial' in self.terminal:
serial_format = '"serial {0} {1} {2} {3} {4}"'
grub_default_entries['GRUB_SERIAL_COMMAND'] = serial_format.format(
'--speed=38400', '--unit=0', '--word=8', '--parity=no',
'--stop=1')
if self.theme:
theme_setup = '{0}/{1}/theme.txt'
grub_default_entries['GRUB_THEME'] = theme_setup.format(
''.join(['/boot/', self.boot_directory_name, '/themes']),
self.theme)
theme_background = '{0}/{1}/background.png'.format(
''.join(['/boot/', self.boot_directory_name, '/themes']),
self.theme)
if os.path.exists(os.sep.join([self.root_dir, theme_background])):
grub_default_entries['GRUB_BACKGROUND'] = theme_background
if self.firmware.efi_mode():
# linuxefi/initrdefi only exist on x86, others always use efi
if self.arch == 'ix86' or self.arch == 'x86_64':
use_linuxefi_implemented = Command.run([
'grep', '-q', 'GRUB_USE_LINUXEFI',
Defaults.get_grub_config_tool()
],
raise_on_error=False)
if use_linuxefi_implemented.returncode == 0:
grub_default_entries['GRUB_USE_LINUXEFI'] = 'true'
grub_default_entries['GRUB_USE_INITRDEFI'] = 'true'
if self.xml_state.build_type.get_btrfs_root_is_snapshot():
grub_default_entries['SUSE_BTRFS_SNAPSHOT_BOOTING'] = 'true'
if self.custom_args.get('boot_is_crypto'):
grub_default_entries['GRUB_ENABLE_CRYPTODISK'] = 'y'
enable_blscfg_implemented = Command.run([
'grep', '-q', 'GRUB_ENABLE_BLSCFG',
Defaults.get_grub_config_tool()
],
raise_on_error=False)
if enable_blscfg_implemented.returncode == 0:
grub_default_entries['GRUB_ENABLE_BLSCFG'] = 'true'
if grub_default_entries:
log.info('Writing grub2 defaults file')
grub_default_location = ''.join([self.root_dir, '/etc/default/'])
if os.path.exists(grub_default_location):
grub_default_file = ''.join([grub_default_location, 'grub'])
grub_default = SysConfig(grub_default_file)
grub_default_entries_sorted = OrderedDict(
sorted(grub_default_entries.items()))
for key, value in list(grub_default_entries_sorted.items()):
log.info('--> {0}:{1}'.format(key, value))
grub_default[key] = value
grub_default.write()
def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
self.oci_dir = mkdtemp(prefix='kiwi_oci_dir.')
self.oci_root_dir = mkdtemp(prefix='kiwi_oci_root_dir.')
container_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
container_name = ':'.join([container_dir, self.container_tag])
if base_image:
Path.create(container_dir)
image_tar = ArchiveTar(base_image)
image_tar.extract(container_dir)
Command.run([
'umoci', 'config', '--image',
'{0}:base_layer'.format(container_dir), '--tag',
self.container_tag
])
else:
Command.run(['umoci', 'init', '--layout', container_dir])
Command.run(['umoci', 'new', '--image', container_name])
Command.run(
['umoci', 'unpack', '--image', container_name, self.oci_root_dir])
oci_root = DataSync(''.join([self.root_dir, os.sep]),
os.sep.join([self.oci_root_dir, 'rootfs']))
oci_root.sync_data(options=['-a', '-H', '-X', '-A', '--delete'],
exclude=exclude_list)
Command.run(
['umoci', 'repack', '--image', container_name, self.oci_root_dir])
for tag in self.additional_tags:
Command.run(
['umoci', 'config', '--image', container_name, '--tag', tag])
Command.run(['umoci', 'config'] + self.maintainer + self.user +
self.workingdir + self.entry_command +
self.entry_subcommand + self.expose_ports + self.volumes +
self.environment + self.labels + [
'--image', container_name, '--created',
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00')
])
Command.run(['umoci', 'gc', '--layout', container_dir])
return self.pack_image_to_file(filename)
def test_call_command_does_not_exist(self):
with raises(KiwiCommandNotFound):
Command.call(['does-not-exist'], os.environ)
def cleanup(self):
"""
Delete all traces of a kiwi description which are not
required in the later image
"""
Command.run(['rm', '-r', '-f', '/.kconfig', '/image'])
def test_run_command_does_not_exist(self):
with raises(KiwiCommandNotFound):
Command.run(['does-not-exist'])
def setup_disk_image_config(self,
boot_uuid=None,
root_uuid=None,
hypervisor=None,
kernel=None,
initrd=None,
boot_options={}):
"""
Create grub2 config file to boot from disk using grub2-mkconfig
:param string boot_uuid: unused
:param string root_uuid: unused
:param string hypervisor: unused
:param string kernel: unused
:param string initrd: unused
:param dict boot_options:
options dictionary that has to contain the root and boot
device and optional volume configuration. KIWI has to
mount the system prior to run grub2-mkconfig.
.. code:: python
{
'root_device': string,
'boot_device': string,
'efi_device': string,
'system_volumes': volume_manager_instance.get_volumes()
}
"""
self._mount_system(boot_options.get('root_device'),
boot_options.get('boot_device'),
boot_options.get('efi_device'),
boot_options.get('system_volumes'))
config_file = os.sep.join([
self.root_mount.mountpoint, 'boot', self.boot_directory_name,
'grub.cfg'
])
Command.run([
'chroot', self.root_mount.mountpoint,
self._get_grub2_mkconfig_tool(), '-o',
config_file.replace(self.root_mount.mountpoint, '')
])
if self.root_reference:
if self.root_filesystem_is_overlay or \
Defaults.is_buildservice_worker():
# grub2-mkconfig has no idea how the correct root= setup is
# for disk images created with overlayroot enabled or in a
# buildservice worker environment. Because of that the mkconfig
# tool just finds the raw partition loop device and includes it
# which is wrong. In this particular case we have to patch the
# written config file and replace the wrong root= reference with
# the correct value.
with open(config_file) as grub_config_file:
grub_config = grub_config_file.read()
grub_config = grub_config.replace(
'root={0}'.format(boot_options.get('root_device')),
self.root_reference)
with open(config_file, 'w') as grub_config_file:
grub_config_file.write(grub_config)
if self.firmware.efi_mode():
self._copy_grub_config_to_efi_path(self.efi_mount.mountpoint,
config_file)
def _create_passwd_hash(self, password):
openssl = Command.run(
['openssl', 'passwd', '-1', '-salt', 'xyz', password])
return openssl.output[:-1]
def post_process(self):
"""
Cleanup unused data from operations
"""
Command.run(['umoci', 'gc', '--layout', self.container_dir])
def process_install_requests_bootstrap(self, root_bind=None):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
:param object root_bind: instance of RootBind to manage kernel
file systems before debootstrap call
:raises KiwiDebootstrapError: if no main distribution repository
is configured, if the debootstrap script is not found or if the
debootstrap script execution fails
:return: process results in command type
:rtype: namedtuple
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured')
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution)
# APT package manager does not support bootstrapping. To circumvent
# this limitation there is the debootstrap tool for APT based distros.
# Because of that there is a little overlap between KIWI and
# debootstrap. Debootstrap manages itself the kernel file systems for
# chroot environment, thus we need to umount the kernel file systems
# before calling debootstrap and remount them afterwards.
root_bind.umount_kernel_file_systems()
# debootsrap will create its own dev/fd device
os.unlink(os.path.normpath(os.sep.join([self.root_dir, 'dev/fd'])))
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
cmd = ['debootstrap']
if self.repository.unauthenticated == 'false' and \
os.path.exists(self.repository.keyring):
cmd.append('--keyring={}'.format(self.repository.keyring))
else:
cmd.append('--no-check-gpg')
if self.deboostrap_minbase:
cmd.append('--variant=minbase')
if self.package_requests:
cmd.append('--include={}'.format(','.join(
self.package_requests)))
if self.repository.components:
cmd.append('--components={0}'.format(','.join(
self.repository.components)))
self.cleanup_requests()
cmd.extend(
[self.distribution, self.root_dir, self.distribution_path])
return Command.call(cmd, self.command_env)
except Exception as e:
raise KiwiDebootstrapError('%s: %s' %
(type(e).__name__, format(e)))
def process(self):
"""
Create result bundle from the image build results in the
specified target directory. Each result image will contain
the specified bundle identifier as part of its filename.
Uncompressed image files will also become xz compressed
and a sha sum will be created from every result image
"""
self.manual = Help()
if self._help():
return
# load serialized result object from target directory
result_directory = os.path.abspath(self.command_args['--target-dir'])
bundle_directory = os.path.abspath(self.command_args['--bundle-dir'])
if result_directory == bundle_directory:
raise KiwiBundleError(
'Bundle directory must be different from target directory'
)
log.info(
'Bundle build results from %s', result_directory
)
result = Result.load(
result_directory + '/kiwi.result'
)
image_version = result.xml_state.get_image_version()
image_name = result.xml_state.xml_data.get_name()
ordered_results = OrderedDict(sorted(result.get_results().items()))
# hard link bundle files, compress and build checksum
if not os.path.exists(bundle_directory):
Path.create(bundle_directory)
for result_file in list(ordered_results.values()):
if result_file.use_for_bundle:
bundle_file_basename = os.path.basename(result_file.filename)
# The bundle id is only taken into account for image results
# which contains the image version appended in its file name
part_name = list(bundle_file_basename.partition(image_name))
bundle_file_basename = ''.join([
part_name[0], part_name[1],
part_name[2].replace(
image_version,
image_version + '-' + self.command_args['--id']
)
])
log.info('Creating %s', bundle_file_basename)
bundle_file = ''.join(
[bundle_directory, '/', bundle_file_basename]
)
Command.run(
[
'cp', result_file.filename, bundle_file
]
)
if result_file.compress:
log.info('--> XZ compressing')
compress = Compress(bundle_file)
compress.xz(self.runtime_config.get_xz_options())
bundle_file = compress.compressed_filename
if self.command_args['--zsync-source'] and result_file.shasum:
# Files with a checksum are considered to be image files
# and are therefore eligible to be provided via the
# requested Partial/differential file download based on
# zsync
zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK)
if zsyncmake:
log.info('--> Creating zsync control file')
Command.run(
[
zsyncmake, '-e', '-u', os.sep.join(
[
self.command_args['--zsync-source'],
os.path.basename(bundle_file)
]
), '-o', bundle_file + '.zsync', bundle_file
]
)
else:
log.warning(
'--> zsyncmake missing, zsync setup skipped'
)
if result_file.shasum:
log.info('--> Creating SHA 256 sum')
checksum = Checksum(bundle_file)
with open(bundle_file + '.sha256', 'w') as shasum:
shasum.write(
'{0} {1}'.format(
checksum.sha256(), bundle_file_basename
)
)
def create_recovery_archive(self) -> None:
"""
Create a compressed recovery archive from the root tree
for use with kiwi's recvoery system. The method creates
additional data into the image root filesystem which is
deleted prior to the creation of a new recovery data set
"""
# cleanup
bash_comand = ['rm', '-f', self.root_dir + '/recovery.*']
Command.run(['bash', '-c', ' '.join(bash_comand)])
if not self.oemconfig['recovery']:
return
# recovery.tar
log.info('Creating recovery tar archive')
metadata = {
'archive_name': self.root_dir + '/recovery.tar',
'archive_filecount': self.root_dir + '/recovery.tar.files',
'archive_size': self.root_dir + '/recovery.tar.size',
'partition_size': self.root_dir + '/recovery.partition.size',
'partition_filesystem': self.root_dir + '/recovery.tar.filesystem'
}
pathlib.Path(metadata['archive_name']).touch()
archive = ArchiveTar(filename=metadata['archive_name'],
create_from_file_list=False)
archive.create(source_dir=self.root_dir,
exclude=['dev', 'proc', 'sys'],
options=[
'--numeric-owner', '--hard-dereference',
'--preserve-permissions'
])
# recovery.tar.filesystem
recovery_filesystem = self.xml_state.build_type.get_filesystem()
with open(metadata['partition_filesystem'], 'w') as partfs:
partfs.write('{0}'.format(recovery_filesystem))
log.info('--> Recovery partition filesystem: {0}'.format(
recovery_filesystem))
# recovery.tar.files
bash_comand = ['tar', '-tf', metadata['archive_name'], '|', 'wc', '-l']
tar_files_call = Command.run(['bash', '-c', ' '.join(bash_comand)])
tar_files_count = int(tar_files_call.output.rstrip('\n'))
with open(metadata['archive_filecount'], 'w') as files:
files.write('{0}{1}'.format(tar_files_count, os.linesep))
log.info('--> Recovery file count: {0} files'.format(tar_files_count))
# recovery.tar.size
recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
with open(metadata['archive_size'], 'w') as size:
size.write('{0}'.format(recovery_archive_size_bytes))
log.info('--> Recovery uncompressed size: {0} mbytes'.format(
int(recovery_archive_size_bytes / 1048576)))
# recovery.tar.gz
log.info('--> Compressing recovery archive')
compress = Compress(self.root_dir + '/recovery.tar')
compress.gzip()
# recovery.partition.size
recovery_archive_gz_size_mbytes = int(
os.path.getsize(metadata['archive_name'] + '.gz') / 1048576)
recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
+ Defaults.get_recovery_spare_mbytes()
with open(metadata['partition_size'], 'w') as gzsize:
gzsize.write('{0}'.format(recovery_partition_mbytes))
log.info('--> Recovery partition size: {0} mbytes'.format(
recovery_partition_mbytes))
# delete recovery archive if inplace recovery is requested
# In this mode the recovery archive is created at install time
# and not at image creation time. However the recovery metadata
# is preserved in order to be able to check if enough space
# is available on the disk to create the recovery archive.
if self.oemconfig['recovery_inplace']:
log.info('--> Inplace recovery requested, deleting archive')
Path.wipe(metadata['archive_name'] + '.gz')
