def _get_target_offset(self):
if self.target_table_type == 'dasd':
blocks = self._read_dasd_disk_geometry('blocks per track')
bash_command = [
'fdasd', '-f', '-s', '-p', self.target_device,
'|', 'head', '-n', '1', '|', 'tr', '-s', '" "'
]
fdasd_call = Command.run(
['bash', '-c', ' '.join(bash_command)]
)
fdasd_output = fdasd_call.output
try:
start_track = int(fdasd_output.split(' ')[2].lstrip())
except Exception:
raise KiwiDiskGeometryError(
'unknown partition format: %s' % fdasd_output
)
return start_track * blocks
else:
blocks = self._read_msdos_disk_geometry('blocks per track')
parted_call = Command.run(
['parted', '-m', self.target_device, 'unit', 's', 'print']
)
parted_output = parted_call.output.lstrip()
first_partition_format = re.search('1:(.*?)s', parted_output)
if not first_partition_format:
raise KiwiDiskGeometryError(
'unknown partition format: %s' % parted_output
)
start_track = int(first_partition_format.group(1))
return start_track * blocks
def _setup_secure_boot_efi_image(self, lookup_path):
"""
Provide the shim loader and the shim signed grub2 loader
in the required boot path. Normally this task is done by
the shim-install tool. However, shim-install does not exist
on all distributions and the script does not operate well
in e.g CD environments from which we generate live and/or
install media. Thus shim-install is used if possible at
install time of the bootloader because it requires access
to the target block device. In any other case this setup
code should act as the fallback solution
"""
log.warning(
'--> Running fallback setup for shim secure boot efi image'
)
if not lookup_path:
lookup_path = self.root_dir
shim_image = Defaults.get_shim_loader(lookup_path)
if not shim_image:
raise KiwiBootLoaderGrubSecureBootError(
'Microsoft signed shim loader not found'
)
grub_image = Defaults.get_signed_grub_loader(lookup_path)
if not grub_image:
raise KiwiBootLoaderGrubSecureBootError(
'Shim signed grub2 efi loader not found'
)
Command.run(
['cp', shim_image, self._get_efi_image_name()]
)
Command.run(
['cp', grub_image, self.efi_boot_path]
)
def pack_image_to_file(self, filename):
"""
Packs the given oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
oci_image = os.sep.join([
self.oci_dir, ':'.join(['umoci_layout', self.container_tag])
])
additional_tags = []
for tag in self.additional_tags:
additional_tags.extend([
'--additional-tag', '{0}:{1}'.format(self.container_name, tag)
])
# make sure the target tar file does not exist
# skopeo doesn't support force overwrite
Path.wipe(filename)
Command.run(
[
'skopeo', 'copy', 'oci:{0}'.format(
oci_image
),
'docker-archive:{0}:{1}:{2}'.format(
filename, self.container_name, self.container_tag
)
] + additional_tags
)
container_compressor = self.runtime_config.get_container_compression()
if container_compressor:
compressor = Compress(filename)
return compressor.xz(self.runtime_config.get_xz_options())
else:
return filename
def process_delete_requests(self, force=False):
"""
Process package delete requests (chroot)
:param bool force: force deletion: true|false
"""
delete_items = []
for delete_item in self.package_requests:
try:
Command.run(['chroot', self.root_dir, 'rpm', '-q', delete_item])
delete_items.append(delete_item)
except Exception:
# ignore packages which are not installed
pass
if not delete_items:
raise KiwiRequestError(
'None of the requested packages to delete are installed'
)
delete_options = ['--nodeps', '--allmatches', '--noscripts']
self.cleanup_requests()
return Command.call(
[
'chroot', self.root_dir, 'rpm', '-e'
] + delete_options + delete_items,
self.command_env
)
def process_install_requests(self):
"""
Process package install requests for image phase (chroot)
"""
yum = self._get_yum_binary_name(root=self.root_dir)
if self.exclude_requests:
# For Yum, excluding a package means removing it from
# the solver operation. This is done by adding --exclude
# to the command line. This means that if the package is
# hard required by another package, it will break the transaction.
for package in self.exclude_requests:
self.custom_args.append('--exclude=' + package)
Command.run(
['chroot', self.root_dir, 'rpm', '--rebuilddb']
)
chroot_yum_args = self.root_bind.move_to_root(
self.yum_args
)
bash_command = [
'chroot', self.root_dir, yum
] + chroot_yum_args + self.custom_args + [
'install'
] + self.package_requests
if self.collection_requests:
bash_command += [
'&&', 'chroot', self.root_dir, yum
] + chroot_yum_args + self.custom_args + [
'groupinstall'
] + self.collection_requests
self.cleanup_requests()
return Command.call(
['bash', '-c', ' '.join(bash_command)], self.command_env
)
def process_delete_requests(self, force=False):
"""
Process package delete requests (chroot)
:param bool force: force deletion: true|false
"""
delete_items = []
for delete_item in self._delete_items():
try:
Command.run(['chroot', self.root_dir, 'rpm', '-q', delete_item])
delete_items.append(delete_item)
except Exception:
# ignore packages which are not installed
pass
if not delete_items:
raise KiwiRequestError(
'None of the requested packages to delete are installed'
)
if force:
force_options = ['--nodeps', '--allmatches', '--noscripts']
return Command.call(
[
'chroot', self.root_dir, 'rpm', '-e'
] + force_options + delete_items,
self.chroot_command_env
)
else:
return Command.call(
[
'chroot', self.root_dir, 'zypper'
] + self.chroot_zypper_args + [
'remove', '-u', '--force-resolution'
] + delete_items,
self.chroot_command_env
)
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
:return: process results in command type
:rtype: namedtuple
"""
yum = self._get_yum_binary_name()
Command.run(
[yum] + self.yum_args + ['makecache']
)
bash_command = [
yum
] + self.yum_args + [
'--installroot', self.root_dir
] + self.custom_args + ['install'] + self.package_requests
if self.collection_requests:
bash_command += [
'&&', yum
] + self.yum_args + [
'--installroot', self.root_dir
] + self.custom_args + ['groupinstall'] + self.collection_requests
self.cleanup_requests()
return Command.call(
['bash', '-c', ' '.join(bash_command)], self.command_env
)
def create(self, name, mbsize, type_name, flags=None):
"""
Create DASD partition
:param string name: partition name
:param int mbsize: partition size
:param string type_name: unused
:param list flags: unused
"""
self.partition_id += 1
fdasd_input = NamedTemporaryFile()
with open(fdasd_input.name, 'w') as partition:
log.debug(
'%s: fdasd: n p cur_position +%sM w q',
name, format(mbsize)
)
if mbsize == 'all_free':
partition.write('n\np\n\n\nw\nq\n')
else:
partition.write('n\np\n\n+%dM\nw\nq\n' % mbsize)
bash_command = ' '.join(
['cat', fdasd_input.name, '|', 'fdasd', '-f', self.disk_device]
)
try:
Command.run(
['bash', '-c', bash_command]
)
except Exception:
# unfortunately fdasd reports that it can't read in the partition
# table which I consider a bug in fdasd. However the table was
# correctly created and therefore we continue. Problem is that we
# are not able to detect real errors with the fdasd operation at
# that point.
log.debug('potential fdasd errors were ignored')
def resize_raw_disk(self, size_bytes):
"""
Resize raw disk image to specified size. If the request
would actually shrink the disk an exception is raised.
If the disk got changed the method returns True, if
the new size is the same as the current size nothing
gets resized and the method returns False
:param int size: new size in bytes
:rtype: bool
"""
current_byte_size = os.path.getsize(self.diskname)
size_bytes = int(size_bytes)
if size_bytes < current_byte_size:
raise KiwiResizeRawDiskError(
'shrinking {0} disk to {1} bytes corrupts the image'.format(
self.diskname, size_bytes
)
)
elif size_bytes == current_byte_size:
return False
Command.run(
['qemu-img', 'resize', self.diskname, format(size_bytes)]
)
return True
def create(self, source_dir, exclude=None):
"""
Create cpio archive
:param string source_dir: data source directory
:param list exclude: list of excluded items
"""
find_excludes = []
find_command = ['cd', source_dir, '&&', 'find', '.']
cpio_command = [
'cpio', '--quiet', '-o', '-H', 'newc', '>', self.filename
]
if exclude:
for path in exclude:
if find_excludes:
find_excludes.append('-or')
find_excludes.append('-path')
find_excludes.append('.' + path)
find_excludes.append('-prune')
find_excludes.append('-o')
find_excludes.append('-print')
find_command += find_excludes
bash_command = find_command + ['|'] + cpio_command
Command.run(
['bash', '-c', ' '.join(bash_command)]
)
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image',
'{0}:base_layer'.format(self.oci_layout_dir), self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep])
)
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def uncompress(self, temporary=False):
"""
Uncompress with format autodetection
By default the original source file will be changed into
the uncompressed variant. If temporary is set to True
a temporary file is created instead
:param bool temporary: uncompress to a temporary file
"""
zipper = self.get_format()
if not zipper:
raise KiwiCompressionFormatUnknown(
'could not detect compression format for %s' %
self.source_filename
)
if not temporary:
Command.run([zipper, '-d', self.source_filename])
self.uncompressed_filename = self.source_filename
else:
self.temp_file = NamedTemporaryFile()
bash_command = [
zipper, '-c', '-d', self.source_filename,
'>', self.temp_file.name
]
Command.run(['bash', '-c', ' '.join(bash_command)])
self.uncompressed_filename = self.temp_file.name
return self.uncompressed_filename
def create_xz_compressed(
self, source_dir, exclude=None, options=None, xz_options=None
):
"""
Create XZ compressed tar archive
:param string source_dir: data source directory
:param list exclude: list of excluded items
:param list options: custom tar creation options
:param list xz_options: custom xz compression options
"""
if not options:
options = []
if not xz_options:
xz_options = Defaults.get_xz_compression_options()
bash_command = [
'tar', '-C', source_dir
] + options + self.xattrs_options + [
'-c', '--to-stdout'
] + self._get_archive_items(source_dir, exclude) + [
'|', 'xz', '-f'
] + xz_options + [
'>', self.filename + '.xz'
]
Command.run(['bash', '-c', ' '.join(bash_command)])
return self.filename + '.xz'
def test_run_raises_error(self, mock_popen, mock_which):
mock_which.return_value = "command"
mock_process = mock.Mock()
mock_process.communicate = mock.Mock(return_value=[str.encode("stdout"), str.encode("stderr")])
mock_process.returncode = 1
mock_popen.return_value = mock_process
Command.run(["command", "args"])
def setup_locale(self):
"""
Setup UTF8 system wide locale
"""
if 'locale' in self.preferences:
if 'POSIX' in self.preferences['locale'].split(','):
locale = 'POSIX'
else:
locale = '{0}.UTF-8'.format(
self.preferences['locale'].split(',')[0]
)
log.info('Setting up locale: %s', self.preferences['locale'])
if CommandCapabilities.has_option_in_help(
'systemd-firstboot', '--locale',
root=self.root_dir, raise_on_error=False
):
Path.wipe(self.root_dir + '/etc/locale.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--locale=' + locale
])
elif os.path.exists(self.root_dir + '/etc/sysconfig/language'):
Shell.run_common_function(
'baseUpdateSysConfig', [
self.root_dir + '/etc/sysconfig/language',
'RC_LANG', locale
]
)
else:
log.warning(
'locale setup skipped no capable '
'systemd-firstboot or etc/sysconfig/language not found'
)
def setup_keyboard_map(self):
"""
Setup console keyboard
"""
if 'keytable' in self.preferences:
log.info(
'Setting up keytable: %s', self.preferences['keytable']
)
if CommandCapabilities.has_option_in_help(
'systemd-firstboot', '--keymap',
root=self.root_dir, raise_on_error=False
):
Path.wipe(self.root_dir + '/etc/vconsole.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--keymap=' + self.preferences['keytable']
])
elif os.path.exists(self.root_dir + '/etc/sysconfig/keyboard'):
Shell.run_common_function(
'baseUpdateSysConfig', [
self.root_dir + '/etc/sysconfig/keyboard', 'KEYTABLE',
'"' + self.preferences['keytable'] + '"'
]
)
else:
log.warning(
'keyboard setup skipped no capable '
'systemd-firstboot or etc/sysconfig/keyboard found'
)
def create(self, name, mbsize, type_name, flags=None):
"""
Create GPT partition
:param string name: partition name
:param int mbsize: partition size
:param string type_name: partition type
:param list flags: additional flags
"""
self.partition_id += 1
if mbsize == 'all_free':
partition_end = '0'
else:
partition_end = '+' + format(mbsize) + 'M'
if self.partition_id > 1 or not self.start_sector:
# A start sector value of 0 specifies the default value
# defined in sgdisk
self.start_sector = 0
Command.run(
[
'sgdisk', '-n', ':'.join(
[
format(self.partition_id),
format(self.start_sector),
partition_end
]
), '-c', ':'.join([format(self.partition_id), name]),
self.disk_device
]
)
self.set_flag(self.partition_id, type_name)
if flags:
for flag_name in flags:
self.set_flag(self.partition_id, flag_name)
def process_install_requests(self):
"""
Process package install requests for image phase (chroot)
:return: process results in command type
:rtype: namedtuple
"""
if self.exclude_requests:
# For zypper excluding a package means, removing it from
# the solver operation. This is done by adding a package
# lock. This means that if the package is hard required
# by another package, it will break the transaction.
metadata_dir = ''.join([self.root_dir, '/etc/zypp'])
if not os.path.exists(metadata_dir):
Path.create(metadata_dir)
for package in self.exclude_requests:
Command.run(
['chroot', self.root_dir, 'zypper'] +
self.chroot_zypper_args + ['al'] + [package],
self.chroot_command_env
)
return Command.call(
['chroot', self.root_dir, 'zypper'] + self.chroot_zypper_args + [
'install', '--auto-agree-with-licenses'
] + self.custom_args + self._install_items(),
self.chroot_command_env
)
def set_flag(self, partition_id, flag_name):
"""
Set msdos partition flag
:param int partition_id: partition number
:param string flag_name: name from flag map
"""
if flag_name not in self.flag_map:
raise KiwiPartitionerMsDosFlagError(
'Unknown partition flag %s' % flag_name
)
if self.flag_map[flag_name]:
if flag_name == 'f.active':
Command.run(
[
'parted', self.disk_device,
'set', format(partition_id), 'boot', 'on'
]
)
else:
Command.run(
[
'sfdisk', '-c', self.disk_device,
format(partition_id), self.flag_map[flag_name]
]
)
else:
log.warning('Flag %s ignored on msdos', flag_name)
def _copy_first_boot_files_to_system_image(self):
boot_names = self.boot_image.get_boot_names()
if self.initrd_system == 'kiwi':
log.info('Copy boot files to system image')
kernel = Kernel(self.boot_image.boot_root_directory)
log.info('--> boot image kernel as %s', boot_names.kernel_name)
kernel.copy_kernel(
self.root_dir, ''.join(['/boot/', boot_names.kernel_name])
)
if self.xen_server:
if kernel.get_xen_hypervisor():
log.info('--> boot image Xen hypervisor as xen.gz')
kernel.copy_xen_hypervisor(
self.root_dir, '/boot/xen.gz'
)
else:
raise KiwiDiskBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image.boot_root_directory
)
log.info('--> initrd archive as %s', boot_names.initrd_name)
Command.run(
[
'mv', self.boot_image.initrd_filename,
self.root_dir + ''.join(['/boot/', boot_names.initrd_name])
]
)
def create(self, name, mbsize, type_name, flags=None):
"""
Create GPT partition
:param string name: partition name
:param int mbsize: partition size
:param string type_name: partition type
:param list flags: additional flags
"""
self.partition_id += 1
if mbsize == 'all_free':
partition_end = '0'
else:
partition_end = '+' + format(mbsize) + 'M'
Command.run(
[
'sgdisk', '-n',
':'.join([format(self.partition_id), '0', partition_end]),
'-c', ':'.join([format(self.partition_id), name]),
self.disk_device
]
)
self.set_flag(self.partition_id, type_name)
if flags:
for flag_name in flags:
self.set_flag(self.partition_id, flag_name)
def wipe(self):
"""
Zap (destroy) any GPT and MBR data structures if present
For DASD disks create a new VTOC table
"""
if 'dasd' in self.table_type:
log.debug('Initialize DASD disk with new VTOC table')
fdasd_input = NamedTemporaryFile()
with open(fdasd_input.name, 'w') as vtoc:
vtoc.write('y\n\nw\nq\n')
bash_command = ' '.join(
[
'cat', fdasd_input.name, '|',
'fdasd', '-f', self.storage_provider.get_device()
]
)
try:
Command.run(
['bash', '-c', bash_command]
)
except Exception:
# unfortunately fdasd reports that it can't read in the
# partition table which I consider a bug in fdasd. However
# the table was correctly created and therefore we continue.
# Problem is that we are not able to detect real errors
# with the fdasd operation at that point.
log.debug('potential fdasd errors were ignored')
else:
log.debug('Initialize %s disk', self.table_type)
Command.run(
[
'sgdisk', '--zap-all', self.storage_provider.get_device()
]
)
def setup_intermediate_config(self):
"""
Create intermediate config files
Some config files e.g etc/hosts needs to be temporarly copied
from the buildsystem host to the image root system in order to
allow e.g DNS resolution in the way as it is configured on the
buildsystem host. These config files only exists during the image
build process and are not part of the final image
:raises KiwiSetupIntermediateConfigError: if the management of
intermediate configuration files fails
"""
try:
for config in self.config_files:
if os.path.exists(config):
self.cleanup_files.append(config + '.kiwi')
Command.run(
['cp', config, self.root_dir + config + '.kiwi']
)
link_target = os.path.basename(config) + '.kiwi'
Command.run(
['ln', '-s', '-f', link_target, self.root_dir + config]
)
except Exception as e:
self.cleanup()
raise KiwiSetupIntermediateConfigError(
'%s: %s' % (type(e).__name__, format(e))
)
def _create_iso_install_kernel_and_initrd(self):
boot_path = self.media_dir + '/boot/' + self.arch + '/loader'
Path.create(boot_path)
kernel = Kernel(self.boot_image_task.boot_root_directory)
if kernel.get_kernel():
kernel.copy_kernel(boot_path, '/linux')
else:
raise KiwiInstallBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
if self.xml_state.is_xen_server():
if kernel.get_xen_hypervisor():
kernel.copy_xen_hypervisor(boot_path, '/xen.gz')
else:
raise KiwiInstallBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
if self.initrd_system == 'dracut':
self._create_dracut_install_config()
self._add_system_image_boot_options_to_boot_image()
self.boot_image_task.create_initrd(self.mbrid, 'initrd_kiwi_install')
Command.run(
[
'mv', self.boot_image_task.initrd_filename,
boot_path + '/initrd'
]
)
def create_degraded_raid(self, raid_level):
"""
Create a raid array in degraded mode with one device missing.
This only works in the raid levels 0(striping) and 1(mirroring)
:param string raid_level: raid level name
"""
if raid_level not in self.raid_level_map:
raise KiwiRaidSetupError(
'Only raid levels 0(striping) and 1(mirroring) are supported'
)
raid_device = None
for raid_id in range(9):
raid_device = '/dev/md' + format(raid_id)
if os.path.exists(raid_device):
raid_device = None
else:
break
if not raid_device:
raise KiwiRaidSetupError(
'Could not find free raid device in range md0-8'
)
log.info(
'Creating raid array in %s mode as %s',
raid_level, raid_device
)
Command.run(
[
'mdadm', '--create', '--run', raid_device,
'--level', self.raid_level_map[raid_level],
'--raid-disks', '2',
self.storage_provider.get_device(), 'missing'
]
)
self.raid_device = raid_device
def import_trusted_keys(self, signing_keys):
"""
Imports trusted keys into the image
:param list signing_keys: list of the key files to import
"""
for key in signing_keys:
Command.run(['rpm', '--root', self.root_dir, '--import', key])
def test_run_raises_error(self, mock_popen):
mock_process = mock.Mock()
mock_process.communicate = mock.Mock(
return_value=[str.encode('stdout'), str.encode('stderr')]
)
mock_process.returncode = 1
mock_popen.return_value = mock_process
Command.run(['command', 'args'])
def process_install_requests_bootstrap(self):
"""
Process package install requests for bootstrap phase (no chroot)
The debootstrap program is used to bootstrap a new system with
a collection of predefined packages. The kiwi bootstrap section
information is not used in this case
"""
if not self.distribution:
raise KiwiDebootstrapError(
'No main distribution repository is configured'
)
bootstrap_script = '/usr/share/debootstrap/scripts/' + \
self.distribution
if not os.path.exists(bootstrap_script):
raise KiwiDebootstrapError(
'debootstrap script for %s distribution not found' %
self.distribution
)
bootstrap_dir = self.root_dir + '.debootstrap'
if 'apt-get' in self.package_requests:
# debootstrap takes care to install apt-get
self.package_requests.remove('apt-get')
try:
dev_mount = MountManager(
device='/dev', mountpoint=self.root_dir + '/dev'
)
dev_mount.umount()
if self.repository.unauthenticated == 'false':
log.warning(
'KIWI does not support signature checks for apt-get '
'package manager during the bootstrap procedure, any '
'provided key will only be used inside the chroot '
'environment'
)
Command.run(
[
'debootstrap', '--no-check-gpg', self.distribution,
bootstrap_dir, self.distribution_path
], self.command_env
)
data = DataSync(
bootstrap_dir + '/', self.root_dir
)
data.sync_data(
options=['-a', '-H', '-X', '-A']
)
for key in self.repository.signing_keys:
Command.run([
'chroot', self.root_dir, 'apt-key', 'add', key
], self.command_env)
except Exception as e:
raise KiwiDebootstrapError(
'%s: %s' % (type(e).__name__, format(e))
)
finally:
Path.wipe(bootstrap_dir)
return self.process_install_requests()
def _search_for(self, name, in_file):
search = '^' + name + ':'
try:
Command.run(
['chroot', self.root_dir, 'grep', '-q', search, in_file]
)
except Exception:
return False
return True
def __del__(self):
if self.node_name:
log.info('Cleaning up %s instance', type(self).__name__)
try:
Command.run(['losetup', '-d', self.node_name])
except Exception:
log.warning(
'loop device %s still busy', self.node_name
)
def _export_pacman_package_list(self, filename):
log.info('Export pacman packages metadata')
query_call = Command.run([
'pacman', '--query', '--dbpath',
os.sep.join([self.root_dir, 'var/lib/pacman'])
])
with open(filename, 'w') as packages:
for line in query_call.output.splitlines():
package, _, version_release = line.partition(' ')
version, _, release = version_release.partition('-')
packages.writelines([
'{0}|None|{1}|{2}|None|None|None{3}'.format(
package, version, release, os.linesep)
])
def setup_locale(self) -> None:
"""
Setup UTF8 system wide locale
"""
if 'locale' in self.preferences:
if 'POSIX' in self.preferences['locale'].split(','):
locale = 'POSIX'
else:
locale = '{0}.UTF-8'.format(
self.preferences['locale'].split(',')[0]
)
log.info(
'Setting up locale: {0}'.format(self.preferences['locale'])
)
if CommandCapabilities.has_option_in_help(
'systemd-firstboot', '--locale',
root=self.root_dir, raise_on_error=False
):
Path.wipe(self.root_dir + '/etc/locale.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--locale=' + locale
])
def setup_timezone(self) -> None:
"""
Setup timezone symlink
"""
if 'timezone' in self.preferences:
log.info(
'Setting up timezone: {0}'.format(self.preferences['timezone'])
)
if CommandCapabilities.has_option_in_help(
'systemd-firstboot', '--timezone',
root=self.root_dir, raise_on_error=False
):
Path.wipe(self.root_dir + '/etc/localtime')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--timezone=' + self.preferences['timezone']
])
else:
zoneinfo = '/usr/share/zoneinfo/' + self.preferences['timezone']
Command.run([
'chroot', self.root_dir,
'ln', '-s', '-f', zoneinfo, '/etc/localtime'
])
def _cleanup_intermediate_config(self):
# delete kiwi copied config files
config_files_to_delete = []
for config in self.cleanup_files:
config_files_to_delete.append(self.root_dir + config)
del self.cleanup_files[:]
# delete stale symlinks if there are any. normally the package
# installation process should have replaced the symlinks with
# real files from the packages
for config in self.config_files:
if os.path.islink(self.root_dir + config):
config_files_to_delete.append(self.root_dir + config)
try:
Command.run(['rm', '-f'] + config_files_to_delete)
except Exception as e:
log.warning('Failed to remove intermediate config files: %s',
format(e))
self._restore_intermediate_config_rpmnew_variants()
def get_blkid(self, id_type):
"""
Retrieve information for specified metadata ID from block device
:param string id_type: metadata ID, see man blkid for details
:return: ID of the block device
:rtype: str
"""
blkid_result = Command.run(
['blkid', self.device, '-s', id_type, '-o', 'value'],
raise_on_error=False)
return blkid_result.output.strip(os.linesep) if blkid_result else ''
def set_flag(self, partition_id, flag_name):
"""
Set msdos partition flag
:param int partition_id: partition number
:param string flag_name: name from flag map
"""
if flag_name not in self.flag_map:
raise KiwiPartitionerMsDosFlagError('Unknown partition flag %s' %
flag_name)
if self.flag_map[flag_name]:
if flag_name == 'f.active':
Command.run([
'parted', self.disk_device, 'set',
format(partition_id), 'boot', 'on'
])
else:
Command.run([
'sfdisk', '-c', self.disk_device,
format(partition_id), self.flag_map[flag_name]
])
else:
log.warning('Flag %s ignored on msdos', flag_name)
def process_install_requests(self):
"""
Process package install requests for image phase (chroot)
:return: process results in command type
:rtype: namedtuple
"""
update_command = ['chroot', self.root_dir, 'apt-get']
update_command.extend(
Path.move_to_root(self.root_dir, self.apt_get_args))
update_command.extend(self.custom_args)
update_command.append('update')
Command.run(update_command, self.command_env)
apt_get_command = ['chroot', self.root_dir, 'apt-get']
apt_get_command.extend(
Path.move_to_root(self.root_dir, self.apt_get_args))
apt_get_command.extend(self.custom_args)
apt_get_command.append('install')
apt_get_command.extend(self._package_requests())
return Command.call(apt_get_command, self.command_env)
def _export_deb_package_list(self, filename):
log.info('Export deb packages metadata')
query_call = Command.run([
'dpkg-query', '--admindir',
os.sep.join([self.root_dir, 'var/lib/dpkg']), '-W', '-f',
'|'.join([
'${Package}', 'None', '${Version}', 'None', '${Architecture}',
'None', 'None'
]) + '\\n'
])
with open(filename, 'w') as packages:
packages.write(
os.linesep.join(sorted(query_call.output.splitlines())))
packages.write(os.linesep)
def get_uuid(self, device):
"""
UUID of device
:param string device: node name
:return: UUID from blkid
:rtype: str
"""
uuid_call = Command.run(
['blkid', device, '-s', 'UUID', '-o', 'value']
)
return uuid_call.output.rstrip('\n')
def get_byte_size(self, device):
"""
Size of device in bytes
:param string device: node name
:return: byte value from blockdev
:rtype: int
"""
blockdev_call = Command.run(
['blockdev', '--getsize64', device]
)
return int(blockdev_call.output.rstrip('\n'))
def create(self, overwrite=True):
"""
Setup a loop device of the blocksize given in the constructor
The file to loop is created with the size specified in the
constructor unless an existing one should not be overwritten
:param bool overwrite: overwrite existing file to loop
"""
if overwrite:
qemu_img_size = format(self.filesize_mbytes) + 'M'
Command.run(['qemu-img', 'create', self.filename, qemu_img_size])
loop_options = []
if self.blocksize_bytes and self.blocksize_bytes != 512:
if CommandCapabilities.has_option_in_help('losetup',
'--sector-size',
raise_on_error=False):
loop_options.append('--sector-size')
else:
loop_options.append('--logical-blocksize')
loop_options.append(format(self.blocksize_bytes))
loop_call = Command.run(['losetup'] + loop_options +
['-f', '--show', self.filename])
self.node_name = loop_call.output.rstrip('\n')
def install(self):
"""
Install bootloader on self.device
"""
log.info('Installing zipl on disk %s', self.device)
self.boot_mount.mount()
bash_command = ' '.join([
'cd', self.boot_mount.mountpoint, '&&', 'zipl', '-V', '-c',
self.boot_mount.mountpoint + '/config', '-m', 'menu'
])
zipl_call = Command.run(['bash', '-c', bash_command])
log.debug('zipl install succeeds with: %s', zipl_call.output)
def create_volumes(self, filesystem_name):
"""
Create configured lvm volumes and filesystems
All volumes receive the same filesystem
:param string filesystem_name: volumes filesystem name
"""
log.info('Creating volumes(%s)', filesystem_name)
self.create_volume_paths_in_root_dir()
canonical_volume_list = self.get_canonical_volume_list()
for volume in canonical_volume_list.volumes:
volume_mbsize = self.get_volume_mbsize(
volume, self.volumes, filesystem_name,
self.custom_args['image_type'])
log.info('--> volume %s with %s MB', volume.name, volume_mbsize)
Command.run([
'lvcreate', '-L',
format(volume_mbsize), '-n', volume.name, self.volume_group
])
self.apply_attributes_on_volume(self.root_dir, volume)
self._add_to_volume_map(volume.name)
self._create_filesystem(volume.name, filesystem_name)
self._add_to_mount_list(volume.name, volume.realpath)
if canonical_volume_list.full_size_volume:
full_size_volume = canonical_volume_list.full_size_volume
log.info('--> fullsize volume %s', full_size_volume.name)
Command.run([
'lvcreate', '-l', '+100%FREE', '-n', full_size_volume.name,
self.volume_group
])
self._add_to_volume_map(full_size_volume.name)
self._create_filesystem(full_size_volume.name, filesystem_name)
self._add_to_mount_list(full_size_volume.name,
full_size_volume.realpath)
def _sync_system_to_image(self, device_map: Dict, system: Any,
system_boot: Optional[FileSystemBase],
system_efi: Optional[FileSystemBase],
system_spare: Optional[FileSystemBase]) -> None:
log.info('Syncing system to image')
if system_spare:
system_spare.sync_data()
if system_efi:
log.info('--> Syncing EFI boot data to EFI partition')
system_efi.sync_data()
if system_boot:
log.info('--> Syncing boot data at extra partition')
system_boot.sync_data(self._get_exclude_list_for_boot_data_sync())
log.info('--> Syncing root filesystem data')
if self.root_filesystem_is_overlay:
squashed_root_file = Temporary().new_file()
squashed_root = FileSystemSquashFs(
device_provider=DeviceProvider(),
root_dir=self.root_dir,
custom_args={
'compression':
self.xml_state.build_type.get_squashfscompression()
})
squashed_root.create_on_file(
filename=squashed_root_file.name,
exclude=self._get_exclude_list_for_root_data_sync(device_map))
Command.run([
'dd',
'if=%s' % squashed_root_file.name,
'of=%s' % device_map['readonly'].get_device()
])
else:
system.sync_data(
self._get_exclude_list_for_root_data_sync(device_map))
def setup_isolinux_boot_path(self):
"""
Write the base boot path into the isolinux loader binary
:raises KiwiIsoLoaderError: if loader/isolinux.bin is not found
"""
loader_base_directory = self.boot_path + '/loader'
loader_file = '/'.join(
[self.source_dir, self.boot_path, 'loader/isolinux.bin']
)
if not os.path.exists(loader_file):
raise KiwiIsoLoaderError(
'No isolinux loader %s found'.format(loader_file)
)
try:
Command.run(
[
'isolinux-config', '--base', loader_base_directory,
loader_file
]
)
except Exception:
# Setup of the base directory failed. This happens if
# isolinux-config was not able to identify the isolinux
# signature. As a workaround a compat directory /isolinux
# is created which hardlinks all loader files
compat_base_directory = self.source_dir + '/isolinux'
loader_files = '/'.join(
[self.source_dir, self.boot_path, 'loader/*']
)
Path.create(compat_base_directory)
bash_command = ' '.join(
['ln', loader_files, compat_base_directory]
)
Command.run(
['bash', '-c', bash_command]
)
def _create_embedded_fat_efi_image(self):
Path.create(self.boot_dir + '/boot/' + self.arch)
efi_fat_image = ''.join([self.boot_dir + '/boot/', self.arch, '/efi'])
Command.run(['qemu-img', 'create', efi_fat_image, '15M'])
Command.run(['mkdosfs', '-n', 'BOOT', efi_fat_image])
Command.run([
'mcopy', '-Do', '-s', '-i', efi_fat_image, self.boot_dir + '/EFI',
'::'
])
def process_install_requests(self):
"""
Process package install requests for image phase (chroot)
:return: process results in command type
:rtype: namedtuple
"""
yum = self._get_yum_binary_name(root=self.root_dir)
if self.exclude_requests:
# For Yum, excluding a package means removing it from
# the solver operation. This is done by adding --exclude
# to the command line. This means that if the package is
# hard required by another package, it will break the transaction.
for package in self.exclude_requests:
self.custom_args.append('--exclude=' + package)
Command.run(
['chroot', self.root_dir, 'rpm', '--rebuilddb']
)
chroot_yum_args = self.root_bind.move_to_root(
self.yum_args
)
bash_command = [
'chroot', self.root_dir, yum
] + chroot_yum_args + self.custom_args + [
'install'
] + self.package_requests
if self.collection_requests:
bash_command += [
'&&', 'chroot', self.root_dir, yum
] + chroot_yum_args + self.custom_args + [
'groupinstall'
] + self.collection_requests
self.cleanup_requests()
return Command.call(
['bash', '-c', ' '.join(bash_command)], self.command_env
)
def process_delete_requests(self, force=False):
"""
Process package delete requests (chroot)
:param bool force: force deletion: true|false
:raises KiwiRequestError: if none of the packages to delete is
installed
:return: process results in command type
:rtype: namedtuple
"""
delete_items = []
for delete_item in self.package_requests:
try:
Command.run(
['chroot', self.root_dir, 'rpm', '-q', delete_item])
delete_items.append(delete_item)
except Exception:
# ignore packages which are not installed
pass
if not delete_items:
raise KiwiRequestError(
'None of the requested packages to delete are installed')
self.cleanup_requests()
if force:
delete_options = ['--nodeps', '--allmatches', '--noscripts']
return Command.call(['chroot', self.root_dir, 'rpm', '-e'] +
delete_options + delete_items,
self.command_env)
else:
chroot_yum_args = self.root_bind.move_to_root(self.yum_args)
return Command.call(
['chroot', self.root_dir,
self._get_yum_binary_name()] + chroot_yum_args +
self.custom_args + ['autoremove'] + delete_items,
self.command_env)
def _copy_loader_data_to_boot_directory(self, lookup_path):
if not lookup_path:
lookup_path = self.root_dir
loader_data = lookup_path + '/image/loader/'
Path.wipe(loader_data)
Path.create(loader_data)
syslinux_file_names = [
'isolinux.bin', 'ldlinux.c32', 'libcom32.c32', 'libutil.c32',
'gfxboot.c32', 'gfxboot.com', 'menu.c32', 'chain.c32', 'mboot.c32'
]
syslinux_dirs = [
'/usr/share/syslinux/', '/usr/lib/syslinux/modules/bios/'
]
for syslinux_file_name in syslinux_file_names:
for syslinux_dir in syslinux_dirs:
syslinux_file = ''.join(
[lookup_path, syslinux_dir, syslinux_file_name])
if os.path.exists(syslinux_file):
shutil.copy(syslinux_file, loader_data)
bash_command = ' '.join(
['cp', lookup_path + '/boot/memtest*', loader_data + '/memtest'])
Command.run(command=['bash', '-c', bash_command], raise_on_error=False)
if self.get_boot_theme():
theme_path = ''.join([
lookup_path, '/etc/bootsplash/themes/',
self.get_boot_theme()
])
if os.path.exists(theme_path + '/cdrom/gfxboot.cfg'):
bash_command = ' '.join(
['cp', theme_path + '/cdrom/*', loader_data])
Command.run(['bash', '-c', bash_command])
# don't move down one menu entry the first time a F-key is used
Command.run([
'gfxboot', '--config-file', loader_data + '/gfxboot.cfg',
'--change-config', 'install::autodown=0'
])
if os.path.exists(theme_path + '/bootloader/message'):
Command.run(
['cp', theme_path + '/bootloader/message', loader_data])
Path.create(self._get_iso_boot_path())
data = DataSync(loader_data, self._get_iso_boot_path())
data.sync_data(options=['-z', '-a'])
def _import_custom_archives(self):
"""
Import custom tar archive files
"""
archive_list = []
system_archives = self.xml_state.get_system_archives()
bootstrap_archives = self.xml_state.get_bootstrap_archives()
if system_archives:
archive_list += system_archives
if bootstrap_archives:
archive_list += bootstrap_archives
archive_target_dir = os.path.join(self.root_dir,
defaults.IMAGE_METADATA_DIR) + os.sep
for archive in archive_list:
archive_is_absolute = archive.startswith(os.sep)
if archive_is_absolute:
archive_file = archive
else:
archive_file = os.path.join(self.description_dir, archive)
archive_exists = os.path.exists(archive_file)
if not archive_exists:
if self.derived_description_dir and not archive_is_absolute:
archive_file = self.derived_description_dir + '/' + archive
archive_exists = os.path.exists(archive_file)
if archive_exists:
log.info('--> Importing {0} archive to {1}'.format(
archive_file, archive_target_dir))
Command.run(['cp', archive_file, archive_target_dir])
else:
raise KiwiImportDescriptionError(
'Specified archive {0} does not exist'.format(
archive_file))
def _get_relaxng_validation_details(
schema_file, description_file, error_log
):
"""
Run jing program to validate description against the schema
Jing provides detailed error information in case of a schema
validation failure. If jing is not present the standard
error_log as provided from the raw XML libraries is used
"""
try:
Command.run(
['jing', schema_file, description_file]
)
except KiwiCommandError as issue:
log.info('RelaxNG validation failed. See jing report:')
log.info('--> {0}'.format(issue))
except KiwiCommandNotFound as issue:
log.warning(issue)
log.warning(
'For detailed schema validation report, install: jing'
)
log.info('Showing only raw library error log:')
log.info('--> {0}'.format(error_log))
def setup_keyboard_map(self):
"""
Setup console keyboard
"""
if 'keytable' in self.preferences:
log.info('Setting up keytable: %s', self.preferences['keytable'])
if CommandCapabilities.has_option_in_help('systemd-firstboot',
'--keymap',
root=self.root_dir,
raise_on_error=False):
Path.wipe(self.root_dir + '/etc/vconsole.conf')
Command.run([
'chroot', self.root_dir, 'systemd-firstboot',
'--keymap=' + self.preferences['keytable']
])
elif os.path.exists(self.root_dir + '/etc/sysconfig/keyboard'):
Shell.run_common_function('baseUpdateSysConfig', [
self.root_dir + '/etc/sysconfig/keyboard', 'KEYTABLE',
'"' + self.preferences['keytable'] + '"'
])
else:
log.warning(
'keyboard setup skipped no capable '
'systemd-firstboot or etc/sysconfig/keyboard found')
def is_mounted(self):
"""
Check if mounted
:return: True or False
:rtype: bool
"""
mountpoint_call = Command.run(
command=['mountpoint', '-q', self.mountpoint],
raise_on_error=False)
if mountpoint_call.returncode == 0:
return True
else:
return False
def _setup_secure_boot_efi_image(self, lookup_path):
"""
Provide the shim loader and the shim signed grub2 loader
in the required boot path. Normally this task is done by
the shim-install tool. However, shim-install does not exist
on all distributions and the script does not operate well
in e.g CD environments from which we generate live and/or
install media. Thus shim-install is used if possible at
install time of the bootloader because it requires access
to the target block device. In any other case this setup
code should act as the fallback solution
"""
if not lookup_path:
lookup_path = self.root_dir
shim_image = Defaults.get_shim_loader(lookup_path)
if not shim_image:
raise KiwiBootLoaderGrubSecureBootError(
'Microsoft signed shim loader not found')
grub_image = Defaults.get_signed_grub_loader(lookup_path)
if not grub_image:
raise KiwiBootLoaderGrubSecureBootError(
'Shim signed grub2 efi loader not found')
Command.run(['cp', shim_image, self._get_efi_image_name()])
Command.run(['cp', grub_image, self.efi_boot_path])
def import_container_image(self, container_image_ref):
"""
Imports container image reference to the OCI containers storage.
:param str container_image_ref: container image reference
"""
if not self.imported_image:
self.imported_image = 'kiwi-image-{0}:{1}'.format(
self._random_string_generator(),
Defaults.get_container_base_image_tag())
else:
raise KiwiBuildahError(
"Image already imported, called: '{0}'".format(
self.imported_image))
# We are making use of skopeo instead of only calling 'buildah from'
# because we want to control the image name loaded into the containers
# storage. This way we are certain to not leave any left over after the
# build.
Command.run([
'skopeo', 'copy', container_image_ref,
'containers-storage:{0}'.format(self.imported_image)
])
if not self.working_container:
self.working_container = 'kiwi-container-{0}'.format(
self._random_string_generator())
else:
raise KiwiBuildahError(
"Container already initated, called: '{0}'".format(
self.working_container))
Command.run([
'buildah', 'from', '--name', self.working_container,
'containers-storage:{0}'.format(self.imported_image)
])
def create_on_file(self, filename, label=None, exclude=None):
"""
Create iso filesystem from data tree
There is no label which could be set for iso filesystem
thus this parameter is not used
:param string filename: result file path name
:param string label: unused
:param string exclude: unused
"""
iso = Iso(self.root_dir)
iso.init_iso_creation_parameters(
self.custom_args['create_options']
)
iso.add_efi_loader_parameters()
Command.run(
[
self._find_iso_creation_tool()
] + iso.get_iso_creation_parameters() + [
'-o', filename, self.root_dir
]
)
hybrid_offset = iso.create_header_end_block(filename)
Command.run(
[
self._find_iso_creation_tool(),
'-hide', iso.header_end_name,
'-hide-joliet', iso.header_end_name
] + iso.get_iso_creation_parameters() + [
'-o', filename, self.root_dir
]
)
iso.relocate_boot_catalog(filename)
iso.fix_boot_catalog(filename)
return hybrid_offset
def commit(self):
"""
Update instance directory with contents of registered files
Use an atomic operation that prepares a tmp directory with
all registered files and move it to the directory given at
instance creation time. Please note the operation is not
fully atomic as it uses two move commands in a series
"""
Path.create(self.dirname_tmp)
for origin, tmpname in list(self.collection.items()):
Command.run(
['mv', tmpname,
os.sep.join([self.dirname_tmp, origin])])
Command.run(['mv', self.dirname, self.dirname_wipe])
Command.run(['mv', self.dirname_tmp, self.dirname])
Command.run(['rm', '-rf', self.dirname_wipe])
def _create_iso_install_kernel_and_initrd(self):
boot_path = self.media_dir + '/boot/' + self.arch + '/loader'
Path.create(boot_path)
kernel = Kernel(self.boot_image_task.boot_root_directory)
if kernel.get_kernel():
kernel.copy_kernel(boot_path, '/linux')
else:
raise KiwiInstallBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
if self.xml_state.is_xen_server():
if kernel.get_xen_hypervisor():
kernel.copy_xen_hypervisor(boot_path, '/xen.gz')
else:
raise KiwiInstallBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
if self.initrd_system == 'dracut':
self._create_dracut_install_config()
self._add_system_image_boot_options_to_boot_image()
self.boot_image_task.create_initrd(self.mbrid, 'initrd_kiwi_install')
Command.run([
'mv', self.boot_image_task.initrd_filename, boot_path + '/initrd'
])
def _copy_first_boot_files_to_system_image(self):
boot_names = self.boot_image.get_boot_names()
if self.initrd_system == 'kiwi':
log.info('Copy boot files to system image')
kernel = Kernel(self.boot_image.boot_root_directory)
log.info('--> boot image kernel as %s', boot_names.kernel_name)
kernel.copy_kernel(self.root_dir,
''.join(['/boot/', boot_names.kernel_name]))
if self.xen_server:
if kernel.get_xen_hypervisor():
log.info('--> boot image Xen hypervisor as xen.gz')
kernel.copy_xen_hypervisor(self.root_dir, '/boot/xen.gz')
else:
raise KiwiDiskBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image.boot_root_directory)
log.info('--> initrd archive as %s', boot_names.initrd_name)
Command.run([
'mv', self.boot_image.initrd_filename,
self.root_dir + ''.join(['/boot/', boot_names.initrd_name])
])
def _create_pxe_install_kernel_and_initrd(self):
kernel = Kernel(self.boot_image_task.boot_root_directory)
if kernel.get_kernel():
kernel.copy_kernel(self.pxe_dir, '/pxeboot.kernel')
else:
raise KiwiInstallBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
if self.machine and self.machine.get_domain() == 'dom0':
if kernel.get_xen_hypervisor():
kernel.copy_xen_hypervisor(self.pxe_dir, '/pxeboot.xen.gz')
else:
raise KiwiInstallBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
self.boot_image_task.create_initrd(self.mbrid)
Command.run(
[
'mv', self.boot_image_task.initrd_filename,
self.pxe_dir + '/pxeboot.initrd.xz'
]
)