def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
if not self.unknown_uri:
compressor = Compress(self.image_file)
if compressor.get_format():
compressor.uncompress(True)
self.uncompressed_image = compressor.uncompressed_filename
else:
self.uncompressed_image = self.image_file
image_uri = '{0}:{1}'.format(
self.archive_transport, self.uncompressed_image
)
else:
log.warning('Bypassing base image URI to OCI tools')
image_uri = self.unknown_uri
oci = OCI.new()
oci.import_container_image(image_uri)
oci.unpack()
oci.import_rootfs(self.root_dir)
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
oci.export_container_image(
image_copy, 'oci-archive', Defaults.get_container_base_image_tag()
)
self._make_checksum(image_copy)
def pack_image_to_file(self, filename):
"""
Packs the given oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
oci_image = os.sep.join([
self.oci_dir, ':'.join(['umoci_layout', self.container_tag])
])
additional_tags = []
for tag in self.additional_tags:
additional_tags.extend([
'--additional-tag', '{0}:{1}'.format(self.container_name, tag)
])
# make sure the target tar file does not exist
# skopeo doesn't support force overwrite
Path.wipe(filename)
Command.run(
[
'skopeo', 'copy', 'oci:{0}'.format(
oci_image
),
'docker-archive:{0}:{1}:{2}'.format(
filename, self.container_name, self.container_tag
)
] + additional_tags
)
container_compressor = self.runtime_config.get_container_compression()
if container_compressor:
compressor = Compress(filename)
return compressor.xz(self.runtime_config.get_xz_options())
else:
return filename
def _create_checksum_file(self, checksum, filename):
"""
Creates the text file that contains the checksum
:param str checksum: checksum to include into the file
:param str filename: filename of the output file
"""
compressed_blocks = None
compress = Compress(self.source_filename)
if compress.get_format():
compressed_blocks = self._block_list(
os.path.getsize(self.source_filename))
compress.uncompress(temporary=True)
blocks = self._block_list(
os.path.getsize(compress.uncompressed_filename))
checksum = self._calculate_hash_hexdigest(
hashlib.md5(), compress.uncompressed_filename)
else:
blocks = self._block_list(os.path.getsize(self.source_filename))
with open(filename, encoding=self.ascii, mode='w') as checksum_file:
if compressed_blocks:
checksum_file.write(
'%s %s %s %s %s\n' %
(checksum, blocks.blocks, blocks.blocksize,
compressed_blocks.blocks, compressed_blocks.blocksize))
else:
checksum_file.write(
'%s %s %s\n' % (checksum, blocks.blocks, blocks.blocksize))
def pack_image_to_file(self, filename):
"""
Packs the given oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
additional_tags = []
if 'additional_tags' in self.oci_config:
for tag in self.oci_config['additional_tags']:
additional_tags.extend([
'--additional-tag',
'{0}:{1}'.format(self.oci_config['container_name'], tag)
])
# make sure the target tar file does not exist
# skopeo doesn't support force overwrite
Path.wipe(filename)
Command.run([
'skopeo', 'copy', 'oci:{0}'.format(self.oci.container_name),
'docker-archive:{0}:{1}:{2}'.format(
filename, self.oci_config['container_name'],
self.oci_config['container_tag'])
] + additional_tags)
container_compressor = self.runtime_config.get_container_compression()
if container_compressor:
compressor = Compress(filename)
return compressor.xz(self.runtime_config.get_xz_options())
else:
return filename
def _create_checksum_file(self, checksum, filename):
compressed_blocks = None
compress = Compress(self.source_filename)
if compress.get_format():
compressed_blocks = self._block_list(
os.path.getsize(self.source_filename)
)
compress.uncompress(temporary=True)
blocks = self._block_list(
os.path.getsize(compress.uncompressed_filename)
)
checksum = self._calculate_hash_hexdigest(
hashlib.md5(), compress.uncompressed_filename
)
else:
blocks = self._block_list(
os.path.getsize(self.source_filename)
)
with open(filename, 'w') as checksum_file:
if compressed_blocks:
checksum_file.write(
'%s %s %s %s %s\n' % (
checksum, blocks.blocks, blocks.blocksize,
compressed_blocks.blocks, compressed_blocks.blocksize
)
)
else:
checksum_file.write(
'%s %s %s\n' % (
checksum, blocks.blocks, blocks.blocksize
)
)
def create_initrd(self,
mbrid: Optional[SystemIdentifier] = None,
basename: Optional[str] = None,
install_initrd: bool = False) -> None:
"""
Create initrd from prepared boot system tree and compress the result
:param SystemIdentifier mbrid: instance of ImageIdentifier
:param str basename: base initrd file name
:param bool install_initrd: installation media initrd
"""
if self.is_prepared():
log.info('Creating initrd cpio archive')
# we can't simply exclude boot when building the archive
# because the file boot/mbrid must be preserved. Because of
# that we create a copy of the boot directory and remove
# everything in boot/ except for boot/mbrid. The original
# boot directory should not be changed because we rely
# on other data in boot/ e.g the kernel to be available
# for the entire image building process
if basename:
kiwi_initrd_basename = basename
else:
kiwi_initrd_basename = self.initrd_base_name
temp_boot_root = Temporary(prefix='kiwi_boot_root_copy.').new_dir()
temp_boot_root_directory = temp_boot_root.name
os.chmod(temp_boot_root_directory, 0o755)
data = DataSync(self.boot_root_directory + '/',
temp_boot_root_directory)
data.sync_data(options=['-a'])
boot_directory = temp_boot_root_directory + '/boot'
Path.wipe(boot_directory)
if mbrid:
log.info('--> Importing mbrid: %s', mbrid.get_id())
Path.create(boot_directory)
image_identifier = boot_directory + '/mbrid'
mbrid.write(image_identifier)
cpio = ArchiveCpio(
os.sep.join([self.target_dir, kiwi_initrd_basename]))
# the following is a list of directories which were needed
# during the process of creating an image but not when the
# image is actually booting with this initrd
exclude_from_archive = [
'/' + Defaults.get_shared_cache_location(), '/image',
'/usr/lib/grub*'
]
# the following is a list of directories to exclude which
# are not needed inside of the initrd
exclude_from_archive += [
'/usr/share/doc', '/usr/share/man', '/home', '/media', '/srv'
]
cpio.create(source_dir=temp_boot_root_directory,
exclude=exclude_from_archive)
log.info('--> xz compressing archive')
compress = Compress(
os.sep.join([self.target_dir, kiwi_initrd_basename]))
compress.xz(['--check=crc32', '--lzma2=dict=1MiB', '--threads=0'])
self.initrd_filename = compress.compressed_filename
class TestCompress(object):
@patch('os.path.exists')
def setup(self, mock_exists):
mock_exists.return_value = True
self.compress = Compress('some-file', True)
@raises(KiwiFileNotFound)
def test_source_file_not_found(self):
Compress('some-file')
@patch('kiwi.command.Command.run')
def test_xz(self, mock_command):
self.compress.xz()
mock_command.assert_called_once_with([
'xz', '-f', '--check=crc32', '--lzma2=dict=512KiB', '--keep',
'some-file'
])
assert self.compress.compressed_filename == 'some-file.xz'
@patch('kiwi.command.Command.run')
def test_gzip(self, mock_command):
self.compress.gzip()
mock_command.assert_called_once_with(
['gzip', '-f', '-9', '--keep', 'some-file'])
assert self.compress.compressed_filename == 'some-file.gz'
@patch('kiwi.command.Command.run')
@patch('kiwi.utils.compress.NamedTemporaryFile')
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress(self, mock_format, mock_temp, mock_command):
mock_format.return_value = 'xz'
self.compress.uncompress()
mock_command.assert_called_once_with(['xz', '-d', 'some-file'])
assert self.compress.uncompressed_filename == 'some-file'
@patch('kiwi.command.Command.run')
@patch('kiwi.utils.compress.NamedTemporaryFile')
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress_temporary(self, mock_format, mock_temp, mock_command):
tempfile = mock.Mock()
tempfile.name = 'tempfile'
mock_temp.return_value = tempfile
mock_format.return_value = 'xz'
self.compress.uncompress(temporary=True)
mock_command.assert_called_once_with(
['bash', '-c', 'xz -c -d some-file > tempfile'])
assert self.compress.uncompressed_filename == 'tempfile'
@raises(KiwiCompressionFormatUnknown)
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress_unknown_format(self, mock_format):
mock_format.return_value = None
self.compress.uncompress()
def test_get_format(self):
xz = Compress('../data/xz_data.xz')
assert xz.get_format() == 'xz'
gzip = Compress('../data/gz_data.gz')
assert gzip.get_format() == 'gzip'
def create(self, filename, base_image):
"""
Create compressed oci system container tar archive
:param string filename: archive file name
:param string base_image: archive used as a base image
"""
exclude_list = Defaults.get_exclude_list_for_root_data_sync()
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
oci = OCI()
if base_image:
oci.import_container_image(
'oci-archive:{0}:{1}'.format(
base_image, Defaults.get_container_base_image_tag()
)
)
else:
oci.init_container()
image_ref = '{0}:{1}'.format(
self.oci_config['container_name'], self.oci_config['container_tag']
)
oci.unpack()
oci.sync_rootfs(self.root_dir, exclude_list)
oci.repack(self.oci_config)
oci.set_config(self.oci_config)
oci.post_process()
if self.archive_transport == 'docker-archive':
image_ref = '{0}:{1}'.format(
self.oci_config['container_name'],
self.oci_config['container_tag']
)
additional_refs = []
if 'additional_tags' in self.oci_config:
additional_refs = []
for tag in self.oci_config['additional_tags']:
additional_refs.append('{0}:{1}'.format(
self.oci_config['container_name'], tag
))
else:
image_ref = self.oci_config['container_tag']
additional_refs = []
oci.export_container_image(
filename, self.archive_transport, image_ref, additional_refs
)
if self.runtime_config.get_container_compression():
compressor = Compress(filename)
return compressor.xz(self.runtime_config.get_xz_options())
else:
return filename
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains the raw disk image and its checksum
as well as an install initrd and kernel plus the required
kernel commandline information which needs to be added
as append line in the pxelinux config file on the boot
server
Image types which triggers this builder are:
* installpxe="true|false"
"""
self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.',
dir=self.target_dir)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.xz'])
compress = Compress(source_filename=self.diskname,
keep_source_on_compress=True)
compress.xz(self.xz_options)
Command.run(['mv', compress.compressed_filename, pxe_image_filename])
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.md5'])
checksum = Checksum(self.diskname)
checksum.md5(pxe_md5_filename)
# the kiwi initrd code triggers the install by trigger files
self._create_pxe_install_trigger_files()
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.append'])
cmdline = 'pxe=1'
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self._create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(self.pxename.replace('.xz', ''))
archive.create_xz_compressed(self.pxe_dir, xz_options=self.xz_options)
def test_get_format_invalid_format(self, mock_run):
mock_run.side_effect = ValueError("nothing")
invalid = Compress("../data/gz_data.gz")
invalid.supported_zipper = ["mock_zip"]
with self._caplog.at_level(logging.DEBUG):
assert invalid.get_format() is None
mock_run.assert_called_once_with(
['mock_zip', '-l', '../data/gz_data.gz'])
assert 'Error running "mock_zip -l ../data/gz_data.gz", got a'
' ValueError: nothing' in self._caplog.text
def test_get_format_invalid_format(self, mock_run, mock_log):
mock_run.side_effect = ValueError("nothing")
invalid = Compress("../data/gz_data.gz")
invalid.supported_zipper = ["mock_zip"]
assert invalid.get_format() is None
mock_run.assert_called_once_with(
['mock_zip', '-l', '../data/gz_data.gz'])
mock_log.assert_called_once_with(
'Error running "mock_zip -l ../data/gz_data.gz", got a'
' ValueError: nothing')
def create(self, filename, base_image=None, ensure_empty_tmpdirs=None):
"""
Create WSL/Appx archive
:param string filename: archive file name
:param string base_image: not-supported
:param string ensure_empty_tmpdirs: not-supported
"""
exclude_list = Defaults.\
get_exclude_list_for_root_data_sync() + Defaults.\
get_exclude_list_from_custom_exclude_files(self.root_dir)
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
# The C code of WSL-DistroLauncher harcodes the name for the
# root tarball to be install.tar.gz. Thus we have to use this
# name for the root tarball
archive_file_name = os.sep.join(
[self.meta_data_path, 'install.tar']
)
archive = ArchiveTar(
archive_file_name
)
archive_file_name = archive.create(
self.root_dir, exclude=exclude_list
)
compressor = Compress(archive_file_name)
archive_file_name = compressor.gzip()
filemap_file = Temporary().new_file()
with open(filemap_file.name, 'w') as filemap:
filemap.write('[Files]{0}'.format(os.linesep))
for topdir, dirs, files in sorted(os.walk(self.meta_data_path)):
for entry in sorted(dirs + files):
if entry in files:
mapfile = os.sep.join([topdir, entry])
mapfile_relative = os.path.relpath(mapfile, start=self.meta_data_path)
log.info(
'Adding {0} to Appx filemap as relative path {1}'.format(mapfile, mapfile_relative)
)
filemap.write(
'"{0}" "{1}"{2}'.format(
mapfile, mapfile_relative, os.linesep
)
)
Command.run(
['appx', '-o', filename, '-f', filemap_file.name]
)
return filename
def create_initrd(self, mbrid=None):
"""
Create initrd from prepared boot system tree and compress the result
:param object mbrid: instance of ImageIdentifier
"""
if self.is_prepared():
log.info('Creating initrd cpio archive')
# we can't simply exclude boot when building the archive
# because the file boot/mbrid must be preserved. Because of
# that we create a copy of the boot directory and remove
# everything in boot/ except for boot/mbrid. The original
# boot directory should not be changed because we rely
# on other data in boot/ e.g the kernel to be available
# for the entire image building process
temp_boot_root_directory = mkdtemp(prefix='kiwi_boot_root_copy.')
self.temp_directories.append(temp_boot_root_directory)
data = DataSync(self.boot_root_directory + '/',
temp_boot_root_directory)
data.sync_data(options=['-z', '-a'])
boot_directory = temp_boot_root_directory + '/boot'
Path.wipe(boot_directory)
if mbrid:
log.info('--> Importing mbrid: %s', mbrid.get_id())
Path.create(boot_directory)
image_identifier = boot_directory + '/mbrid'
mbrid.write(image_identifier)
cpio = ArchiveCpio(
os.sep.join([self.target_dir, self.initrd_base_name]))
# the following is a list of directories which were needed
# during the process of creating an image but not when the
# image is actually booting with this initrd
exclude_from_archive = [
'/' + Defaults.get_shared_cache_location(), '/image',
'/usr/lib/grub*'
]
# the following is a list of directories to exclude which
# are not needed inside of the initrd
exclude_from_archive += [
'/usr/share/doc', '/usr/share/man', '/home', '/media', '/srv'
]
cpio.create(source_dir=temp_boot_root_directory,
exclude=exclude_from_archive)
log.info('--> xz compressing archive')
compress = Compress(
os.sep.join([self.target_dir, self.initrd_base_name]))
compress.xz()
self.initrd_filename = compress.compressed_filename
def extract_oci_image(self):
"""
Extract and converts to OCI the image from the provided
image file to a temporary location to KIWI can work with it.
"""
if not self.unknown_uri:
compressor = Compress(self.image_file)
compressor.uncompress(True)
self.uncompressed_image = compressor.uncompressed_filename
skopeo_uri = 'docker-archive:{0}'.format(self.uncompressed_image)
else:
log.warning('Bypassing base image URI to skopeo tool')
skopeo_uri = self.unknown_uri
Command.run([
'skopeo', 'copy', skopeo_uri, 'oci:{0}'.format(self.oci_layout_dir)
])
def extract_oci_image(self):
"""
Extract and converts to OCI the image from the provided
image file to a temporary location to KIWI can work with it.
"""
if not self.unknown_uri:
compressor = Compress(self.image_file)
compressor.uncompress(True)
self.uncompressed_image = compressor.uncompressed_filename
skopeo_uri = 'docker-archive:{0}'.format(self.uncompressed_image)
else:
log.warning('Bypassing base image URI to skopeo tool')
skopeo_uri = self.unknown_uri
Command.run([
'skopeo', 'copy', skopeo_uri,
'oci:{0}:base_layer'.format(self.oci_layout_dir)
])
def pack_image_to_file(self, filename):
"""
Packs the given oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
docker_tarfile = filename.replace('.xz', '')
oci_image = os.sep.join(
[self.oci_dir, ':'.join(['umoci_layout', self.container_tag])])
# make sure the target tar file does not exist
# skopeo doesn't support force overwrite
Path.wipe(docker_tarfile)
Command.run([
'skopeo', 'copy', 'oci:{0}'.format(oci_image),
'docker-archive:{0}:{1}:{2}'.format(docker_tarfile,
self.container_name,
self.container_tag)
])
compressor = Compress(docker_tarfile)
compressor.xz(self.xz_options)
def _create_checksum_file(self, checksum, filename):
compressed_blocks = None
compress = Compress(self.source_filename)
if compress.get_format():
compressed_blocks = self._block_list(
os.path.getsize(self.source_filename))
compress.uncompress(temporary=True)
blocks = self._block_list(
os.path.getsize(compress.uncompressed_filename))
checksum = self._calculate_hash_hexdigest(
hashlib.md5(), compress.uncompressed_filename)
else:
blocks = self._block_list(os.path.getsize(self.source_filename))
with open(filename, 'w') as checksum_file:
if compressed_blocks:
checksum_file.write(
'%s %s %s %s %s\n' %
(checksum, blocks.blocks, blocks.blocksize,
compressed_blocks.blocks, compressed_blocks.blocksize))
else:
checksum_file.write(
'%s %s %s\n' % (checksum, blocks.blocks, blocks.blocksize))
def _create_checksum_file(self, checksum, filename):
"""
Creates the text file that contains the checksum
:param str checksum: checksum to include into the file
:param str filename: filename of the output file
"""
compressed_blocks = None
compress = Compress(self.source_filename)
if compress.get_format():
compressed_blocks = self._block_list(
os.path.getsize(self.source_filename)
)
compress.uncompress(temporary=True)
blocks = self._block_list(
os.path.getsize(compress.uncompressed_filename)
)
checksum = self._calculate_hash_hexdigest(
hashlib.md5(), compress.uncompressed_filename
)
else:
blocks = self._block_list(
os.path.getsize(self.source_filename)
)
with open(filename, 'w') as checksum_file:
if compressed_blocks:
checksum_file.write(
'%s %s %s %s %s\n' % (
checksum, blocks.blocks, blocks.blocksize,
compressed_blocks.blocks, compressed_blocks.blocksize
)
)
else:
checksum_file.write(
'%s %s %s\n' % (
checksum, blocks.blocks, blocks.blocksize
)
)
def setup(self, mock_exists):
mock_exists.return_value = True
self.compress = Compress('some-file', True)
def test_source_file_not_found(self):
Compress('some-file')
def process(self):
"""
Create result bundle from the image build results in the
specified target directory. Each result image will contain
the specified bundle identifier as part of its filename.
Uncompressed image files will also become xz compressed
and a sha sum will be created from every result image
"""
self.manual = Help()
if self._help():
return
if self.command_args['--package-as-rpm']:
Privileges.check_for_root_permissions()
# load serialized result object from target directory
result_directory = os.path.abspath(self.command_args['--target-dir'])
bundle_directory = os.path.abspath(self.command_args['--bundle-dir'])
if result_directory == bundle_directory:
raise KiwiBundleError(
'Bundle directory must be different from target directory')
log.info('Bundle build results from %s', result_directory)
result = Result.load(result_directory + '/kiwi.result')
image_version = result.xml_state.get_image_version()
image_name = result.xml_state.xml_data.get_name()
image_description = result.xml_state.get_description_section()
ordered_results = OrderedDict(sorted(result.get_results().items()))
# hard link bundle files, compress and build checksum
if self.command_args['--package-as-rpm']:
Path.wipe(bundle_directory)
if not os.path.exists(bundle_directory):
Path.create(bundle_directory)
bundle_file_format_name = ''
if 'bundle_format' in ordered_results:
bundle_format = ordered_results['bundle_format']
tags = bundle_format['tags']
bundle_file_format_name = bundle_format['pattern']
# Insert image name
bundle_file_format_name = bundle_file_format_name.replace(
'%N', tags.N)
# Insert Concatenated profile name (_)
bundle_file_format_name = bundle_file_format_name.replace(
'%P', tags.P)
# Insert Architecture name
bundle_file_format_name = bundle_file_format_name.replace(
'%A', tags.A)
# Insert Image build type name
bundle_file_format_name = bundle_file_format_name.replace(
'%T', tags.T)
# Insert Image Major version number
bundle_file_format_name = bundle_file_format_name.replace(
'%M', format(tags.M))
# Insert Image Minor version number
bundle_file_format_name = bundle_file_format_name.replace(
'%m', format(tags.m))
# Insert Image Patch version number
bundle_file_format_name = bundle_file_format_name.replace(
'%p', format(tags.p))
# Insert Bundle ID
bundle_file_format_name = bundle_file_format_name.replace(
'%I', self.command_args['--id'])
del (ordered_results['bundle_format'])
for result_file in list(ordered_results.values()):
if result_file.use_for_bundle:
extension = result_file.filename.split('.').pop()
if bundle_file_format_name:
bundle_file_basename = '.'.join(
[bundle_file_format_name, extension])
else:
bundle_file_basename = os.path.basename(
result_file.filename)
# The bundle id is only taken into account for image results
# which contains the image version appended in its file name
part_name = list(
bundle_file_basename.partition(image_name))
bundle_file_basename = ''.join([
part_name[0], part_name[1], part_name[2].replace(
image_version,
image_version + '-' + self.command_args['--id'])
])
log.info('Creating %s', bundle_file_basename)
bundle_file = ''.join(
[bundle_directory, '/', bundle_file_basename])
Command.run(['cp', result_file.filename, bundle_file])
if result_file.compress:
log.info('--> XZ compressing')
compress = Compress(bundle_file)
compress.xz(self.runtime_config.get_xz_options())
bundle_file = compress.compressed_filename
if self.command_args['--zsync-source'] and result_file.shasum:
# Files with a checksum are considered to be image files
# and are therefore eligible to be provided via the
# requested Partial/differential file download based on
# zsync
zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK)
if zsyncmake:
log.info('--> Creating zsync control file')
Command.run([
zsyncmake, '-e', '-u',
os.sep.join([
self.command_args['--zsync-source'],
os.path.basename(bundle_file)
]), '-o', bundle_file + '.zsync', bundle_file
])
else:
log.warning(
'--> zsyncmake missing, zsync setup skipped')
if result_file.shasum:
log.info('--> Creating SHA 256 sum')
checksum = Checksum(bundle_file)
with open(bundle_file + '.sha256', 'w') as shasum:
shasum.write('{0} {1}{2}'.format(
checksum.sha256(), os.path.basename(bundle_file),
os.linesep))
if self.command_args['--package-as-rpm']:
ResultBundleTask._build_rpm_package(
bundle_directory, bundle_file_format_name or image_name,
image_version, image_description.specification,
list(glob.iglob(f'{bundle_directory}/*')))
class TestCompress(object):
@patch('os.path.exists')
def setup(self, mock_exists):
mock_exists.return_value = True
self.compress = Compress('some-file', True)
@raises(KiwiFileNotFound)
def test_source_file_not_found(self):
Compress('some-file')
@patch('kiwi.command.Command.run')
def test_xz(self, mock_command):
assert self.compress.xz() == 'some-file.xz'
mock_command.assert_called_once_with(
['xz', '-f', '--threads=0', '--keep', 'some-file'])
assert self.compress.compressed_filename == 'some-file.xz'
@patch('kiwi.command.Command.run')
def test_xz_with_custom_options(self, mock_command):
assert self.compress.xz(options=['foo', 'bar']) == 'some-file.xz'
mock_command.assert_called_once_with(
['xz', '-f', 'foo', 'bar', '--keep', 'some-file'])
assert self.compress.compressed_filename == 'some-file.xz'
@patch('kiwi.command.Command.run')
def test_gzip(self, mock_command):
assert self.compress.gzip() == 'some-file.gz'
mock_command.assert_called_once_with(
['gzip', '-f', '-9', '--keep', 'some-file'])
assert self.compress.compressed_filename == 'some-file.gz'
@patch('kiwi.command.Command.run')
@patch('kiwi.utils.compress.NamedTemporaryFile')
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress(self, mock_format, mock_temp, mock_command):
mock_format.return_value = 'xz'
self.compress.uncompress()
mock_command.assert_called_once_with(['xz', '-d', 'some-file'])
assert self.compress.uncompressed_filename == 'some-file'
@patch('kiwi.command.Command.run')
@patch('kiwi.utils.compress.NamedTemporaryFile')
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress_temporary(self, mock_format, mock_temp, mock_command):
tempfile = mock.Mock()
tempfile.name = 'tempfile'
mock_temp.return_value = tempfile
mock_format.return_value = 'xz'
self.compress.uncompress(temporary=True)
mock_command.assert_called_once_with(
['bash', '-c', 'xz -c -d some-file > tempfile'])
assert self.compress.uncompressed_filename == 'tempfile'
@raises(KiwiCompressionFormatUnknown)
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress_unknown_format(self, mock_format):
mock_format.return_value = None
self.compress.uncompress()
@patch('kiwi.path.Path.which')
def test_get_format(self, mock_which):
mock_which.return_value = 'ziptool'
xz = Compress('../data/xz_data.xz')
assert xz.get_format() == 'xz'
gzip = Compress('../data/gz_data.gz')
assert gzip.get_format() == 'gzip'
@patch('kiwi.logger.log.debug')
@patch('kiwi.command.Command.run')
def test_get_format_invalid_format(self, mock_run, mock_log):
mock_run.side_effect = ValueError("nothing")
invalid = Compress("../data/gz_data.gz")
invalid.supported_zipper = ["mock_zip"]
assert invalid.get_format() is None
mock_run.assert_called_once_with(
['mock_zip', '-l', '../data/gz_data.gz'])
mock_log.assert_called_once_with(
'Error running "mock_zip -l ../data/gz_data.gz", got a'
' ValueError: nothing')
def create(self):
"""
Build a pxe image set consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum. The result can be used within the kiwi
PXE boot infrastructure
Image types which triggers this builder are:
* image="pxe"
"""
log.info('Creating PXE root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
self.filesystem_checksum = ''.join([self.image, '.md5'])
checksum = Checksum(self.image)
checksum.md5(self.filesystem_checksum)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[os.path.basename(self.image_name), '-', kernel_data.name]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
# put results into a tarball
Command.run(
[
'tar', '-C', self.target_dir, '-cJf', self.archive_name,
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.image),
os.path.basename(self.filesystem_checksum)
]
)
# store results
self.result.add(
key='pxe_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=False,
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
if self.pxedeploy:
log.warning(
'Creation of client config file from pxedeploy not implemented'
)
return self.result
def test_source_file_not_found(self):
with raises(KiwiFileNotFound):
Compress('some-file')
def setup(self, mock_exists):
mock_exists.return_value = True
self.compress = Compress('some-file', True)
def test_get_format(self, mock_which):
mock_which.return_value = 'ziptool'
xz = Compress('../data/xz_data.xz')
assert xz.get_format() == 'xz'
gzip = Compress('../data/gz_data.gz')
assert gzip.get_format() == 'gzip'
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains:
* The raw system image xz compressed
* The raw system image checksum metadata file
* The append file template for the boot server
* The system image initrd for kexec
* The install initrd
* The kernel
Image types which triggers this builder are:
* installpxe="true|false"
"""
self.pxe_dir = mkdtemp(
prefix='kiwi_pxe_install_media.', dir=self.target_dir
)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.xz'
]
)
compress = Compress(
source_filename=self.diskname,
keep_source_on_compress=True
)
compress.xz(self.xz_options)
Command.run(
['mv', compress.compressed_filename, pxe_image_filename]
)
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.md5'
]
)
checksum = Checksum(self.diskname)
checksum.md5(pxe_md5_filename)
# the install image name is stored in a config file
if self.initrd_system == 'kiwi':
self._write_install_image_info_to_boot_image()
# the kexec required system image initrd is stored for dracut kiwi-dump
if self.initrd_system == 'dracut':
boot_names = self.boot_image_task.get_boot_names()
system_image_initrd = os.sep.join(
[self.root_dir, 'boot', boot_names.initrd_name]
)
target_initrd_name = '{0}/{1}.initrd'.format(
self.pxe_dir, self.xml_state.xml_data.get_name()
)
shutil.copy(
system_image_initrd, target_initrd_name
)
os.chmod(target_initrd_name, 420)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.append'
]
)
if self.initrd_system == 'kiwi':
cmdline = 'pxe=1'
else:
cmdline = ' '.join(
[
'rd.kiwi.install.pxe',
'rd.kiwi.install.image=http://example.com/image.xz'
]
)
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self._create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(
self.pxename.replace('.xz', '')
)
archive.create_xz_compressed(
self.pxe_dir, xz_options=self.xz_options
)
def test_get_format(self, mock_which):
mock_which.return_value = 'ziptool'
xz = Compress('../data/xz_data.xz')
assert xz.get_format() == 'xz'
gzip = Compress('../data/gz_data.gz')
assert gzip.get_format() == 'gzip'
def test_get_format(self):
xz = Compress('../data/xz_data.xz')
assert xz.get_format() == 'xz'
gzip = Compress('../data/gz_data.gz')
assert gzip.get_format() == 'gzip'
def process(self):
"""
Create result bundle from the image build results in the
specified target directory. Each result image will contain
the specified bundle identifier as part of its filename.
Uncompressed image files will also become xz compressed
and a sha sum will be created from every result image
"""
self.manual = Help()
if self._help():
return
# load serialized result object from target directory
result_directory = os.path.abspath(self.command_args['--target-dir'])
bundle_directory = os.path.abspath(self.command_args['--bundle-dir'])
if result_directory == bundle_directory:
raise KiwiBundleError(
'Bundle directory must be different from target directory'
)
log.info(
'Bundle build results from %s', result_directory
)
result = Result.load(
result_directory + '/kiwi.result'
)
image_version = result.xml_state.get_image_version()
ordered_results = OrderedDict(sorted(result.get_results().items()))
# hard link bundle files, compress and build checksum
if not os.path.exists(bundle_directory):
Path.create(bundle_directory)
for result_file in list(ordered_results.values()):
if result_file.use_for_bundle:
bundle_file_basename = os.path.basename(result_file.filename)
# The bundle id is only taken into account for image results
# which contains the image version in its nane
bundle_file_basename = bundle_file_basename.replace(
image_version,
image_version + '-' + self.command_args['--id']
)
log.info('Creating %s', bundle_file_basename)
bundle_file = ''.join(
[bundle_directory, '/', bundle_file_basename]
)
checksum_file = ''.join(
[bundle_directory, '/', bundle_file_basename, '.sha256']
)
Command.run(
[
'cp', result_file.filename, bundle_file
]
)
if result_file.compress:
log.info('--> XZ compressing')
compress = Compress(bundle_file)
compress.xz(self.runtime_config.get_xz_options())
bundle_file = compress.compressed_filename
checksum_file = compress.compressed_filename + '.sha256'
if self.command_args['--zsync-source']:
zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK)
if zsyncmake:
log.info('--> Creating zsync control file')
Command.run(
[
zsyncmake, '-e',
'-u', os.sep.join(
[
self.command_args['--zsync-source'],
os.path.basename(bundle_file)
]
),
'-o', bundle_file + '.zsync',
bundle_file
]
)
else:
log.warning(
'--> zsyncmake missing, zsync setup skipped'
)
if result_file.shasum:
log.info('--> Creating SHA 256 sum')
checksum = Checksum(bundle_file)
with open(checksum_file, 'w') as shasum:
shasum.write(checksum.sha256())
def test_get_format(self):
xz = Compress('../data/xz_data.xz')
assert xz.get_format() == 'xz'
gzip = Compress('../data/gz_data.gz')
assert gzip.get_format() == 'gzip'
def process(self):
"""
Create result bundle from the image build results in the
specified target directory. Each result image will contain
the specified bundle identifier as part of its filename.
Uncompressed image files will also become xz compressed
and a sha sum will be created from every result image
"""
self.manual = Help()
if self._help():
return
# load serialized result object from target directory
result_directory = os.path.abspath(self.command_args['--target-dir'])
bundle_directory = os.path.abspath(self.command_args['--bundle-dir'])
if result_directory == bundle_directory:
raise KiwiBundleError(
'Bundle directory must be different from target directory')
log.info('Bundle build results from %s', result_directory)
result = Result.load(result_directory + '/kiwi.result')
image_version = result.xml_state.get_image_version()
image_name = result.xml_state.xml_data.get_name()
ordered_results = OrderedDict(sorted(result.get_results().items()))
# hard link bundle files, compress and build checksum
if not os.path.exists(bundle_directory):
Path.create(bundle_directory)
for result_file in list(ordered_results.values()):
if result_file.use_for_bundle:
bundle_file_basename = os.path.basename(result_file.filename)
# The bundle id is only taken into account for image results
# which contains the image version appended in its file name
part_name = list(bundle_file_basename.partition(image_name))
bundle_file_basename = ''.join([
part_name[0], part_name[1], part_name[2].replace(
image_version,
image_version + '-' + self.command_args['--id'])
])
log.info('Creating %s', bundle_file_basename)
bundle_file = ''.join(
[bundle_directory, '/', bundle_file_basename])
Command.run(['cp', result_file.filename, bundle_file])
if result_file.compress:
log.info('--> XZ compressing')
compress = Compress(bundle_file)
compress.xz(self.runtime_config.get_xz_options())
bundle_file = compress.compressed_filename
if self.command_args['--zsync-source'] and result_file.shasum:
# Files with a checksum are considered to be image files
# and are therefore eligible to be provided via the
# requested Partial/differential file download based on
# zsync
zsyncmake = Path.which('zsyncmake', access_mode=os.X_OK)
if zsyncmake:
log.info('--> Creating zsync control file')
Command.run([
zsyncmake, '-e', '-u',
os.sep.join([
self.command_args['--zsync-source'],
os.path.basename(bundle_file)
]), '-o', bundle_file + '.zsync', bundle_file
])
else:
log.warning(
'--> zsyncmake missing, zsync setup skipped')
if result_file.shasum:
log.info('--> Creating SHA 256 sum')
checksum = Checksum(bundle_file)
with open(bundle_file + '.sha256', 'w') as shasum:
shasum.write('{0} {1}'.format(
checksum.sha256(), os.path.basename(bundle_file)))
class TestCompress(object):
@patch('os.path.exists')
def setup(self, mock_exists):
mock_exists.return_value = True
self.compress = Compress('some-file', True)
@raises(KiwiFileNotFound)
def test_source_file_not_found(self):
Compress('some-file')
@patch('kiwi.command.Command.run')
def test_xz(self, mock_command):
assert self.compress.xz() == 'some-file.xz'
mock_command.assert_called_once_with(
[
'xz', '-f', '--threads=0', '--keep',
'some-file'
]
)
assert self.compress.compressed_filename == 'some-file.xz'
@patch('kiwi.command.Command.run')
def test_xz_with_custom_options(self, mock_command):
assert self.compress.xz(options=['foo', 'bar']) == 'some-file.xz'
mock_command.assert_called_once_with(
[
'xz', '-f', 'foo', 'bar', '--keep',
'some-file'
]
)
assert self.compress.compressed_filename == 'some-file.xz'
@patch('kiwi.command.Command.run')
def test_gzip(self, mock_command):
assert self.compress.gzip() == 'some-file.gz'
mock_command.assert_called_once_with(
['gzip', '-f', '-9', '--keep', 'some-file']
)
assert self.compress.compressed_filename == 'some-file.gz'
@patch('kiwi.command.Command.run')
@patch('kiwi.utils.compress.NamedTemporaryFile')
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress(self, mock_format, mock_temp, mock_command):
mock_format.return_value = 'xz'
self.compress.uncompress()
mock_command.assert_called_once_with(
['xz', '-d', 'some-file']
)
assert self.compress.uncompressed_filename == 'some-file'
@patch('kiwi.command.Command.run')
@patch('kiwi.utils.compress.NamedTemporaryFile')
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress_temporary(self, mock_format, mock_temp, mock_command):
tempfile = mock.Mock()
tempfile.name = 'tempfile'
mock_temp.return_value = tempfile
mock_format.return_value = 'xz'
self.compress.uncompress(temporary=True)
mock_command.assert_called_once_with(
['bash', '-c', 'xz -c -d some-file > tempfile']
)
assert self.compress.uncompressed_filename == 'tempfile'
@raises(KiwiCompressionFormatUnknown)
@patch('kiwi.utils.compress.Compress.get_format')
def test_uncompress_unknown_format(self, mock_format):
mock_format.return_value = None
self.compress.uncompress()
@patch('kiwi.path.Path.which')
def test_get_format(self, mock_which):
mock_which.return_value = 'ziptool'
xz = Compress('../data/xz_data.xz')
assert xz.get_format() == 'xz'
gzip = Compress('../data/gz_data.gz')
assert gzip.get_format() == 'gzip'
def create(self):
"""
Build a pxe image set consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum. The result can be used within the kiwi
PXE boot infrastructure
Image types which triggers this builder are:
* image="pxe"
:raises KiwiPxeBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
log.info('Creating PXE root filesystem image')
self.filesystem.create()
os.rename(self.filesystem.filename, self.image_name)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
checksum = Checksum(self.image)
checksum.md5(self.checksum_name)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join([
os.path.basename(self.image_name), '-', kernel_data.version,
'.kernel'
])
kernel.copy_kernel(self.target_dir, self.kernel_filename)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[os.path.basename(self.image_name), '-', kernel_data.name])
kernel.copy_xen_hypervisor(self.target_dir,
self.hypervisor_filename)
self.result.add(key='xen_hypervisor',
filename=self.target_dir + '/' +
self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
bash_command = ['tar', '-C', self.target_dir, '-c', '--to-stdout'] + [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.image),
os.path.basename(self.checksum_name)
] + ['|', 'xz', '-f'] + self.xz_options + ['>', self.archive_name]
Command.run(['bash', '-c', ' '.join(bash_command)])
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(), self.archive_name)
# store results
self.result.add(
key='pxe_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(default=False),
shasum=True)
# create image root metadata
self.result.add(key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
def create_recovery_archive(self):
"""
Create a compressed recovery archive from the root tree
for use with kiwi's recvoery system. The method creates
additional data into the image root filesystem which is
deleted prior to the creation of a new recovery data set
"""
# cleanup
bash_comand = ['rm', '-f', self.root_dir + '/recovery.*']
Command.run(['bash', '-c', ' '.join(bash_comand)])
if not self.oemconfig['recovery']:
return
# recovery.tar
log.info('Creating recovery tar archive')
metadata = {
'archive_name': self.root_dir + '/recovery.tar',
'archive_filecount': self.root_dir + '/recovery.tar.files',
'archive_size': self.root_dir + '/recovery.tar.size',
'partition_size': self.root_dir + '/recovery.partition.size',
'partition_filesystem': self.root_dir + '/recovery.tar.filesystem'
}
recovery_archive = NamedTemporaryFile(delete=False)
archive = ArchiveTar(filename=recovery_archive.name,
create_from_file_list=False)
archive.create(source_dir=self.root_dir,
exclude=['dev', 'proc', 'sys'],
options=[
'--numeric-owner', '--hard-dereference',
'--preserve-permissions'
])
Command.run(['mv', recovery_archive.name, metadata['archive_name']])
# recovery.tar.filesystem
recovery_filesystem = self.xml_state.build_type.get_filesystem()
with open(metadata['partition_filesystem'], 'w') as partfs:
partfs.write('%s' % recovery_filesystem)
log.info('--> Recovery partition filesystem: %s', recovery_filesystem)
# recovery.tar.files
bash_comand = ['tar', '-tf', metadata['archive_name'], '|', 'wc', '-l']
tar_files_call = Command.run(['bash', '-c', ' '.join(bash_comand)])
tar_files_count = int(tar_files_call.output.rstrip('\n'))
with open(metadata['archive_filecount'], 'w') as files:
files.write('%d\n' % tar_files_count)
log.info('--> Recovery file count: %d files', tar_files_count)
# recovery.tar.size
recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
with open(metadata['archive_size'], 'w') as size:
size.write('%d' % recovery_archive_size_bytes)
log.info('--> Recovery uncompressed size: %d mbytes',
int(recovery_archive_size_bytes / 1048576))
# recovery.tar.gz
log.info('--> Compressing recovery archive')
compress = Compress(self.root_dir + '/recovery.tar')
compress.gzip()
# recovery.partition.size
recovery_archive_gz_size_mbytes = int(
os.path.getsize(metadata['archive_name'] + '.gz') / 1048576)
recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
+ Defaults.get_recovery_spare_mbytes()
with open(metadata['partition_size'], 'w') as gzsize:
gzsize.write('%d' % recovery_partition_mbytes)
log.info('--> Recovery partition size: %d mbytes',
recovery_partition_mbytes)
# delete recovery archive if inplace recovery is requested
# In this mode the recovery archive is created at install time
# and not at image creation time. However the recovery metadata
# is preserved in order to be able to check if enough space
# is available on the disk to create the recovery archive.
if self.oemconfig['recovery_inplace']:
log.info('--> Inplace recovery requested, deleting archive')
Path.wipe(metadata['archive_name'] + '.gz')
def create(self):
"""
Build a pxe image set consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum. The result can be used within the kiwi
PXE boot infrastructure
Image types which triggers this builder are:
* image="pxe"
:raises KiwiPxeBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
log.info('Creating PXE root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
self.filesystem_checksum = ''.join([self.image, '.md5'])
checksum = Checksum(self.image)
checksum.md5(self.filesystem_checksum)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[os.path.basename(self.image_name), '-', kernel_data.name]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
bash_command = [
'tar', '-C', self.target_dir, '-c', '--to-stdout'
] + [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.image),
os.path.basename(self.filesystem_checksum)
] + [
'|', 'xz', '-f'
] + self.xz_options + [
'>', self.archive_name
]
Command.run(['bash', '-c', ' '.join(bash_command)])
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.archive_name
)
# store results
self.result.add(
key='pxe_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=False,
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
if self.pxedeploy:
log.warning(
'Creation of client config file from pxedeploy not implemented'
)
return self.result
def create(self) -> Result:
"""
Build a component image consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum.
Image types which triggers this builder are:
* image="kis"
* image="pxe"
:raises KiwiKisBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
if self.filesystem:
log.info('Creating root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating root filesystem MD5 checksum')
checksum = Checksum(self.image)
checksum.md5(self.checksum_name)
# prepare boot(initrd) root system
log.info('Creating boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiKisBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
hypervisor_data = kernel.get_xen_hypervisor()
if hypervisor_data:
self.hypervisor_filename = ''.join(
[
os.path.basename(self.image_name),
'-', hypervisor_data.name
]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiKisBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd
self.boot_image_task.create_initrd()
# create append information
# this information helps to configure the deployment infrastructure
if self.filesystem and self.filesystem.root_uuid \
and self.initrd_system == 'dracut':
cmdline = 'root=UUID={}'.format(self.filesystem.root_uuid)
if self.custom_cmdline:
cmdline += ' {}'.format(self.custom_cmdline)
with open(self.append_file, 'w') as append:
append.write(cmdline)
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
kis_tarball_files = [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.checksum_name),
]
if self.image:
kis_tarball_files.append(os.path.basename(self.image))
if self.filesystem and self.filesystem.root_uuid \
and self.initrd_system == 'dracut':
kis_tarball_files.append(os.path.basename(self.append_file))
kis_tarball = ArchiveTar(
self.archive_name,
create_from_file_list=True,
file_list=kis_tarball_files
)
if self.compressed:
self.archive_name = kis_tarball.create(self.target_dir)
else:
self.archive_name = kis_tarball.create_xz_compressed(
self.target_dir, xz_options=self.xz_options
)
Result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.archive_name
)
# store results
self.result.add(
key='kis_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(
default=False
),
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_changes',
filename=self.system_setup.export_package_changes(
self.target_dir
),
use_for_bundle=True,
compress=True,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains:
* The raw system image xz compressed
* The raw system image checksum metadata file
* The append file template for the boot server
* The system image initrd for kexec
* The install initrd
* The kernel
Image types which triggers this builder are:
* installpxe="true|false"
"""
self.pxe_dir = mkdtemp(prefix='kiwi_pxe_install_media.',
dir=self.target_dir)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.xz'])
compress = Compress(source_filename=self.diskname,
keep_source_on_compress=True)
compress.xz(self.xz_options)
Command.run(['mv', compress.compressed_filename, pxe_image_filename])
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.md5'])
checksum = Checksum(self.diskname)
checksum.md5(pxe_md5_filename)
# the install image name is stored in a config file
if self.initrd_system == 'kiwi':
self._write_install_image_info_to_boot_image()
# the kexec required system image initrd is stored for dracut kiwi-dump
if self.initrd_system == 'dracut':
boot_names = self.boot_image_task.get_boot_names()
system_image_initrd = os.sep.join(
[self.root_dir, 'boot', boot_names.initrd_name])
target_initrd_name = '{0}/{1}.initrd'.format(
self.pxe_dir, self.xml_state.xml_data.get_name())
shutil.copy(system_image_initrd, target_initrd_name)
os.chmod(target_initrd_name, 420)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.append'])
if self.initrd_system == 'kiwi':
cmdline = 'pxe=1'
else:
cmdline = ' '.join([
'rd.kiwi.install.pxe',
'rd.kiwi.install.image=http://example.com/image.xz'
])
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self._create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(self.pxename.replace('.xz', ''))
archive.create_xz_compressed(self.pxe_dir, xz_options=self.xz_options)
def create_recovery_archive(self):
"""
Create a compressed recovery archive from the root tree
for use with kiwi's recvoery system. The method creates
additional data into the image root filesystem which is
deleted prior to the creation of a new recovery data set
"""
# cleanup
bash_comand = [
'rm', '-f', self.root_dir + '/recovery.*'
]
Command.run(['bash', '-c', ' '.join(bash_comand)])
if not self.oemconfig['recovery']:
return
# recovery.tar
log.info('Creating recovery tar archive')
metadata = {
'archive_name':
self.root_dir + '/recovery.tar',
'archive_filecount':
self.root_dir + '/recovery.tar.files',
'archive_size':
self.root_dir + '/recovery.tar.size',
'partition_size':
self.root_dir + '/recovery.partition.size',
'partition_filesystem':
self.root_dir + '/recovery.tar.filesystem'
}
recovery_archive = NamedTemporaryFile(
delete=False
)
archive = ArchiveTar(
filename=recovery_archive.name,
create_from_file_list=False
)
archive.create(
source_dir=self.root_dir,
exclude=['dev', 'proc', 'sys'],
options=[
'--numeric-owner',
'--hard-dereference',
'--preserve-permissions'
]
)
Command.run(
['mv', recovery_archive.name, metadata['archive_name']]
)
# recovery.tar.filesystem
recovery_filesystem = self.xml_state.build_type.get_filesystem()
with open(metadata['partition_filesystem'], 'w') as partfs:
partfs.write('%s' % recovery_filesystem)
log.info(
'--> Recovery partition filesystem: %s', recovery_filesystem
)
# recovery.tar.files
bash_comand = [
'tar', '-tf', metadata['archive_name'], '|', 'wc', '-l'
]
tar_files_call = Command.run(
['bash', '-c', ' '.join(bash_comand)]
)
tar_files_count = int(tar_files_call.output.rstrip('\n'))
with open(metadata['archive_filecount'], 'w') as files:
files.write('%d\n' % tar_files_count)
log.info(
'--> Recovery file count: %d files', tar_files_count
)
# recovery.tar.size
recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
with open(metadata['archive_size'], 'w') as size:
size.write('%d' % recovery_archive_size_bytes)
log.info(
'--> Recovery uncompressed size: %d mbytes',
int(recovery_archive_size_bytes / 1048576)
)
# recovery.tar.gz
log.info('--> Compressing recovery archive')
compress = Compress(self.root_dir + '/recovery.tar')
compress.gzip()
# recovery.partition.size
recovery_archive_gz_size_mbytes = int(
os.path.getsize(metadata['archive_name'] + '.gz') / 1048576
)
recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
+ Defaults.get_recovery_spare_mbytes()
with open(metadata['partition_size'], 'w') as gzsize:
gzsize.write('%d' % recovery_partition_mbytes)
log.info(
'--> Recovery partition size: %d mbytes',
recovery_partition_mbytes
)
# delete recovery archive if inplace recovery is requested
# In this mode the recovery archive is created at install time
# and not at image creation time. However the recovery metadata
# is preserved in order to be able to check if enough space
# is available on the disk to create the recovery archive.
if self.oemconfig['recovery_inplace']:
log.info(
'--> Inplace recovery requested, deleting archive'
)
Path.wipe(metadata['archive_name'] + '.gz')
