def test_create_from_dir_with_excludes(self, mock_command):
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', '.', '--exclude', './foo', '--exclude', './bar'
])
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image', self.oci_layout_dir,
self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep]))
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
Command.run([
'umoci', 'unpack', '--image',
'{0}:base_layer'.format(self.oci_layout_dir), self.oci_unpack_dir
])
synchronizer = DataSync(
os.sep.join([self.oci_unpack_dir, 'rootfs', '']),
''.join([self.root_dir, os.sep])
)
synchronizer.sync_data(options=['-a', '-H', '-X', '-A'])
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
class TestArchiveTar(object):
def setup(self):
self.archive = ArchiveTar('foo.tar')
@patch('kiwi.archive.tar.Command.run')
def test_extract(self, mock_command):
self.archive.extract('destination')
mock_command.assert_called_once_with(
['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar'])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create('source-dir')
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
def test_create_from_dir_with_excludes(self, mock_command):
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', '.', '--exclude', './foo', '--exclude', './bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create_xz_compressed('source-dir')
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-J', '-f', 'foo.tar.xz', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create_gnu_gzip_compressed('source-dir')
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--format=gnu', '-cSz', '-f',
'foo.tar.gz', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_exclude(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create('source-dir', ['foo'])
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', 'bar'
])
def test_create_from_dir_with_excludes(self, mock_command):
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.',
'--exclude', './foo', '--exclude', './bar'
]
)
def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
command = mock.Mock()
command.output = 'version 1.26.1'
mock_command.return_value = command
archive = ArchiveTar('foo.tar')
mock_os_dir.return_value = ['foo', 'bar']
archive.create('source-dir')
calls = [
call(['tar', '--version']),
call([
'tar', '-C', 'source-dir', '-c', '-f', 'foo.tar', 'foo', 'bar'
])
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
def test_create_from_dir_with_excludes(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
calls = [
call(['tar', '--version']),
call([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.', '--exclude', './foo', '--exclude',
'./bar'
])
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
command = mock.Mock()
command.output = 'version 1.26.1'
mock_command.return_value = command
archive = ArchiveTar('foo.tar')
mock_os_dir.return_value = ['foo', 'bar']
archive.create('source-dir')
calls = [
call(['tar', '--version']),
call(
[
'tar', '-C', 'source-dir',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
def test_create_from_dir_with_excludes(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
calls = [
call(['tar', '--version']),
call(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.',
'--exclude', './foo', '--exclude', './bar'
]
)
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
def sync_data(self):
"""
Synchronize data from the given base image to the target root
directory.
"""
self.extract_oci_image()
oci = OCI('base_layer', self.oci_layout_dir)
oci.unpack()
oci.import_rootfs(self.root_dir)
# A copy of the uncompressed image and its checksum are
# kept inside the root_dir in order to ensure the later steps
# i.e. system create are atomic and don't need any third
# party archive.
image_copy = Defaults.get_imported_root_image(self.root_dir)
Path.create(os.path.dirname(image_copy))
image_tar = ArchiveTar(image_copy)
image_tar.create(self.oci_layout_dir)
self._make_checksum(image_copy)
def create(self, filename, base_image=None, ensure_empty_tmpdirs=None):
"""
Create WSL/Appx archive
:param string filename: archive file name
:param string base_image: not-supported
:param string ensure_empty_tmpdirs: not-supported
"""
exclude_list = Defaults.\
get_exclude_list_for_root_data_sync() + Defaults.\
get_exclude_list_from_custom_exclude_files(self.root_dir)
exclude_list.append('boot')
exclude_list.append('dev')
exclude_list.append('sys')
exclude_list.append('proc')
# The C code of WSL-DistroLauncher harcodes the name for the
# root tarball to be install.tar.gz. Thus we have to use this
# name for the root tarball
archive_file_name = os.sep.join(
[self.meta_data_path, 'install.tar']
)
archive = ArchiveTar(
archive_file_name
)
archive_file_name = archive.create(
self.root_dir, exclude=exclude_list
)
compressor = Compress(archive_file_name)
archive_file_name = compressor.gzip()
filemap_file = Temporary().new_file()
with open(filemap_file.name, 'w') as filemap:
filemap.write('[Files]{0}'.format(os.linesep))
for topdir, dirs, files in sorted(os.walk(self.meta_data_path)):
for entry in sorted(dirs + files):
if entry in files:
mapfile = os.sep.join([topdir, entry])
mapfile_relative = os.path.relpath(mapfile, start=self.meta_data_path)
log.info(
'Adding {0} to Appx filemap as relative path {1}'.format(mapfile, mapfile_relative)
)
filemap.write(
'"{0}" "{1}"{2}'.format(
mapfile, mapfile_relative, os.linesep
)
)
Command.run(
['appx', '-o', filename, '-f', filemap_file.name]
)
return filename
def pack_image_to_file(self, filename):
"""
Packs the oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
oci_tarfile = ArchiveTar(filename)
container_compressor = self.runtime_config.get_container_compression()
if container_compressor:
return oci_tarfile.create_xz_compressed(
image_dir, xz_options=self.runtime_config.get_xz_options())
else:
return oci_tarfile.create(image_dir)
def pack_image_to_file(self, filename):
"""
Packs the oci image into the given filename.
:param string filename: file name of the resulting packed image
"""
image_dir = os.sep.join([self.oci_dir, 'umoci_layout'])
oci_tarfile = ArchiveTar(filename)
container_compressor = self.runtime_config.get_container_compression()
if container_compressor:
return oci_tarfile.create_xz_compressed(
image_dir, xz_options=self.runtime_config.get_xz_options()
)
else:
return oci_tarfile.create(image_dir)
class TestArchiveTar(object):
@patch('kiwi.archive.tar.Command.run')
def setup(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
@raises(KiwiCommandCapabilitiesError)
@patch('kiwi.archive.tar.Command.run')
def test_invalid_tar_command_version(self, mock_command):
command = mock.Mock()
command.output = 'version cannot be parsed'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
@patch('kiwi.archive.tar.Command.run')
def test_extract(self, mock_command):
self.archive.extract('destination')
mock_command.assert_called_once_with(
['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar'])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir') == 'foo.tar'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
def test_append_files(self, mock_command):
assert self.archive.append_files('source-dir', ['foo', 'bar']) \
== 'foo.tar'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '-r', '--file=' + self.archive.filename,
'--xattrs', '--xattrs-include=*', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_with_options(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir',
options=['--fake-option',
'fake_arg']) == 'foo.tar'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--fake-option', 'fake_arg', '--xattrs',
'--xattrs-include=*', '-c', '-f', 'foo.tar', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
command = mock.Mock()
command.output = 'version 1.26.1'
mock_command.return_value = command
archive = ArchiveTar('foo.tar')
mock_os_dir.return_value = ['foo', 'bar']
assert archive.create('source-dir') == 'foo.tar'
calls = [
call(['tar', '--version']),
call([
'tar', '-C', 'source-dir', '-c', '-f', 'foo.tar', 'foo', 'bar'
])
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
@patch('kiwi.archive.tar.Command.run')
def test_create_from_dir_with_excludes(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
archive = ArchiveTar('foo.tar', False)
assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
calls = [
call(['tar', '--version']),
call([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.', '--exclude', './foo', '--exclude',
'./bar'
])
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_xz_compressed('source-dir') == 'foo.tar.xz'
mock_command.assert_called_once_with([
'bash', '-c', ' '.join([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '--to-stdout', 'foo', 'bar', '|', 'xz', '-f',
'--threads=0', '>', 'foo.tar.xz'
])
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed_with_custom_xz_options(
self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_xz_compressed('source-dir',
xz_options=['-a', '-b'
]) == 'foo.tar.xz'
mock_command.assert_called_once_with([
'bash', '-c', ' '.join([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '--to-stdout', 'foo', 'bar', '|', 'xz', '-f', '-a', '-b',
'>', 'foo.tar.xz'
])
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_gnu_gzip_compressed('source-dir') \
== 'foo.tar.gz'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--format=gnu', '-cSz', '-f',
'foo.tar.gz', 'foo', 'bar'
])
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_exclude(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir', ['foo']) == 'foo.tar'
mock_command.assert_called_once_with([
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*', '-c',
'-f', 'foo.tar', 'bar'
])
class TestArchiveTar(object):
@patch('kiwi.archive.tar.Command.run')
def setup(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
@raises(KiwiCommandCapabilitiesError)
@patch('kiwi.archive.tar.Command.run')
def test_invalid_tar_command_version(self, mock_command):
command = mock.Mock()
command.output = 'version cannot be parsed'
mock_command.return_value = command
self.archive = ArchiveTar('foo.tar')
@patch('kiwi.archive.tar.Command.run')
def test_extract(self, mock_command):
self.archive.extract('destination')
mock_command.assert_called_once_with(
['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar']
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir') == 'foo.tar'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
def test_append_files(self, mock_command):
assert self.archive.append_files('source-dir', ['foo', 'bar']) \
== 'foo.tar'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '-r',
'--file=' + self.archive.filename,
'--xattrs', '--xattrs-include=*',
'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_with_options(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir', options=[
'--fake-option', 'fake_arg'
]) == 'foo.tar'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--fake-option', 'fake_arg',
'--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_with_old_tar_version(self, mock_os_dir, mock_command):
command = mock.Mock()
command.output = 'version 1.26.1'
mock_command.return_value = command
archive = ArchiveTar('foo.tar')
mock_os_dir.return_value = ['foo', 'bar']
assert archive.create('source-dir') == 'foo.tar'
calls = [
call(['tar', '--version']),
call(
[
'tar', '-C', 'source-dir',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
@patch('kiwi.archive.tar.Command.run')
def test_create_from_dir_with_excludes(self, mock_command):
command = mock.Mock()
command.output = 'version 1.27.0'
mock_command.return_value = command
archive = ArchiveTar('foo.tar', False)
assert archive.create('source-dir', ['foo', 'bar']) == 'foo.tar'
calls = [
call(['tar', '--version']),
call(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.',
'--exclude', './foo', '--exclude', './bar'
]
)
]
mock_command.assert_has_calls(calls)
assert mock_command.call_count == 2
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_xz_compressed('source-dir') == 'foo.tar.xz'
mock_command.assert_called_once_with(
[
'bash', '-c',
' '.join([
'tar', '-C', 'source-dir', '--xattrs',
'--xattrs-include=*', '-c', '--to-stdout',
'foo', 'bar', '|', 'xz', '-f', '--threads=0',
'>', 'foo.tar.xz'
])
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed_with_custom_xz_options(
self, mock_os_dir, mock_command
):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_xz_compressed(
'source-dir', xz_options=['-a', '-b']
) == 'foo.tar.xz'
mock_command.assert_called_once_with(
[
'bash', '-c',
' '.join([
'tar', '-C', 'source-dir', '--xattrs',
'--xattrs-include=*', '-c', '--to-stdout',
'foo', 'bar', '|', 'xz', '-f', '-a', '-b',
'>', 'foo.tar.xz'
])
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create_gnu_gzip_compressed('source-dir') \
== 'foo.tar.gz'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--format=gnu', '-cSz', '-f', 'foo.tar.gz', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_exclude(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
assert self.archive.create('source-dir', ['foo']) == 'foo.tar'
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'bar'
]
)
def create(self):
"""
Build a pxe image set consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum. The result can be used within the kiwi
PXE boot infrastructure
Image types which triggers this builder are:
* image="pxe"
:raises KiwiPxeBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
log.info('Creating PXE root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
checksum = Checksum(self.image)
checksum.md5(self.checksum_name)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[os.path.basename(self.image_name), '-', kernel_data.name]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
pxe_tarball_files = [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.image),
os.path.basename(self.checksum_name)
]
pxe_tarball = ArchiveTar(
self.archive_name,
create_from_file_list=True,
file_list=pxe_tarball_files
)
if self.compressed:
self.archive_name = pxe_tarball.create(self.target_dir)
else:
self.archive_name = pxe_tarball.create_xz_compressed(
self.target_dir, xz_options=self.xz_options
)
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.archive_name
)
# store results
self.result.add(
key='pxe_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(
default=False
),
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result
def create(self) -> Result:
"""
Build a component image consisting out of a boot image(initrd)
plus its appropriate kernel files and the root filesystem
image with a checksum.
Image types which triggers this builder are:
* image="kis"
* image="pxe"
:raises KiwiKisBootImageError: if no kernel or hipervisor is found
in boot image tree
:return: result
:rtype: instance of :class:`Result`
"""
if self.filesystem:
log.info('Creating root filesystem image')
self.filesystem.create()
os.rename(
self.filesystem.filename, self.image_name
)
self.image = self.image_name
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz(self.xz_options)
self.image = compress.compressed_filename
log.info('Creating root filesystem MD5 checksum')
checksum = Checksum(self.image)
checksum.md5(self.checksum_name)
# prepare boot(initrd) root system
log.info('Creating boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[
os.path.basename(self.image_name), '-',
kernel_data.version, '.kernel'
]
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiKisBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.xen_server:
hypervisor_data = kernel.get_xen_hypervisor()
if hypervisor_data:
self.hypervisor_filename = ''.join(
[
os.path.basename(self.image_name),
'-', hypervisor_data.name
]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
key='xen_hypervisor',
filename=self.target_dir + '/' + self.hypervisor_filename,
use_for_bundle=True,
compress=False,
shasum=True
)
else:
raise KiwiKisBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd
self.boot_image_task.create_initrd()
# create append information
# this information helps to configure the deployment infrastructure
if self.filesystem and self.filesystem.root_uuid \
and self.initrd_system == 'dracut':
cmdline = 'root=UUID={}'.format(self.filesystem.root_uuid)
if self.custom_cmdline:
cmdline += ' {}'.format(self.custom_cmdline)
with open(self.append_file, 'w') as append:
append.write(cmdline)
# put results into a tarball
if not self.xz_options:
self.xz_options = Defaults.get_xz_compression_options()
kis_tarball_files = [
self.kernel_filename,
os.path.basename(self.boot_image_task.initrd_filename),
os.path.basename(self.checksum_name),
]
if self.image:
kis_tarball_files.append(os.path.basename(self.image))
if self.filesystem and self.filesystem.root_uuid \
and self.initrd_system == 'dracut':
kis_tarball_files.append(os.path.basename(self.append_file))
kis_tarball = ArchiveTar(
self.archive_name,
create_from_file_list=True,
file_list=kis_tarball_files
)
if self.compressed:
self.archive_name = kis_tarball.create(self.target_dir)
else:
self.archive_name = kis_tarball.create_xz_compressed(
self.target_dir, xz_options=self.xz_options
)
Result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.archive_name
)
# store results
self.result.add(
key='kis_archive',
filename=self.archive_name,
use_for_bundle=True,
compress=self.runtime_config.get_bundle_compression(
default=False
),
shasum=True
)
# create image root metadata
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_changes',
filename=self.system_setup.export_package_changes(
self.target_dir
),
use_for_bundle=True,
compress=True,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result
class TestArchiveTar(object):
def setup(self):
self.archive = ArchiveTar('foo.tar')
@patch('kiwi.archive.tar.Command.run')
def test_extract(self, mock_command):
self.archive.extract('destination')
mock_command.assert_called_once_with(
['tar', '-C', 'destination', '-x', '-v', '-f', 'foo.tar']
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create('source-dir')
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
def test_create_from_dir_with_excludes(self, mock_command):
archive = ArchiveTar('foo.tar', False)
archive.create('source-dir', ['foo', 'bar'])
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', '.',
'--exclude', './foo', '--exclude', './bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_xz_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create_xz_compressed('source-dir')
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--xattrs', '--xattrs-include=*',
'-c', '-J', '-f', 'foo.tar.xz', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_gnu_gzip_compressed(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create_gnu_gzip_compressed('source-dir')
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir',
'--format=gnu', '-cSz', '-f', 'foo.tar.gz', 'foo', 'bar'
]
)
@patch('kiwi.archive.tar.Command.run')
@patch('os.listdir')
def test_create_exclude(self, mock_os_dir, mock_command):
mock_os_dir.return_value = ['foo', 'bar']
self.archive.create('source-dir', ['foo'])
mock_command.assert_called_once_with(
[
'tar', '-C', 'source-dir', '--xattrs', '--xattrs-include=*',
'-c', '-f', 'foo.tar', 'bar'
]
)
def create_recovery_archive(self):
"""
Create a compressed recovery archive from the root tree
for use with kiwi's recvoery system. The method creates
additional data into the image root filesystem which is
deleted prior to the creation of a new recovery data set
"""
# cleanup
bash_comand = ['rm', '-f', self.root_dir + '/recovery.*']
Command.run(['bash', '-c', ' '.join(bash_comand)])
if not self.oemconfig['recovery']:
return
# recovery.tar
log.info('Creating recovery tar archive')
metadata = {
'archive_name': self.root_dir + '/recovery.tar',
'archive_filecount': self.root_dir + '/recovery.tar.files',
'archive_size': self.root_dir + '/recovery.tar.size',
'partition_size': self.root_dir + '/recovery.partition.size',
'partition_filesystem': self.root_dir + '/recovery.tar.filesystem'
}
recovery_archive = NamedTemporaryFile(delete=False)
archive = ArchiveTar(filename=recovery_archive.name,
create_from_file_list=False)
archive.create(source_dir=self.root_dir,
exclude=['dev', 'proc', 'sys'],
options=[
'--numeric-owner', '--hard-dereference',
'--preserve-permissions'
])
Command.run(['mv', recovery_archive.name, metadata['archive_name']])
# recovery.tar.filesystem
recovery_filesystem = self.xml_state.build_type.get_filesystem()
with open(metadata['partition_filesystem'], 'w') as partfs:
partfs.write('%s' % recovery_filesystem)
log.info('--> Recovery partition filesystem: %s', recovery_filesystem)
# recovery.tar.files
bash_comand = ['tar', '-tf', metadata['archive_name'], '|', 'wc', '-l']
tar_files_call = Command.run(['bash', '-c', ' '.join(bash_comand)])
tar_files_count = int(tar_files_call.output.rstrip('\n'))
with open(metadata['archive_filecount'], 'w') as files:
files.write('%d\n' % tar_files_count)
log.info('--> Recovery file count: %d files', tar_files_count)
# recovery.tar.size
recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
with open(metadata['archive_size'], 'w') as size:
size.write('%d' % recovery_archive_size_bytes)
log.info('--> Recovery uncompressed size: %d mbytes',
int(recovery_archive_size_bytes / 1048576))
# recovery.tar.gz
log.info('--> Compressing recovery archive')
compress = Compress(self.root_dir + '/recovery.tar')
compress.gzip()
# recovery.partition.size
recovery_archive_gz_size_mbytes = int(
os.path.getsize(metadata['archive_name'] + '.gz') / 1048576)
recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
+ Defaults.get_recovery_spare_mbytes()
with open(metadata['partition_size'], 'w') as gzsize:
gzsize.write('%d' % recovery_partition_mbytes)
log.info('--> Recovery partition size: %d mbytes',
recovery_partition_mbytes)
# delete recovery archive if inplace recovery is requested
# In this mode the recovery archive is created at install time
# and not at image creation time. However the recovery metadata
# is preserved in order to be able to check if enough space
# is available on the disk to create the recovery archive.
if self.oemconfig['recovery_inplace']:
log.info('--> Inplace recovery requested, deleting archive')
Path.wipe(metadata['archive_name'] + '.gz')
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains:
* The raw system image xz compressed
* The raw system image checksum metadata file
* The append file template for the boot server
* The system image initrd for kexec
* The install initrd
* The kernel
Image types which triggers this builder are:
* installpxe="true|false"
"""
self.pxe_dir = mkdtemp(
prefix='kiwi_pxe_install_media.', dir=self.target_dir
)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[
self.pxe_dir, '/',
self.pxename, '.xz'
]
)
compress = Compress(
source_filename=self.diskname,
keep_source_on_compress=True
)
compress.xz(self.xz_options)
Command.run(
['mv', compress.compressed_filename, pxe_image_filename]
)
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[
self.pxe_dir, '/',
self.pxename, '.md5'
]
)
checksum = Checksum(self.diskname)
checksum.md5(pxe_md5_filename)
# the install image name is stored in a config file
if self.initrd_system == 'kiwi':
self._write_install_image_info_to_boot_image()
# the kexec required system image initrd is stored for dracut kiwi-dump
if self.initrd_system == 'dracut':
boot_names = self.boot_image_task.get_boot_names()
system_image_initrd = os.sep.join(
[self.root_dir, 'boot', boot_names.initrd_name]
)
target_initrd_name = '{0}/{1}.initrd'.format(
self.pxe_dir, self.pxename
)
shutil.copy(
system_image_initrd, target_initrd_name
)
os.chmod(target_initrd_name, 420)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[
self.pxe_dir, '/', self.pxename, '.append'
]
)
if self.initrd_system == 'kiwi':
cmdline = 'pxe=1'
else:
cmdline = ' '.join(
[
'rd.kiwi.install.pxe',
'rd.kiwi.install.image=http://example.com/image.xz'
]
)
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self._create_pxe_install_kernel_and_initrd()
# create pxe image bound boot config file, contents can be
# changed but presence is required.
log.info('Creating pxe install boot options file')
configname = 'pxeboot.{0}.config.bootoptions'.format(self.pxename)
shutil.copy(
os.sep.join([self.root_dir, 'config.bootoptions']),
os.sep.join([self.pxe_dir, configname])
)
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(self.pxetarball)
archive.create(self.pxe_dir)
def create_recovery_archive(self):
"""
Create a compressed recovery archive from the root tree
for use with kiwi's recvoery system. The method creates
additional data into the image root filesystem which is
deleted prior to the creation of a new recovery data set
"""
# cleanup
bash_comand = [
'rm', '-f', self.root_dir + '/recovery.*'
]
Command.run(['bash', '-c', ' '.join(bash_comand)])
if not self.oemconfig['recovery']:
return
# recovery.tar
log.info('Creating recovery tar archive')
metadata = {
'archive_name':
self.root_dir + '/recovery.tar',
'archive_filecount':
self.root_dir + '/recovery.tar.files',
'archive_size':
self.root_dir + '/recovery.tar.size',
'partition_size':
self.root_dir + '/recovery.partition.size',
'partition_filesystem':
self.root_dir + '/recovery.tar.filesystem'
}
recovery_archive = NamedTemporaryFile(
delete=False
)
archive = ArchiveTar(
filename=recovery_archive.name,
create_from_file_list=False
)
archive.create(
source_dir=self.root_dir,
exclude=['dev', 'proc', 'sys'],
options=[
'--numeric-owner',
'--hard-dereference',
'--preserve-permissions'
]
)
Command.run(
['mv', recovery_archive.name, metadata['archive_name']]
)
# recovery.tar.filesystem
recovery_filesystem = self.xml_state.build_type.get_filesystem()
with open(metadata['partition_filesystem'], 'w') as partfs:
partfs.write('%s' % recovery_filesystem)
log.info(
'--> Recovery partition filesystem: %s', recovery_filesystem
)
# recovery.tar.files
bash_comand = [
'tar', '-tf', metadata['archive_name'], '|', 'wc', '-l'
]
tar_files_call = Command.run(
['bash', '-c', ' '.join(bash_comand)]
)
tar_files_count = int(tar_files_call.output.rstrip('\n'))
with open(metadata['archive_filecount'], 'w') as files:
files.write('%d\n' % tar_files_count)
log.info(
'--> Recovery file count: %d files', tar_files_count
)
# recovery.tar.size
recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
with open(metadata['archive_size'], 'w') as size:
size.write('%d' % recovery_archive_size_bytes)
log.info(
'--> Recovery uncompressed size: %d mbytes',
int(recovery_archive_size_bytes / 1048576)
)
# recovery.tar.gz
log.info('--> Compressing recovery archive')
compress = Compress(self.root_dir + '/recovery.tar')
compress.gzip()
# recovery.partition.size
recovery_archive_gz_size_mbytes = int(
os.path.getsize(metadata['archive_name'] + '.gz') / 1048576
)
recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
+ Defaults.get_recovery_spare_mbytes()
with open(metadata['partition_size'], 'w') as gzsize:
gzsize.write('%d' % recovery_partition_mbytes)
log.info(
'--> Recovery partition size: %d mbytes',
recovery_partition_mbytes
)
# delete recovery archive if inplace recovery is requested
# In this mode the recovery archive is created at install time
# and not at image creation time. However the recovery metadata
# is preserved in order to be able to check if enough space
# is available on the disk to create the recovery archive.
if self.oemconfig['recovery_inplace']:
log.info(
'--> Inplace recovery requested, deleting archive'
)
Path.wipe(metadata['archive_name'] + '.gz')