def create(self) -> Result:
"""
Builds a container image which is usually a data archive
including container specific metadata.
Image types which triggers this builder are:
* image="docker"
* image="oci"
* image="appx"
:return: result
:rtype: instance of :class:`Result`
"""
if not self.base_image:
log.info('Setting up %s container', self.requested_container_type)
container_setup = ContainerSetup.new(self.requested_container_type,
self.root_dir,
self.container_config)
container_setup.setup()
else:
checksum = Checksum(self.base_image)
if not checksum.matches(checksum.md5(), self.base_image_md5):
raise KiwiContainerBuilderError(
'base image file {0} checksum validation failed'.format(
self.base_image))
log.info('--> Creating container image')
container_image = ContainerImage.new(self.requested_container_type,
self.root_dir,
self.container_config)
self.filename = container_image.create(self.filename, self.base_image)
Result.verify_image_size(self.runtime_config.get_max_size_constraint(),
self.filename)
self.result.add(key='container',
filename=self.filename,
use_for_bundle=True,
compress=False,
shasum=True)
self.result.add(key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(key='image_changes',
filename=self.system_setup.export_package_changes(
self.target_dir),
use_for_bundle=True,
compress=True,
shasum=False)
self.result.add(key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
def _cleanup_intermediate_config(self):
# delete kiwi copied config files
config_files_to_delete = []
for config in self.cleanup_files:
config_file = self.root_dir + config
shasum_file = config_file.replace('.kiwi', '.sha')
config_files_to_delete.append(config_file)
config_files_to_delete.append(shasum_file)
checksum = Checksum(config_file)
if not checksum.matches(checksum.sha256(), shasum_file):
message = dedent('''\n
Modifications to intermediate config file detected
The file: {0}
is a copy from the host system and symlinked
to its origin in the image root during build time.
Modifications to this file by e.g script code will not
have any effect because the file gets restored by one
of the following actions:
1. A package during installation provides it
2. A custom version of the file is setup as overlay file
3. The file is not provided by install or overlay and will
be deleted at the end of the build process
If you need a custom version of that file please
provide it as an overlay file in your image
description
''')
log.warning(message.format(config_file))
del self.cleanup_files[:]
# delete stale symlinks if there are any. normally the package
# installation process should have replaced the symlinks with
# real files from the packages. On deletion check for the
# presence of a config file template and restore it
try:
for config in self.config_files:
config_file = self.root_dir + config
if os.path.islink(config_file):
Command.run(['rm', '-f', config_file])
self._restore_config_template(config_file)
Command.run(['rm', '-f'] + config_files_to_delete)
except Exception as issue:
log.warning(
'Failed to cleanup intermediate config files: {0}'.format(issue)
)
self._restore_intermediate_config_rpmnew_variants()
def create(self):
"""
Builds a container image which is usually a tarball including
container specific metadata.
Image types which triggers this builder are:
* image="docker"
"""
if not self.base_image:
log.info('Setting up %s container', self.requested_container_type)
container_setup = ContainerSetup(self.requested_container_type,
self.root_dir,
self.container_config)
container_setup.setup()
else:
checksum = Checksum(self.base_image)
if not checksum.matches(checksum.md5(), self.base_image_md5):
raise KiwiContainerBuilderError(
'base image file {0} checksum validation failed'.format(
self.base_image))
log.info('--> Creating container image')
container_image = ContainerImage(self.requested_container_type,
self.root_dir, self.container_config)
container_image.create(self.filename, self.base_image)
self.result.add(key='container',
filename=self.filename,
use_for_bundle=True,
compress=False,
shasum=True)
self.result.add(key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
self.result.add(key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir),
use_for_bundle=True,
compress=False,
shasum=False)
return self.result
def create(self):
"""
Builds a container image which is usually a tarball including
container specific metadata.
Image types which triggers this builder are:
* image="docker"
:return: result
:rtype: instance of :class:`Result`
"""
if not self.base_image:
log.info(
'Setting up %s container', self.requested_container_type
)
container_setup = ContainerSetup(
self.requested_container_type, self.root_dir,
self.container_config
)
container_setup.setup()
else:
checksum = Checksum(self.base_image)
if not checksum.matches(checksum.md5(), self.base_image_md5):
raise KiwiContainerBuilderError(
'base image file {0} checksum validation failed'.format(
self.base_image
)
)
log.info(
'--> Creating container image'
)
container_image = ContainerImage(
self.requested_container_type, self.root_dir, self.container_config
)
self.filename = container_image.create(
self.filename, self.base_image
)
self.result.verify_image_size(
self.runtime_config.get_max_size_constraint(),
self.filename
)
self.result.add(
key='container',
filename=self.filename,
use_for_bundle=True,
compress=False,
shasum=True
)
self.result.add(
key='image_packages',
filename=self.system_setup.export_package_list(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
self.result.add(
key='image_verified',
filename=self.system_setup.export_package_verification(
self.target_dir
),
use_for_bundle=True,
compress=False,
shasum=False
)
return self.result
class TestChecksum(object):
@patch('os.path.exists')
def setup(self, mock_exists):
self.context_manager_mock = mock.Mock()
self.file_mock = mock.Mock()
self.enter_mock = mock.Mock()
self.exit_mock = mock.Mock()
self.enter_mock.return_value = self.file_mock
setattr(self.context_manager_mock, '__enter__', self.enter_mock)
setattr(self.context_manager_mock, '__exit__', self.exit_mock)
read_results = [bytes(b''), bytes(b'data')]
def side_effect(arg):
return read_results.pop()
self.file_mock.read.side_effect = side_effect
mock_exists.return_value = True
self.checksum = Checksum('some-file')
@raises(KiwiFileNotFound)
def test_checksum_file_not_found(self):
Checksum('some-file')
@patch('os.path.exists')
def test_matches_checksum_file_does_not_exist(self, mock_exists):
mock_exists.return_value = False
assert self.checksum.matches('sum', 'some-file') is False
@patch('os.path.exists')
@patch_open
def test_matches(self, mock_open, mock_exists):
mock_exists.return_value = True
mock_open.return_value = self.context_manager_mock
self.file_mock.read.side_effect = None
self.file_mock.read.return_value = 'sum'
assert self.checksum.matches('sum', 'some-file') is True
mock_open.assert_called_once_with('some-file')
assert self.checksum.matches('foo', 'some-file') is False
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.md5')
@patch('os.path.getsize')
@patch_open
def test_md5_xz(
self, mock_open, mock_size, mock_md5, mock_compress, mock_which
):
checksum = mock.Mock
checksum.uncompressed_filename = 'some-file-uncompressed'
mock_which.return_value = 'factor'
compress = mock.Mock()
digest = mock.Mock()
digest.block_size = 1024
digest._calculate_hash_hexdigest = mock.Mock(
return_value=checksum
)
digest.hexdigest = mock.Mock(
return_value='sum'
)
compress.get_format = mock.Mock(
return_value='xz'
)
mock_open.return_value = self.context_manager_mock
mock_size.return_value = 1343225856
mock_md5.return_value = digest
mock_compress.return_value = compress
self.checksum.md5('outfile')
assert mock_open.call_args_list == [
call('some-file', 'rb'),
call('some-file-uncompressed', 'rb'),
call('outfile', 'w')
]
self.file_mock.write.assert_called_once_with(
'sum 163968 8192 163968 8192\n'
)
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.md5')
@patch('os.path.getsize')
@patch_open
def test_md5(
self, mock_open, mock_size, mock_md5, mock_compress, mock_which
):
mock_which.return_value = 'factor'
compress = mock.Mock()
digest = mock.Mock()
digest.block_size = 1024
digest.hexdigest = mock.Mock(
return_value='sum'
)
compress.get_format = mock.Mock(
return_value=None
)
mock_open.return_value = self.context_manager_mock
mock_size.return_value = 1343225856
mock_md5.return_value = digest
mock_compress.return_value = compress
self.checksum.md5('outfile')
assert mock_open.call_args_list == [
call('some-file', 'rb'),
call('outfile', 'w')
]
self.file_mock.write.assert_called_once_with(
'sum 163968 8192\n'
)
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.sha256')
@patch('os.path.getsize')
@patch_open
def test_sha256(
self, mock_open, mock_size, mock_sha256, mock_compress, mock_which
):
mock_which.return_value = 'factor'
compress = mock.Mock()
digest = mock.Mock()
digest.block_size = 1024
digest.hexdigest = mock.Mock(
return_value='sum'
)
compress.get_format = mock.Mock(
return_value=None
)
mock_open.return_value = self.context_manager_mock
mock_size.return_value = 1343225856
mock_sha256.return_value = digest
mock_compress.return_value = compress
self.checksum.sha256('outfile')
assert mock_open.call_args_list == [
call('some-file', 'rb'),
call('outfile', 'w')
]
self.file_mock.write.assert_called_once_with(
'sum 163968 8192\n'
)
@patch('hashlib.sha256')
@patch_open
def test_sha256_plain(self, mock_open, mock_sha256):
digest = mock.Mock()
digest.block_size = 1024
digest.hexdigest = mock.Mock(
return_value='sum'
)
mock_sha256.return_value = digest
mock_open.return_value = self.context_manager_mock
assert self.checksum.sha256() == digest.hexdigest.return_value
@patch('hashlib.md5')
@patch_open
def test_md5_plain(self, mock_open, mock_md5):
digest = mock.Mock()
digest.block_size = 1024
digest.hexdigest = mock.Mock(
return_value='sum'
)
mock_md5.return_value = digest
mock_open.return_value = self.context_manager_mock
assert self.checksum.md5() == digest.hexdigest.return_value
class TestChecksum(object):
@patch('os.path.exists')
def setup(self, mock_exists):
self.context_manager_mock = mock.Mock()
self.file_mock = mock.Mock()
self.enter_mock = mock.Mock()
self.exit_mock = mock.Mock()
self.enter_mock.return_value = self.file_mock
setattr(self.context_manager_mock, '__enter__', self.enter_mock)
setattr(self.context_manager_mock, '__exit__', self.exit_mock)
read_results = [bytes(b''), bytes(b'data')]
def side_effect(arg):
return read_results.pop()
self.file_mock.read.side_effect = side_effect
mock_exists.return_value = True
self.checksum = Checksum('some-file')
@raises(KiwiFileNotFound)
def test_checksum_file_not_found(self):
Checksum('some-file')
@patch('os.path.exists')
def test_matches_checksum_file_does_not_exist(self, mock_exists):
mock_exists.return_value = False
assert self.checksum.matches('sum', 'some-file') is False
@patch('os.path.exists')
@patch_open
def test_matches(self, mock_open, mock_exists):
mock_exists.return_value = True
mock_open.return_value = self.context_manager_mock
self.file_mock.read.side_effect = None
self.file_mock.read.return_value = 'sum'
assert self.checksum.matches('sum', 'some-file') is True
mock_open.assert_called_once_with('some-file')
assert self.checksum.matches('foo', 'some-file') is False
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.md5')
@patch('os.path.getsize')
@patch_open
def test_md5_xz(self, mock_open, mock_size, mock_md5, mock_compress,
mock_which):
checksum = mock.Mock
checksum.uncompressed_filename = 'some-file-uncompressed'
mock_which.return_value = 'factor'
compress = mock.Mock()
digest = mock.Mock()
digest.block_size = 1024
digest._calculate_hash_hexdigest = mock.Mock(return_value=checksum)
digest.hexdigest = mock.Mock(return_value='sum')
compress.get_format = mock.Mock(return_value='xz')
mock_open.return_value = self.context_manager_mock
mock_size.return_value = 1343225856
mock_md5.return_value = digest
mock_compress.return_value = compress
self.checksum.md5('outfile')
assert mock_open.call_args_list == [
call('some-file', 'rb'),
call('some-file-uncompressed', 'rb'),
call('outfile', 'w')
]
self.file_mock.write.assert_called_once_with(
'sum 163968 8192 163968 8192\n')
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.md5')
@patch('os.path.getsize')
@patch_open
def test_md5(self, mock_open, mock_size, mock_md5, mock_compress,
mock_which):
mock_which.return_value = 'factor'
compress = mock.Mock()
digest = mock.Mock()
digest.block_size = 1024
digest.hexdigest = mock.Mock(return_value='sum')
compress.get_format = mock.Mock(return_value=None)
mock_open.return_value = self.context_manager_mock
mock_size.return_value = 1343225856
mock_md5.return_value = digest
mock_compress.return_value = compress
self.checksum.md5('outfile')
assert mock_open.call_args_list == [
call('some-file', 'rb'),
call('outfile', 'w')
]
self.file_mock.write.assert_called_once_with('sum 163968 8192\n')
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.sha256')
@patch('os.path.getsize')
@patch_open
def test_sha256(self, mock_open, mock_size, mock_sha256, mock_compress,
mock_which):
mock_which.return_value = 'factor'
compress = mock.Mock()
digest = mock.Mock()
digest.block_size = 1024
digest.hexdigest = mock.Mock(return_value='sum')
compress.get_format = mock.Mock(return_value=None)
mock_open.return_value = self.context_manager_mock
mock_size.return_value = 1343225856
mock_sha256.return_value = digest
mock_compress.return_value = compress
self.checksum.sha256('outfile')
assert mock_open.call_args_list == [
call('some-file', 'rb'),
call('outfile', 'w')
]
self.file_mock.write.assert_called_once_with('sum 163968 8192\n')
@patch('hashlib.sha256')
@patch_open
def test_sha256_plain(self, mock_open, mock_sha256):
digest = mock.Mock()
digest.block_size = 1024
digest.hexdigest = mock.Mock(return_value='sum')
mock_sha256.return_value = digest
mock_open.return_value = self.context_manager_mock
assert self.checksum.sha256() == digest.hexdigest.return_value
@patch('hashlib.md5')
@patch_open
def test_md5_plain(self, mock_open, mock_md5):
digest = mock.Mock()
digest.block_size = 1024
digest.hexdigest = mock.Mock(return_value='sum')
mock_md5.return_value = digest
mock_open.return_value = self.context_manager_mock
assert self.checksum.md5() == digest.hexdigest.return_value
class TestChecksum:
@patch('os.path.exists')
def setup(self, mock_exists):
self.ascii = encoding.getregentry().name
read_results = [bytes(b''), bytes(b'data'), bytes(b''), bytes(b'data')]
def side_effect(arg):
print(read_results[0])
return read_results.pop()
self.m_open = mock_open()
self.m_open.return_value.read.side_effect = side_effect
mock_exists.return_value = True
self.checksum = Checksum('some-file')
def test_checksum_file_not_found(self):
with raises(KiwiFileNotFound):
Checksum('some-file')
@patch('os.path.exists')
def test_matches_checksum_file_does_not_exist(self, mock_exists):
mock_exists.return_value = False
assert self.checksum.matches('sum', 'some-file') is False
@patch('os.path.exists')
def test_matches(self, mock_exists):
mock_exists.return_value = True
self.m_open.return_value.read.side_effect = None
self.m_open.return_value.read.return_value = 'sum'
with patch('builtins.open', self.m_open, create=True):
assert self.checksum.matches('sum', 'some-file') is True
self.m_open.assert_called_once_with(
'some-file', encoding=self.ascii
)
with patch('builtins.open', self.m_open, create=True):
assert self.checksum.matches('foo', 'some-file') is False
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.md5')
@patch('os.path.getsize')
def test_md5_xz(self, mock_size, mock_md5, mock_compress, mock_which):
checksum = Mock
checksum.uncompressed_filename = 'some-file-uncompressed'
mock_which.return_value = 'factor'
compress = Mock()
digest = Mock()
digest.block_size = 1024
digest._calculate_hash_hexdigest = Mock(
return_value=checksum
)
digest.hexdigest = Mock(
return_value='sum'
)
compress.get_format = Mock(
return_value='xz'
)
mock_size.return_value = 1343225856
mock_md5.return_value = digest
mock_compress.return_value = compress
with patch('builtins.open', self.m_open, create=True):
self.checksum.md5('outfile')
assert self.m_open.call_args_list == [
call('some-file', 'rb'),
call('some-file-uncompressed', 'rb'),
call('outfile', encoding=self.ascii, mode='w')
]
self.m_open.return_value.write.assert_called_once_with(
'sum 163968 8192 163968 8192\n'
)
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.md5')
@patch('os.path.getsize')
def test_md5(
self, mock_size, mock_md5, mock_compress, mock_which
):
mock_which.return_value = 'factor'
compress = Mock()
digest = Mock()
digest.block_size = 1024
digest.hexdigest = Mock(
return_value='sum'
)
compress.get_format = Mock(
return_value=None
)
mock_size.return_value = 1343225856
mock_md5.return_value = digest
mock_compress.return_value = compress
with patch('builtins.open', self.m_open, create=True):
self.checksum.md5('outfile')
assert self.m_open.call_args_list == [
call('some-file', 'rb'),
call('outfile', encoding=self.ascii, mode='w')
]
self.m_open.return_value.write.assert_called_once_with(
'sum 163968 8192\n'
)
@patch('kiwi.path.Path.which')
@patch('kiwi.utils.checksum.Compress')
@patch('hashlib.sha256')
@patch('os.path.getsize')
def test_sha256(
self, mock_size, mock_sha256, mock_compress, mock_which
):
mock_which.return_value = 'factor'
compress = Mock()
digest = Mock()
digest.block_size = 1024
digest.hexdigest = Mock(
return_value='sum'
)
compress.get_format = Mock(
return_value=None
)
mock_size.return_value = 1343225856
mock_sha256.return_value = digest
mock_compress.return_value = compress
with patch('builtins.open', self.m_open, create=True):
self.checksum.sha256('outfile')
assert self.m_open.call_args_list == [
call('some-file', 'rb'),
call('outfile', encoding=self.ascii, mode='w')
]
self.m_open.return_value.write.assert_called_once_with(
'sum 163968 8192\n'
)
@patch('hashlib.sha256')
def test_sha256_plain(self, mock_sha256):
digest = Mock()
digest.block_size = 1024
digest.hexdigest = Mock(
return_value='sum'
)
mock_sha256.return_value = digest
with patch('builtins.open', self.m_open, create=True):
assert self.checksum.sha256() == digest.hexdigest.return_value
@patch('hashlib.md5')
def test_md5_plain(self, mock_md5):
digest = Mock()
digest.block_size = 1024
digest.hexdigest = Mock(
return_value='sum'
)
mock_md5.return_value = digest
with patch('builtins.open', self.m_open, create=True):
assert self.checksum.md5() == digest.hexdigest.return_value
def fetch(self, update_check=True):
"""
Download box from the open build service
:param bool update_check: check for box updates True|False
"""
download = update_check
repo_source = self.box_config.get_box_source()
if repo_source:
repo = SolverRepository.new(Uri(repo_source, 'rpm-md'))
packages_file = self.box_config.get_box_packages_file()
packages_shasum_file = \
self.box_config.get_box_packages_shasum_file()
if update_check and packages_file and packages_shasum_file:
local_packages_file = os.sep.join(
[self.box_dir, packages_file])
local_packages_shasum_file = os.sep.join(
[self.box_dir, packages_shasum_file])
local_packages_file_tmp = self.box_stage.register(
local_packages_file)
local_packages_shasum_file_tmp = self.box_stage.register(
local_packages_shasum_file)
repo.download_from_repository(packages_file,
local_packages_file_tmp)
checksum = Checksum(local_packages_file_tmp)
shasum = checksum.sha256()
if checksum.matches(shasum, local_packages_shasum_file):
download = False
else:
self._create_packages_checksum(
local_packages_shasum_file_tmp, shasum)
for box_file in self.box_config.get_box_files():
local_box_file = os.sep.join([self.box_dir, box_file])
if not os.path.exists(local_box_file):
download = True
if download:
log.info('Downloading {0}'.format(box_file))
local_box_file_tmp = self.box_stage.register(
local_box_file)
repo.download_from_repository(box_file, local_box_file_tmp)
if download:
self.box_stage.commit()
for box_file in self.box_config.get_box_files():
local_box_file = os.sep.join([self.box_dir, box_file])
if box_file.endswith('.qcow2'):
self.system = local_box_file
if box_file.endswith('.tar.xz'):
self.kernel = self._extract_kernel_from_tarball(
local_box_file)
if self.box_config.use_initrd():
self.initrd = self._extract_initrd_from_tarball(
local_box_file)
return self.vm_setup_type(
system=self.system,
kernel=self.kernel,
initrd=self.initrd,
append='root={0} console={1} {2}'.format(
self.box_config.get_box_root(),
self.box_config.get_box_console(),
self.box_config.get_box_kernel_cmdline()),
ram=self.box_config.get_box_memory_mbytes(),
smp=self.box_config.get_box_processors())
