def create(self) -> Result: """ Builds a container image which is usually a data archive including container specific metadata. Image types which triggers this builder are: * image="docker" * image="oci" * image="appx" :return: result :rtype: instance of :class:`Result` """ if not self.base_image: log.info('Setting up %s container', self.requested_container_type) container_setup = ContainerSetup.new(self.requested_container_type, self.root_dir, self.container_config) container_setup.setup() else: checksum = Checksum(self.base_image) if not checksum.matches(checksum.md5(), self.base_image_md5): raise KiwiContainerBuilderError( 'base image file {0} checksum validation failed'.format( self.base_image)) log.info('--> Creating container image') container_image = ContainerImage.new(self.requested_container_type, self.root_dir, self.container_config) self.filename = container_image.create(self.filename, self.base_image) Result.verify_image_size(self.runtime_config.get_max_size_constraint(), self.filename) self.result.add(key='container', filename=self.filename, use_for_bundle=True, compress=False, shasum=True) self.result.add(key='image_packages', filename=self.system_setup.export_package_list( self.target_dir), use_for_bundle=True, compress=False, shasum=False) self.result.add(key='image_changes', filename=self.system_setup.export_package_changes( self.target_dir), use_for_bundle=True, compress=True, shasum=False) self.result.add(key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir), use_for_bundle=True, compress=False, shasum=False) return self.result
def _cleanup_intermediate_config(self): # delete kiwi copied config files config_files_to_delete = [] for config in self.cleanup_files: config_file = self.root_dir + config shasum_file = config_file.replace('.kiwi', '.sha') config_files_to_delete.append(config_file) config_files_to_delete.append(shasum_file) checksum = Checksum(config_file) if not checksum.matches(checksum.sha256(), shasum_file): message = dedent('''\n Modifications to intermediate config file detected The file: {0} is a copy from the host system and symlinked to its origin in the image root during build time. Modifications to this file by e.g script code will not have any effect because the file gets restored by one of the following actions: 1. A package during installation provides it 2. A custom version of the file is setup as overlay file 3. The file is not provided by install or overlay and will be deleted at the end of the build process If you need a custom version of that file please provide it as an overlay file in your image description ''') log.warning(message.format(config_file)) del self.cleanup_files[:] # delete stale symlinks if there are any. normally the package # installation process should have replaced the symlinks with # real files from the packages. On deletion check for the # presence of a config file template and restore it try: for config in self.config_files: config_file = self.root_dir + config if os.path.islink(config_file): Command.run(['rm', '-f', config_file]) self._restore_config_template(config_file) Command.run(['rm', '-f'] + config_files_to_delete) except Exception as issue: log.warning( 'Failed to cleanup intermediate config files: {0}'.format(issue) ) self._restore_intermediate_config_rpmnew_variants()
def create(self): """ Builds a container image which is usually a tarball including container specific metadata. Image types which triggers this builder are: * image="docker" """ if not self.base_image: log.info('Setting up %s container', self.requested_container_type) container_setup = ContainerSetup(self.requested_container_type, self.root_dir, self.container_config) container_setup.setup() else: checksum = Checksum(self.base_image) if not checksum.matches(checksum.md5(), self.base_image_md5): raise KiwiContainerBuilderError( 'base image file {0} checksum validation failed'.format( self.base_image)) log.info('--> Creating container image') container_image = ContainerImage(self.requested_container_type, self.root_dir, self.container_config) container_image.create(self.filename, self.base_image) self.result.add(key='container', filename=self.filename, use_for_bundle=True, compress=False, shasum=True) self.result.add(key='image_packages', filename=self.system_setup.export_package_list( self.target_dir), use_for_bundle=True, compress=False, shasum=False) self.result.add(key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir), use_for_bundle=True, compress=False, shasum=False) return self.result
def create(self): """ Builds a container image which is usually a tarball including container specific metadata. Image types which triggers this builder are: * image="docker" :return: result :rtype: instance of :class:`Result` """ if not self.base_image: log.info( 'Setting up %s container', self.requested_container_type ) container_setup = ContainerSetup( self.requested_container_type, self.root_dir, self.container_config ) container_setup.setup() else: checksum = Checksum(self.base_image) if not checksum.matches(checksum.md5(), self.base_image_md5): raise KiwiContainerBuilderError( 'base image file {0} checksum validation failed'.format( self.base_image ) ) log.info( '--> Creating container image' ) container_image = ContainerImage( self.requested_container_type, self.root_dir, self.container_config ) self.filename = container_image.create( self.filename, self.base_image ) self.result.verify_image_size( self.runtime_config.get_max_size_constraint(), self.filename ) self.result.add( key='container', filename=self.filename, use_for_bundle=True, compress=False, shasum=True ) self.result.add( key='image_packages', filename=self.system_setup.export_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result
class TestChecksum(object): @patch('os.path.exists') def setup(self, mock_exists): self.context_manager_mock = mock.Mock() self.file_mock = mock.Mock() self.enter_mock = mock.Mock() self.exit_mock = mock.Mock() self.enter_mock.return_value = self.file_mock setattr(self.context_manager_mock, '__enter__', self.enter_mock) setattr(self.context_manager_mock, '__exit__', self.exit_mock) read_results = [bytes(b''), bytes(b'data')] def side_effect(arg): return read_results.pop() self.file_mock.read.side_effect = side_effect mock_exists.return_value = True self.checksum = Checksum('some-file') @raises(KiwiFileNotFound) def test_checksum_file_not_found(self): Checksum('some-file') @patch('os.path.exists') def test_matches_checksum_file_does_not_exist(self, mock_exists): mock_exists.return_value = False assert self.checksum.matches('sum', 'some-file') is False @patch('os.path.exists') @patch_open def test_matches(self, mock_open, mock_exists): mock_exists.return_value = True mock_open.return_value = self.context_manager_mock self.file_mock.read.side_effect = None self.file_mock.read.return_value = 'sum' assert self.checksum.matches('sum', 'some-file') is True mock_open.assert_called_once_with('some-file') assert self.checksum.matches('foo', 'some-file') is False @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') @patch_open def test_md5_xz( self, mock_open, mock_size, mock_md5, mock_compress, mock_which ): checksum = mock.Mock checksum.uncompressed_filename = 'some-file-uncompressed' mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest._calculate_hash_hexdigest = mock.Mock( return_value=checksum ) digest.hexdigest = mock.Mock( return_value='sum' ) compress.get_format = mock.Mock( return_value='xz' ) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress self.checksum.md5('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('some-file-uncompressed', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with( 'sum 163968 8192 163968 8192\n' ) @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') @patch_open def test_md5( self, mock_open, mock_size, mock_md5, mock_compress, mock_which ): mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock( return_value='sum' ) compress.get_format = mock.Mock( return_value=None ) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress self.checksum.md5('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with( 'sum 163968 8192\n' ) @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.sha256') @patch('os.path.getsize') @patch_open def test_sha256( self, mock_open, mock_size, mock_sha256, mock_compress, mock_which ): mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock( return_value='sum' ) compress.get_format = mock.Mock( return_value=None ) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_sha256.return_value = digest mock_compress.return_value = compress self.checksum.sha256('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with( 'sum 163968 8192\n' ) @patch('hashlib.sha256') @patch_open def test_sha256_plain(self, mock_open, mock_sha256): digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock( return_value='sum' ) mock_sha256.return_value = digest mock_open.return_value = self.context_manager_mock assert self.checksum.sha256() == digest.hexdigest.return_value @patch('hashlib.md5') @patch_open def test_md5_plain(self, mock_open, mock_md5): digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock( return_value='sum' ) mock_md5.return_value = digest mock_open.return_value = self.context_manager_mock assert self.checksum.md5() == digest.hexdigest.return_value
class TestChecksum(object): @patch('os.path.exists') def setup(self, mock_exists): self.context_manager_mock = mock.Mock() self.file_mock = mock.Mock() self.enter_mock = mock.Mock() self.exit_mock = mock.Mock() self.enter_mock.return_value = self.file_mock setattr(self.context_manager_mock, '__enter__', self.enter_mock) setattr(self.context_manager_mock, '__exit__', self.exit_mock) read_results = [bytes(b''), bytes(b'data')] def side_effect(arg): return read_results.pop() self.file_mock.read.side_effect = side_effect mock_exists.return_value = True self.checksum = Checksum('some-file') @raises(KiwiFileNotFound) def test_checksum_file_not_found(self): Checksum('some-file') @patch('os.path.exists') def test_matches_checksum_file_does_not_exist(self, mock_exists): mock_exists.return_value = False assert self.checksum.matches('sum', 'some-file') is False @patch('os.path.exists') @patch_open def test_matches(self, mock_open, mock_exists): mock_exists.return_value = True mock_open.return_value = self.context_manager_mock self.file_mock.read.side_effect = None self.file_mock.read.return_value = 'sum' assert self.checksum.matches('sum', 'some-file') is True mock_open.assert_called_once_with('some-file') assert self.checksum.matches('foo', 'some-file') is False @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') @patch_open def test_md5_xz(self, mock_open, mock_size, mock_md5, mock_compress, mock_which): checksum = mock.Mock checksum.uncompressed_filename = 'some-file-uncompressed' mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest._calculate_hash_hexdigest = mock.Mock(return_value=checksum) digest.hexdigest = mock.Mock(return_value='sum') compress.get_format = mock.Mock(return_value='xz') mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress self.checksum.md5('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('some-file-uncompressed', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with( 'sum 163968 8192 163968 8192\n') @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') @patch_open def test_md5(self, mock_open, mock_size, mock_md5, mock_compress, mock_which): mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock(return_value='sum') compress.get_format = mock.Mock(return_value=None) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress self.checksum.md5('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with('sum 163968 8192\n') @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.sha256') @patch('os.path.getsize') @patch_open def test_sha256(self, mock_open, mock_size, mock_sha256, mock_compress, mock_which): mock_which.return_value = 'factor' compress = mock.Mock() digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock(return_value='sum') compress.get_format = mock.Mock(return_value=None) mock_open.return_value = self.context_manager_mock mock_size.return_value = 1343225856 mock_sha256.return_value = digest mock_compress.return_value = compress self.checksum.sha256('outfile') assert mock_open.call_args_list == [ call('some-file', 'rb'), call('outfile', 'w') ] self.file_mock.write.assert_called_once_with('sum 163968 8192\n') @patch('hashlib.sha256') @patch_open def test_sha256_plain(self, mock_open, mock_sha256): digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock(return_value='sum') mock_sha256.return_value = digest mock_open.return_value = self.context_manager_mock assert self.checksum.sha256() == digest.hexdigest.return_value @patch('hashlib.md5') @patch_open def test_md5_plain(self, mock_open, mock_md5): digest = mock.Mock() digest.block_size = 1024 digest.hexdigest = mock.Mock(return_value='sum') mock_md5.return_value = digest mock_open.return_value = self.context_manager_mock assert self.checksum.md5() == digest.hexdigest.return_value
class TestChecksum: @patch('os.path.exists') def setup(self, mock_exists): self.ascii = encoding.getregentry().name read_results = [bytes(b''), bytes(b'data'), bytes(b''), bytes(b'data')] def side_effect(arg): print(read_results[0]) return read_results.pop() self.m_open = mock_open() self.m_open.return_value.read.side_effect = side_effect mock_exists.return_value = True self.checksum = Checksum('some-file') def test_checksum_file_not_found(self): with raises(KiwiFileNotFound): Checksum('some-file') @patch('os.path.exists') def test_matches_checksum_file_does_not_exist(self, mock_exists): mock_exists.return_value = False assert self.checksum.matches('sum', 'some-file') is False @patch('os.path.exists') def test_matches(self, mock_exists): mock_exists.return_value = True self.m_open.return_value.read.side_effect = None self.m_open.return_value.read.return_value = 'sum' with patch('builtins.open', self.m_open, create=True): assert self.checksum.matches('sum', 'some-file') is True self.m_open.assert_called_once_with( 'some-file', encoding=self.ascii ) with patch('builtins.open', self.m_open, create=True): assert self.checksum.matches('foo', 'some-file') is False @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') def test_md5_xz(self, mock_size, mock_md5, mock_compress, mock_which): checksum = Mock checksum.uncompressed_filename = 'some-file-uncompressed' mock_which.return_value = 'factor' compress = Mock() digest = Mock() digest.block_size = 1024 digest._calculate_hash_hexdigest = Mock( return_value=checksum ) digest.hexdigest = Mock( return_value='sum' ) compress.get_format = Mock( return_value='xz' ) mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress with patch('builtins.open', self.m_open, create=True): self.checksum.md5('outfile') assert self.m_open.call_args_list == [ call('some-file', 'rb'), call('some-file-uncompressed', 'rb'), call('outfile', encoding=self.ascii, mode='w') ] self.m_open.return_value.write.assert_called_once_with( 'sum 163968 8192 163968 8192\n' ) @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.md5') @patch('os.path.getsize') def test_md5( self, mock_size, mock_md5, mock_compress, mock_which ): mock_which.return_value = 'factor' compress = Mock() digest = Mock() digest.block_size = 1024 digest.hexdigest = Mock( return_value='sum' ) compress.get_format = Mock( return_value=None ) mock_size.return_value = 1343225856 mock_md5.return_value = digest mock_compress.return_value = compress with patch('builtins.open', self.m_open, create=True): self.checksum.md5('outfile') assert self.m_open.call_args_list == [ call('some-file', 'rb'), call('outfile', encoding=self.ascii, mode='w') ] self.m_open.return_value.write.assert_called_once_with( 'sum 163968 8192\n' ) @patch('kiwi.path.Path.which') @patch('kiwi.utils.checksum.Compress') @patch('hashlib.sha256') @patch('os.path.getsize') def test_sha256( self, mock_size, mock_sha256, mock_compress, mock_which ): mock_which.return_value = 'factor' compress = Mock() digest = Mock() digest.block_size = 1024 digest.hexdigest = Mock( return_value='sum' ) compress.get_format = Mock( return_value=None ) mock_size.return_value = 1343225856 mock_sha256.return_value = digest mock_compress.return_value = compress with patch('builtins.open', self.m_open, create=True): self.checksum.sha256('outfile') assert self.m_open.call_args_list == [ call('some-file', 'rb'), call('outfile', encoding=self.ascii, mode='w') ] self.m_open.return_value.write.assert_called_once_with( 'sum 163968 8192\n' ) @patch('hashlib.sha256') def test_sha256_plain(self, mock_sha256): digest = Mock() digest.block_size = 1024 digest.hexdigest = Mock( return_value='sum' ) mock_sha256.return_value = digest with patch('builtins.open', self.m_open, create=True): assert self.checksum.sha256() == digest.hexdigest.return_value @patch('hashlib.md5') def test_md5_plain(self, mock_md5): digest = Mock() digest.block_size = 1024 digest.hexdigest = Mock( return_value='sum' ) mock_md5.return_value = digest with patch('builtins.open', self.m_open, create=True): assert self.checksum.md5() == digest.hexdigest.return_value
def fetch(self, update_check=True): """ Download box from the open build service :param bool update_check: check for box updates True|False """ download = update_check repo_source = self.box_config.get_box_source() if repo_source: repo = SolverRepository.new(Uri(repo_source, 'rpm-md')) packages_file = self.box_config.get_box_packages_file() packages_shasum_file = \ self.box_config.get_box_packages_shasum_file() if update_check and packages_file and packages_shasum_file: local_packages_file = os.sep.join( [self.box_dir, packages_file]) local_packages_shasum_file = os.sep.join( [self.box_dir, packages_shasum_file]) local_packages_file_tmp = self.box_stage.register( local_packages_file) local_packages_shasum_file_tmp = self.box_stage.register( local_packages_shasum_file) repo.download_from_repository(packages_file, local_packages_file_tmp) checksum = Checksum(local_packages_file_tmp) shasum = checksum.sha256() if checksum.matches(shasum, local_packages_shasum_file): download = False else: self._create_packages_checksum( local_packages_shasum_file_tmp, shasum) for box_file in self.box_config.get_box_files(): local_box_file = os.sep.join([self.box_dir, box_file]) if not os.path.exists(local_box_file): download = True if download: log.info('Downloading {0}'.format(box_file)) local_box_file_tmp = self.box_stage.register( local_box_file) repo.download_from_repository(box_file, local_box_file_tmp) if download: self.box_stage.commit() for box_file in self.box_config.get_box_files(): local_box_file = os.sep.join([self.box_dir, box_file]) if box_file.endswith('.qcow2'): self.system = local_box_file if box_file.endswith('.tar.xz'): self.kernel = self._extract_kernel_from_tarball( local_box_file) if self.box_config.use_initrd(): self.initrd = self._extract_initrd_from_tarball( local_box_file) return self.vm_setup_type( system=self.system, kernel=self.kernel, initrd=self.initrd, append='root={0} console={1} {2}'.format( self.box_config.get_box_root(), self.box_config.get_box_console(), self.box_config.get_box_kernel_cmdline()), ram=self.box_config.get_box_memory_mbytes(), smp=self.box_config.get_box_processors())