Exemplo n.º 1
0
 def test_shrink_sparse_file(self, mock_exec, mock_parse):
     mock_parse.return_value = {'block count': 1, 'block size': 2}
     with mock.patch('six.moves.builtins.open', create=True) as mock_open:
         file_handle_mock = mock_open.return_value.__enter__.return_value
         bu.shrink_sparse_file('file')
         mock_open.assert_called_once_with('file', 'rwb+')
         file_handle_mock.truncate.assert_called_once_with(1 * 2)
     expected_mock_exec_calls = [mock.call('e2fsck', '-y', '-f', 'file'),
                                 mock.call('resize2fs', '-M', 'file')]
     mock_parse.assert_called_once_with('dumpe2fs', 'file')
     self.assertEqual(expected_mock_exec_calls, mock_exec.call_args_list)
Exemplo n.º 2
0
 def test_shrink_sparse_file(self, mock_exec, mock_parse):
     mock_parse.return_value = {'block count': 1, 'block size': 2}
     with mock.patch('six.moves.builtins.open', create=True) as mock_open:
         file_handle_mock = mock_open.return_value.__enter__.return_value
         bu.shrink_sparse_file('file')
         mock_open.assert_called_once_with('file', 'rwb+')
         file_handle_mock.truncate.assert_called_once_with(1 * 2)
     expected_mock_exec_calls = [mock.call('e2fsck', '-y', '-f', 'file'),
                                 mock.call('resize2fs', '-M', 'file')]
     mock_parse.assert_called_once_with('dumpe2fs', 'file')
     self.assertEqual(expected_mock_exec_calls, mock_exec.call_args_list)
Exemplo n.º 3
0
    def do_build_image(self):
        """Building OS images

        Includes the following steps
        1) create temporary sparse files for all images (truncate)
        2) attach temporary files to loop devices (losetup)
        3) create file systems on these loop devices
        4) create temporary chroot directory
        5) mount loop devices into chroot directory
        6) install operating system (debootstrap and apt-get)
        7) configure OS (clean sources.list and preferences, etc.)
        8) umount loop devices
        9) resize file systems on loop devices
        10) shrink temporary sparse files (images)
        11) containerize (gzip) temporary sparse files
        12) move temporary gzipped files to their final location
        """
        LOG.info('--- Building image (do_build_image) ---')
        # TODO(kozhukalov): Implement metadata
        # as a pluggable data driver to avoid any fixed format.
        metadata = {}

        # TODO(kozhukalov): implement this using image metadata
        # we need to compare list of packages and repos
        LOG.info('*** Checking if image exists ***')
        if all([
                os.path.exists(img.uri.split('file://', 1)[1])
                for img in self.driver.image_scheme.images
        ]):
            LOG.debug('All necessary images are available. '
                      'Nothing needs to be done.')
            return
        LOG.debug('At least one of the necessary images is unavailable. '
                  'Starting build process.')
        try:
            LOG.debug('Creating temporary chroot directory')
            chroot = tempfile.mkdtemp(dir=CONF.image_build_dir,
                                      suffix=CONF.image_build_suffix)
            LOG.debug('Temporary chroot: %s', chroot)

            proc_path = os.path.join(chroot, 'proc')

            LOG.info('*** Preparing image space ***')
            for image in self.driver.image_scheme.images:
                LOG.debug(
                    'Creating temporary sparsed file for the '
                    'image: %s', image.uri)
                img_tmp_file = bu.create_sparse_tmp_file(
                    dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
                LOG.debug('Temporary file: %s', img_tmp_file)

                # we need to remember those files
                # to be able to shrink them and move in the end
                image.img_tmp_file = img_tmp_file

                LOG.debug('Looking for a free loop device')
                image.target_device.name = bu.get_free_loop_device()

                LOG.debug('Attaching temporary image file to free loop device')
                bu.attach_file_to_loop(img_tmp_file, str(image.target_device))

                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(
                    image.target_device)

                LOG.debug('Creating file system on the image')
                fu.make_fs(fs_type=fs.type,
                           fs_options=fs.options,
                           fs_label=fs.label,
                           dev=str(fs.device))
                if fs.type == 'ext4':
                    LOG.debug('Trying to disable journaling for ext4 '
                              'in order to speed up the build')
                    utils.execute('tune2fs', '-O', '^has_journal',
                                  str(fs.device))

            # mounting all images into chroot tree
            self.mount_target(chroot, treat_mtab=False, pseudo=False)

            LOG.info('*** Shipping image content ***')
            LOG.debug('Installing operating system into image')
            # FIXME(kozhukalov): !!! we need this part to be OS agnostic

            # DEBOOTSTRAP
            # we use first repo as the main mirror
            uri = self.driver.operating_system.repos[0].uri
            suite = self.driver.operating_system.repos[0].suite

            LOG.debug('Preventing services from being get started')
            bu.suppress_services_start(chroot)
            LOG.debug('Installing base operating system using debootstrap')
            bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot)

            # APT-GET
            LOG.debug('Configuring apt inside chroot')
            LOG.debug('Setting environment variables')
            bu.set_apt_get_env()
            LOG.debug('Allowing unauthenticated repos')
            bu.pre_apt_get(chroot)

            for repo in self.driver.operating_system.repos:
                LOG.debug('Adding repository source: name={name}, uri={uri},'
                          'suite={suite}, section={section}'.format(
                              name=repo.name,
                              uri=repo.uri,
                              suite=repo.suite,
                              section=repo.section))
                bu.add_apt_source(name=repo.name,
                                  uri=repo.uri,
                                  suite=repo.suite,
                                  section=repo.section,
                                  chroot=chroot)
                LOG.debug('Adding repository preference: '
                          'name={name}, priority={priority}'.format(
                              name=repo.name, priority=repo.priority))
                if repo.priority is not None:
                    bu.add_apt_preference(name=repo.name,
                                          priority=repo.priority,
                                          suite=repo.suite,
                                          section=repo.section,
                                          chroot=chroot,
                                          uri=repo.uri)

                metadata.setdefault('repos', []).append({
                    'type': 'deb',
                    'name': repo.name,
                    'uri': repo.uri,
                    'suite': repo.suite,
                    'section': repo.section,
                    'priority': repo.priority,
                    'meta': repo.meta
                })

            LOG.debug('Preventing services from being get started')
            bu.suppress_services_start(chroot)

            packages = self.driver.operating_system.packages
            metadata['packages'] = packages

            # we need /proc to be mounted for apt-get success
            utils.makedirs_if_not_exists(proc_path)
            fu.mount_bind(chroot, '/proc')

            LOG.debug('Installing packages using apt-get: %s',
                      ' '.join(packages))
            bu.run_apt_get(chroot, packages=packages)

            LOG.debug('Post-install OS configuration')
            bu.do_post_inst(chroot)

            LOG.debug('Making sure there are no running processes '
                      'inside chroot before trying to umount chroot')
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(chroot,
                                                  signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        'Stopping chrooted processes failed. '
                        'There are some processes running in chroot %s',
                        chroot)

            LOG.info('*** Finalizing image space ***')
            fu.umount_fs(proc_path)
            # umounting all loop devices
            self.umount_target(chroot, pseudo=False, try_lazy_umount=False)

            for image in self.driver.image_scheme.images:
                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(
                    image.target_device)

                if fs.type == 'ext4':
                    LOG.debug('Trying to re-enable journaling for ext4')
                    utils.execute('tune2fs', '-O', 'has_journal',
                                  str(fs.device))

                LOG.debug('Deattaching loop device from file: %s',
                          image.img_tmp_file)
                bu.deattach_loop(str(image.target_device))
                LOG.debug('Shrinking temporary image file: %s',
                          image.img_tmp_file)
                bu.shrink_sparse_file(image.img_tmp_file)

                raw_size = os.path.getsize(image.img_tmp_file)
                raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)

                LOG.debug('Containerizing temporary image file: %s',
                          image.img_tmp_file)
                img_tmp_containerized = bu.containerize(
                    image.img_tmp_file, image.container)
                img_containerized = image.uri.split('file://', 1)[1]

                # NOTE(kozhukalov): implement abstract publisher
                LOG.debug('Moving image file to the final location: %s',
                          img_containerized)
                shutil.move(img_tmp_containerized, img_containerized)

                container_size = os.path.getsize(img_containerized)
                container_md5 = utils.calculate_md5(img_containerized,
                                                    container_size)
                metadata.setdefault('images', []).append({
                    'raw_md5':
                    raw_md5,
                    'raw_size':
                    raw_size,
                    'raw_name':
                    None,
                    'container_name':
                    os.path.basename(img_containerized),
                    'container_md5':
                    container_md5,
                    'container_size':
                    container_size,
                    'container':
                    image.container,
                    'format':
                    image.format
                })

            # NOTE(kozhukalov): implement abstract publisher
            LOG.debug('Image metadata: %s', metadata)
            with open(self.driver.metadata_uri.split('file://', 1)[1],
                      'w') as f:
                yaml.safe_dump(metadata, stream=f)
            LOG.info('--- Building image END (do_build_image) ---')
        except Exception as exc:
            LOG.error('Failed to build image: %s', exc)
            raise
        finally:
            LOG.debug('Finally: stopping processes inside chroot: %s', chroot)

            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
            LOG.debug('Finally: umounting procfs %s', proc_path)
            fu.umount_fs(proc_path)
            LOG.debug('Finally: umounting chroot tree %s', chroot)
            self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
            for image in self.driver.image_scheme.images:
                LOG.debug('Finally: detaching loop device: %s',
                          str(image.target_device))
                try:
                    bu.deattach_loop(str(image.target_device))
                except errors.ProcessExecutionError as e:
                    LOG.warning(
                        'Error occured while trying to detach '
                        'loop device %s. Error message: %s',
                        str(image.target_device), e)

                LOG.debug('Finally: removing temporary file: %s',
                          image.img_tmp_file)
                try:
                    os.unlink(image.img_tmp_file)
                except OSError:
                    LOG.debug(
                        'Finally: file %s seems does not exist '
                        'or can not be removed', image.img_tmp_file)
            LOG.debug('Finally: removing chroot directory: %s', chroot)
            try:
                os.rmdir(chroot)
            except OSError:
                LOG.debug(
                    'Finally: directory %s seems does not exist '
                    'or can not be removed', chroot)
Exemplo n.º 4
0
    def do_build_image(self):
        """Building OS images

        Includes the following steps
        1) create temporary sparse files for all images (truncate)
        2) attach temporary files to loop devices (losetup)
        3) create file systems on these loop devices
        4) create temporary chroot directory
        5) mount loop devices into chroot directory
        6) install operating system (debootstrap and apt-get)
        7) configure OS (clean sources.list and preferences, etc.)
        8) umount loop devices
        9) resize file systems on loop devices
        10) shrink temporary sparse files (images)
        11) containerize (gzip) temporary sparse files
        12) move temporary gzipped files to their final location
        """
        LOG.info('--- Building image (do_build_image) ---')
        # TODO(kozhukalov): Implement metadata
        # as a pluggable data driver to avoid any fixed format.
        metadata = {}

        metadata['os'] = self.driver.operating_system.to_dict()

        # TODO(kozhukalov): implement this using image metadata
        # we need to compare list of packages and repos
        LOG.info('*** Checking if image exists ***')
        if all([os.path.exists(img.uri.split('file://', 1)[1])
                for img in self.driver.image_scheme.images]):
            LOG.debug('All necessary images are available. '
                      'Nothing needs to be done.')
            return
        LOG.debug('At least one of the necessary images is unavailable. '
                  'Starting build process.')
        try:
            LOG.debug('Creating temporary chroot directory')
            utils.makedirs_if_not_exists(CONF.image_build_dir)
            chroot = tempfile.mkdtemp(
                dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
            LOG.debug('Temporary chroot: %s', chroot)

            proc_path = os.path.join(chroot, 'proc')

            LOG.info('*** Preparing image space ***')
            for image in self.driver.image_scheme.images:
                LOG.debug('Creating temporary sparsed file for the '
                          'image: %s', image.uri)
                img_tmp_file = bu.create_sparse_tmp_file(
                    dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
                    size=CONF.sparse_file_size)
                LOG.debug('Temporary file: %s', img_tmp_file)

                # we need to remember those files
                # to be able to shrink them and move in the end
                image.img_tmp_file = img_tmp_file

                LOG.debug('Looking for a free loop device')
                image.target_device.name = bu.get_free_loop_device(
                    loop_device_major_number=CONF.loop_device_major_number,
                    max_loop_devices_count=CONF.max_loop_devices_count)

                LOG.debug('Attaching temporary image file to free loop device')
                bu.attach_file_to_loop(img_tmp_file, str(image.target_device))

                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(
                    image.target_device)

                LOG.debug('Creating file system on the image')
                fu.make_fs(
                    fs_type=fs.type,
                    fs_options=fs.options,
                    fs_label=fs.label,
                    dev=str(fs.device))
                if fs.type == 'ext4':
                    LOG.debug('Trying to disable journaling for ext4 '
                              'in order to speed up the build')
                    utils.execute('tune2fs', '-O', '^has_journal',
                                  str(fs.device))

            # mounting all images into chroot tree
            self.mount_target(chroot, treat_mtab=False, pseudo=False)

            LOG.info('*** Shipping image content ***')
            LOG.debug('Installing operating system into image')
            # FIXME(kozhukalov): !!! we need this part to be OS agnostic

            # DEBOOTSTRAP
            # we use first repo as the main mirror
            uri = self.driver.operating_system.repos[0].uri
            suite = self.driver.operating_system.repos[0].suite

            LOG.debug('Preventing services from being get started')
            bu.suppress_services_start(chroot)
            LOG.debug('Installing base operating system using debootstrap')
            bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
                               attempts=CONF.fetch_packages_attempts)

            # APT-GET
            LOG.debug('Configuring apt inside chroot')
            LOG.debug('Setting environment variables')
            bu.set_apt_get_env()
            LOG.debug('Allowing unauthenticated repos')
            bu.pre_apt_get(chroot,
                           allow_unsigned_file=CONF.allow_unsigned_file,
                           force_ipv4_file=CONF.force_ipv4_file)

            for repo in self.driver.operating_system.repos:
                LOG.debug('Adding repository source: name={name}, uri={uri},'
                          'suite={suite}, section={section}'.format(
                              name=repo.name, uri=repo.uri,
                              suite=repo.suite, section=repo.section))
                bu.add_apt_source(
                    name=repo.name,
                    uri=repo.uri,
                    suite=repo.suite,
                    section=repo.section,
                    chroot=chroot)
                LOG.debug('Adding repository preference: '
                          'name={name}, priority={priority}'.format(
                              name=repo.name, priority=repo.priority))
                if repo.priority is not None:
                    bu.add_apt_preference(
                        name=repo.name,
                        priority=repo.priority,
                        suite=repo.suite,
                        section=repo.section,
                        chroot=chroot,
                        uri=repo.uri)

                metadata.setdefault('repos', []).append({
                    'type': 'deb',
                    'name': repo.name,
                    'uri': repo.uri,
                    'suite': repo.suite,
                    'section': repo.section,
                    'priority': repo.priority,
                    'meta': repo.meta})

            LOG.debug('Preventing services from being get started')
            bu.suppress_services_start(chroot)

            packages = self.driver.operating_system.packages
            metadata['packages'] = packages

            # we need /proc to be mounted for apt-get success
            utils.makedirs_if_not_exists(proc_path)
            fu.mount_bind(chroot, '/proc')

            bu.populate_basic_dev(chroot)

            LOG.debug('Installing packages using apt-get: %s',
                      ' '.join(packages))
            bu.run_apt_get(chroot, packages=packages,
                           attempts=CONF.fetch_packages_attempts)

            LOG.debug('Post-install OS configuration')
            bu.do_post_inst(chroot,
                            allow_unsigned_file=CONF.allow_unsigned_file,
                            force_ipv4_file=CONF.force_ipv4_file)

            LOG.debug('Making sure there are no running processes '
                      'inside chroot before trying to umount chroot')
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(
                        chroot, signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        'Stopping chrooted processes failed. '
                        'There are some processes running in chroot %s',
                        chroot)

            LOG.info('*** Finalizing image space ***')
            fu.umount_fs(proc_path)
            # umounting all loop devices
            self.umount_target(chroot, pseudo=False)

            for image in self.driver.image_scheme.images:
                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(
                    image.target_device)

                if fs.type == 'ext4':
                    LOG.debug('Trying to re-enable journaling for ext4')
                    utils.execute('tune2fs', '-O', 'has_journal',
                                  str(fs.device))

                LOG.debug('Deattaching loop device from file: %s',
                          image.img_tmp_file)
                bu.deattach_loop(str(image.target_device))
                LOG.debug('Shrinking temporary image file: %s',
                          image.img_tmp_file)
                bu.shrink_sparse_file(image.img_tmp_file)

                raw_size = os.path.getsize(image.img_tmp_file)
                raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)

                LOG.debug('Containerizing temporary image file: %s',
                          image.img_tmp_file)
                img_tmp_containerized = bu.containerize(
                    image.img_tmp_file, image.container,
                    chunk_size=CONF.data_chunk_size)
                img_containerized = image.uri.split('file://', 1)[1]

                # NOTE(kozhukalov): implement abstract publisher
                LOG.debug('Moving image file to the final location: %s',
                          img_containerized)
                shutil.move(img_tmp_containerized, img_containerized)

                container_size = os.path.getsize(img_containerized)
                container_md5 = utils.calculate_md5(
                    img_containerized, container_size)
                metadata.setdefault('images', []).append({
                    'raw_md5': raw_md5,
                    'raw_size': raw_size,
                    'raw_name': None,
                    'container_name': os.path.basename(img_containerized),
                    'container_md5': container_md5,
                    'container_size': container_size,
                    'container': image.container,
                    'format': image.format})

            # NOTE(kozhukalov): implement abstract publisher
            LOG.debug('Image metadata: %s', metadata)
            with open(self.driver.metadata_uri.split('file://', 1)[1],
                      'wt', encoding='utf-8') as f:
                yaml.safe_dump(metadata, stream=f)
            LOG.info('--- Building image END (do_build_image) ---')
        except Exception as exc:
            LOG.error('Failed to build image: %s', exc)
            raise
        finally:
            LOG.debug('Finally: stopping processes inside chroot: %s', chroot)

            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
            LOG.debug('Finally: umounting procfs %s', proc_path)
            fu.umount_fs(proc_path)
            LOG.debug('Finally: umounting chroot tree %s', chroot)
            self.umount_target(chroot, pseudo=False)
            for image in self.driver.image_scheme.images:
                LOG.debug('Finally: detaching loop device: %s',
                          str(image.target_device))
                try:
                    bu.deattach_loop(str(image.target_device))
                except errors.ProcessExecutionError as e:
                    LOG.warning('Error occured while trying to detach '
                                'loop device %s. Error message: %s',
                                str(image.target_device), e)

                LOG.debug('Finally: removing temporary file: %s',
                          image.img_tmp_file)
                try:
                    os.unlink(image.img_tmp_file)
                except OSError:
                    LOG.debug('Finally: file %s seems does not exist '
                              'or can not be removed', image.img_tmp_file)
            LOG.debug('Finally: removing chroot directory: %s', chroot)
            try:
                os.rmdir(chroot)
            except OSError:
                LOG.debug('Finally: directory %s seems does not exist '
                          'or can not be removed', chroot)
Exemplo n.º 5
0
    def do_build_image(self):
        """Building OS images

        Includes the following steps
        1) create temporary sparse files for all images (truncate)
        2) attach temporary files to loop devices (losetup)
        3) create file systems on these loop devices
        4) create temporary chroot directory
        5) install operating system (install_base_os)
        6) configure apt-get sources,and perform package install.
        7) configure OS (clean sources.list and preferences, etc.)
        8) umount loop devices
        9) resize file systems on loop devices
        10) shrink temporary sparse files (images)
        11) containerize (gzip) temporary sparse files
        12) move temporary gzipped files to their final location
        """
        LOG.info('--- Building image (do_build_image) ---')
        driver_os = self.driver.operating_system
        # TODO(kozhukalov): Implement metadata
        # as a pluggable data driver to avoid any fixed format.
        metadata = {}

        metadata['os'] = driver_os.to_dict()

        # TODO(kozhukalov): implement this using image metadata
        # we need to compare list of packages and repos
        LOG.info('*** Checking if image exists ***')
        if all([os.path.exists(img.uri.split('file://', 1)[1])
                for img in self.driver.image_scheme.images]):
            LOG.debug('All necessary images are available. '
                      'Nothing needs to be done.')
            return
        LOG.debug('At least one of the necessary images is unavailable. '
                  'Starting build process.')
        try:
            chroot = bu.mkdtemp_smart(
                CONF.image_build_dir, CONF.image_build_suffix)
            self.install_base_os(chroot)
            packages = driver_os.packages
            metadata['packages'] = packages

            self._set_apt_repos(
                chroot, driver_os.repos,
                proxies=driver_os.proxies.proxies,
                direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
            self._update_metadata_with_repos(
                metadata, driver_os.repos)

            LOG.debug('Installing packages using apt-get: %s',
                      ' '.join(packages))
            bu.run_apt_get(chroot, packages=packages,
                           attempts=CONF.fetch_packages_attempts)

            LOG.debug('Post-install OS configuration')
            bu.do_post_inst(chroot,
                            allow_unsigned_file=CONF.allow_unsigned_file,
                            force_ipv4_file=CONF.force_ipv4_file)

            LOG.debug('Making sure there are no running processes '
                      'inside chroot before trying to umount chroot')
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(
                        chroot, signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        'Stopping chrooted processes failed. '
                        'There are some processes running in chroot %s',
                        chroot)

            LOG.info('*** Finalizing image space ***')
            fu.umount_fs(os.path.join(chroot, 'proc'))
            # umounting all loop devices
            self.umount_target(chroot, pseudo=False)

            for image in self.driver.image_scheme.images:
                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(
                    image.target_device)

                if fs.type == 'ext4':
                    LOG.debug('Trying to re-enable journaling for ext4')
                    utils.execute('tune2fs', '-O', 'has_journal',
                                  str(fs.device))

                if image.target_device.name:
                    LOG.debug('Finally: detaching loop device: {0}'.format(
                        image.target_device.name))
                    try:
                        bu.deattach_loop(image.target_device.name)
                    except errors.ProcessExecutionError as e:
                        LOG.warning('Error occured while trying to detach '
                                    'loop device {0}. Error message: {1}'.
                                    format(image.target_device.name, e))

                LOG.debug('Shrinking temporary image file: %s',
                          image.img_tmp_file)
                bu.shrink_sparse_file(image.img_tmp_file)

                raw_size = os.path.getsize(image.img_tmp_file)
                raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)

                LOG.debug('Containerizing temporary image file: %s',
                          image.img_tmp_file)
                img_tmp_containerized = bu.containerize(
                    image.img_tmp_file, image.container,
                    chunk_size=CONF.data_chunk_size)
                img_containerized = image.uri.split('file://', 1)[1]

                # NOTE(kozhukalov): implement abstract publisher
                LOG.debug('Moving image file to the final location: %s',
                          img_containerized)
                shutil.move(img_tmp_containerized, img_containerized)

                container_size = os.path.getsize(img_containerized)
                container_md5 = utils.calculate_md5(
                    img_containerized, container_size)

                metadata.setdefault('images', []).append({
                    'raw_md5': raw_md5,
                    'raw_size': raw_size,
                    'raw_name': None,
                    'container_name': os.path.basename(img_containerized),
                    'container_md5': container_md5,
                    'container_size': container_size,
                    'container': image.container,
                    'format': image.format})

            # NOTE(kozhukalov): implement abstract publisher
            LOG.debug('Image metadata: %s', metadata)
            with open(self.driver.metadata_uri.split('file://', 1)[1],
                      'wt', encoding='utf-8') as f:
                yaml.safe_dump(metadata, stream=f)
            LOG.info('--- Building image END (do_build_image) ---')
        except Exception as exc:
            LOG.error('Failed to build image: %s', exc)
            raise
        finally:
            LOG.info('Cleanup chroot')
            self.destroy_chroot(chroot)
Exemplo n.º 6
0
    def do_build_image(self):
        """Building OS images

        Includes the following steps
        1) create temporary sparse files for all images (truncate)
        2) attach temporary files to loop devices (losetup)
        3) create file systems on these loop devices
        4) create temporary chroot directory
        5) mount loop devices into chroot directory
        6) install operating system (debootstrap and apt-get)
        7) configure OS (clean sources.list and preferences, etc.)
        8) umount loop devices
        9) resize file systems on loop devices
        10) shrink temporary sparse files (images)
        11) containerize (gzip) temporary sparse files
        12) move temporary gzipped files to their final location
        """
        LOG.info("--- Building image (do_build_image) ---")
        # TODO(kozhukalov): Implement metadata
        # as a pluggable data driver to avoid any fixed format.
        metadata = {}

        metadata["os"] = self.driver.operating_system.to_dict()

        # TODO(kozhukalov): implement this using image metadata
        # we need to compare list of packages and repos
        LOG.info("*** Checking if image exists ***")
        if all([os.path.exists(img.uri.split("file://", 1)[1]) for img in self.driver.image_scheme.images]):
            LOG.debug("All necessary images are available. " "Nothing needs to be done.")
            return
        LOG.debug("At least one of the necessary images is unavailable. " "Starting build process.")
        try:
            LOG.debug("Creating temporary chroot directory")
            utils.makedirs_if_not_exists(CONF.image_build_dir)
            chroot = tempfile.mkdtemp(dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
            LOG.debug("Temporary chroot: %s", chroot)

            proc_path = os.path.join(chroot, "proc")

            LOG.info("*** Preparing image space ***")
            for image in self.driver.image_scheme.images:
                LOG.debug("Creating temporary sparsed file for the " "image: %s", image.uri)
                img_tmp_file = bu.create_sparse_tmp_file(
                    dir=CONF.image_build_dir, suffix=CONF.image_build_suffix, size=CONF.sparse_file_size
                )
                LOG.debug("Temporary file: %s", img_tmp_file)

                # we need to remember those files
                # to be able to shrink them and move in the end
                image.img_tmp_file = img_tmp_file

                image.target_device.name = bu.attach_file_to_free_loop_device(
                    img_tmp_file,
                    max_loop_devices_count=CONF.max_loop_devices_count,
                    loop_device_major_number=CONF.loop_device_major_number,
                    max_attempts=CONF.max_allowed_attempts_attach_image,
                )

                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(image.target_device)

                LOG.debug("Creating file system on the image")
                fu.make_fs(fs_type=fs.type, fs_options=fs.options, fs_label=fs.label, dev=str(fs.device))
                if fs.type == "ext4":
                    LOG.debug("Trying to disable journaling for ext4 " "in order to speed up the build")
                    utils.execute("tune2fs", "-O", "^has_journal", str(fs.device))

            # mounting all images into chroot tree
            self.mount_target(chroot, treat_mtab=False, pseudo=False)

            LOG.info("*** Shipping image content ***")
            LOG.debug("Installing operating system into image")
            # FIXME(kozhukalov): !!! we need this part to be OS agnostic

            # DEBOOTSTRAP
            # we use first repo as the main mirror
            uri = self.driver.operating_system.repos[0].uri
            suite = self.driver.operating_system.repos[0].suite

            LOG.debug("Preventing services from being get started")
            bu.suppress_services_start(chroot)
            LOG.debug("Installing base operating system using debootstrap")
            bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot, attempts=CONF.fetch_packages_attempts)

            # APT-GET
            LOG.debug("Configuring apt inside chroot")
            LOG.debug("Setting environment variables")
            bu.set_apt_get_env()
            LOG.debug("Allowing unauthenticated repos")
            bu.pre_apt_get(chroot, allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file)

            for repo in self.driver.operating_system.repos:
                LOG.debug(
                    "Adding repository source: name={name}, uri={uri},"
                    "suite={suite}, section={section}".format(
                        name=repo.name, uri=repo.uri, suite=repo.suite, section=repo.section
                    )
                )
                bu.add_apt_source(name=repo.name, uri=repo.uri, suite=repo.suite, section=repo.section, chroot=chroot)
                LOG.debug(
                    "Adding repository preference: "
                    "name={name}, priority={priority}".format(name=repo.name, priority=repo.priority)
                )
                if repo.priority is not None:
                    bu.add_apt_preference(
                        name=repo.name,
                        priority=repo.priority,
                        suite=repo.suite,
                        section=repo.section,
                        chroot=chroot,
                        uri=repo.uri,
                    )

                metadata.setdefault("repos", []).append(
                    {
                        "type": "deb",
                        "name": repo.name,
                        "uri": repo.uri,
                        "suite": repo.suite,
                        "section": repo.section,
                        "priority": repo.priority,
                        "meta": repo.meta,
                    }
                )

            LOG.debug("Preventing services from being get started")
            bu.suppress_services_start(chroot)

            packages = self.driver.operating_system.packages
            metadata["packages"] = packages

            # we need /proc to be mounted for apt-get success
            utils.makedirs_if_not_exists(proc_path)
            fu.mount_bind(chroot, "/proc")

            bu.populate_basic_dev(chroot)

            LOG.debug("Installing packages using apt-get: %s", " ".join(packages))
            bu.run_apt_get(chroot, packages=packages, attempts=CONF.fetch_packages_attempts)

            LOG.debug("Post-install OS configuration")
            bu.do_post_inst(chroot, allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file)

            LOG.debug("Making sure there are no running processes " "inside chroot before trying to umount chroot")
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        "Stopping chrooted processes failed. " "There are some processes running in chroot %s", chroot
                    )

            LOG.info("*** Finalizing image space ***")
            fu.umount_fs(proc_path)
            # umounting all loop devices
            self.umount_target(chroot, pseudo=False)

            for image in self.driver.image_scheme.images:
                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(image.target_device)

                if fs.type == "ext4":
                    LOG.debug("Trying to re-enable journaling for ext4")
                    utils.execute("tune2fs", "-O", "has_journal", str(fs.device))

                LOG.debug("Deattaching loop device from file: %s", image.img_tmp_file)
                bu.deattach_loop(str(image.target_device))
                LOG.debug("Shrinking temporary image file: %s", image.img_tmp_file)
                bu.shrink_sparse_file(image.img_tmp_file)

                raw_size = os.path.getsize(image.img_tmp_file)
                raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)

                LOG.debug("Containerizing temporary image file: %s", image.img_tmp_file)
                img_tmp_containerized = bu.containerize(
                    image.img_tmp_file, image.container, chunk_size=CONF.data_chunk_size
                )
                img_containerized = image.uri.split("file://", 1)[1]

                # NOTE(kozhukalov): implement abstract publisher
                LOG.debug("Moving image file to the final location: %s", img_containerized)
                shutil.move(img_tmp_containerized, img_containerized)

                container_size = os.path.getsize(img_containerized)
                container_md5 = utils.calculate_md5(img_containerized, container_size)
                metadata.setdefault("images", []).append(
                    {
                        "raw_md5": raw_md5,
                        "raw_size": raw_size,
                        "raw_name": None,
                        "container_name": os.path.basename(img_containerized),
                        "container_md5": container_md5,
                        "container_size": container_size,
                        "container": image.container,
                        "format": image.format,
                    }
                )

            # NOTE(kozhukalov): implement abstract publisher
            LOG.debug("Image metadata: %s", metadata)
            with open(self.driver.metadata_uri.split("file://", 1)[1], "wt", encoding="utf-8") as f:
                yaml.safe_dump(metadata, stream=f)
            LOG.info("--- Building image END (do_build_image) ---")
        except Exception as exc:
            LOG.error("Failed to build image: %s", exc)
            raise
        finally:
            LOG.debug("Finally: stopping processes inside chroot: %s", chroot)

            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
            LOG.debug("Finally: umounting procfs %s", proc_path)
            fu.umount_fs(proc_path)
            LOG.debug("Finally: umounting chroot tree %s", chroot)
            self.umount_target(chroot, pseudo=False)
            for image in self.driver.image_scheme.images:
                if image.target_device.name:
                    LOG.debug("Finally: detaching loop device: %s", image.target_device.name)
                    try:
                        bu.deattach_loop(image.target_device.name)
                    except errors.ProcessExecutionError as e:
                        LOG.warning(
                            "Error occured while trying to detach " "loop device %s. Error message: %s",
                            image.target_device.name,
                            e,
                        )

                if image.img_tmp_file:
                    LOG.debug("Finally: removing temporary file: %s", image.img_tmp_file)
                    try:
                        os.unlink(image.img_tmp_file)
                    except OSError:
                        LOG.debug("Finally: file %s seems does not exist " "or can not be removed", image.img_tmp_file)
            LOG.debug("Finally: removing chroot directory: %s", chroot)
            try:
                os.rmdir(chroot)
            except OSError:
                LOG.debug("Finally: directory %s seems does not exist " "or can not be removed", chroot)
Exemplo n.º 7
0
    def do_build_image(self):
        """Building OS images

        Includes the following steps
        1) create temporary sparse files for all images (truncate)
        2) attach temporary files to loop devices (losetup)
        3) create file systems on these loop devices
        4) create temporary chroot directory
        5) install operating system (install_base_os)
        6) configure apt-get sources,and perform package install.
        7) configure OS (clean sources.list and preferences, etc.)
        8) umount loop devices
        9) resize file systems on loop devices
        10) shrink temporary sparse files (images)
        11) containerize (gzip) temporary sparse files
        12) move temporary gzipped files to their final location
        """
        LOG.info('--- Building image (do_build_image) ---')
        driver_os = self.driver.operating_system
        # TODO(kozhukalov): Implement metadata
        # as a pluggable data driver to avoid any fixed format.
        metadata = {}

        metadata['os'] = driver_os.to_dict()

        # TODO(kozhukalov): implement this using image metadata
        # we need to compare list of packages and repos
        LOG.info('*** Checking if image exists ***')
        if all([os.path.exists(img.uri.split('file://', 1)[1])
                for img in self.driver.image_scheme.images]):
            LOG.debug('All necessary images are available. '
                      'Nothing needs to be done.')
            return
        LOG.debug('At least one of the necessary images is unavailable. '
                  'Starting build process.')
        try:
            chroot = bu.mkdtemp_smart(
                CONF.image_build_dir, CONF.image_build_suffix)
            self.install_base_os(chroot)
            packages = driver_os.packages
            metadata['packages'] = packages

            self._set_apt_repos(
                chroot, driver_os.repos,
                proxies=driver_os.proxies.proxies,
                direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
            self._update_metadata_with_repos(
                metadata, driver_os.repos)

            LOG.debug('Installing packages using apt-get: %s',
                      ' '.join(packages))
            bu.run_apt_get(chroot, packages=packages,
                           attempts=CONF.fetch_packages_attempts)

            LOG.debug('Post-install OS configuration')
            root = driver_os.get_user_by_name('root')
            bu.do_post_inst(chroot,
                            hashed_root_password=root.hashed_password,
                            allow_unsigned_file=CONF.allow_unsigned_file,
                            force_ipv4_file=CONF.force_ipv4_file)

            LOG.debug('Making sure there are no running processes '
                      'inside chroot before trying to umount chroot')
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(
                        chroot, signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        'Stopping chrooted processes failed. '
                        'There are some processes running in chroot %s',
                        chroot)

            LOG.info('*** Finalizing image space ***')
            fu.umount_fs(os.path.join(chroot, 'proc'))
            # umounting all loop devices
            self.umount_target(chroot, pseudo=False)

            for image in self.driver.image_scheme.images:
                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(
                    image.target_device)

                if fs.type == 'ext4':
                    LOG.debug('Trying to re-enable journaling for ext4')
                    utils.execute('tune2fs', '-O', 'has_journal',
                                  str(fs.device))

                if image.target_device.name:
                    LOG.debug('Finally: detaching loop device: {0}'.format(
                        image.target_device.name))
                    try:
                        bu.deattach_loop(image.target_device.name)
                    except errors.ProcessExecutionError as e:
                        LOG.warning('Error occured while trying to detach '
                                    'loop device {0}. Error message: {1}'.
                                    format(image.target_device.name, e))

                LOG.debug('Shrinking temporary image file: %s',
                          image.img_tmp_file)
                bu.shrink_sparse_file(image.img_tmp_file)

                raw_size = os.path.getsize(image.img_tmp_file)
                raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)

                LOG.debug('Containerizing temporary image file: %s',
                          image.img_tmp_file)
                img_tmp_containerized = bu.containerize(
                    image.img_tmp_file, image.container,
                    chunk_size=CONF.data_chunk_size)
                img_containerized = image.uri.split('file://', 1)[1]

                # NOTE(kozhukalov): implement abstract publisher
                LOG.debug('Moving image file to the final location: %s',
                          img_containerized)
                shutil.move(img_tmp_containerized, img_containerized)

                container_size = os.path.getsize(img_containerized)
                container_md5 = utils.calculate_md5(
                    img_containerized, container_size)

                metadata.setdefault('images', []).append({
                    'raw_md5': raw_md5,
                    'raw_size': raw_size,
                    'raw_name': None,
                    'container_name': os.path.basename(img_containerized),
                    'container_md5': container_md5,
                    'container_size': container_size,
                    'container': image.container,
                    'format': image.format})

            # NOTE(kozhukalov): implement abstract publisher
            LOG.debug('Image metadata: %s', metadata)
            with open(self.driver.metadata_uri.split('file://', 1)[1],
                      'wt', encoding='utf-8') as f:
                yaml.safe_dump(metadata, stream=f)
            LOG.info('--- Building image END (do_build_image) ---')
        except Exception as exc:
            LOG.error('Failed to build image: %s', exc)
            raise
        finally:
            LOG.info('Cleanup chroot')
            self.destroy_chroot(chroot)