コード例 #1
0
 def test_run_apt_get_eatmydata(self, mock_exec):
     bu.run_apt_get('chroot', ['package1', 'package2'],
                    eatmydata=True,
                    attempts=2)
     mock_exec_expected_calls = [
         mock.call('chroot',
                   'chroot',
                   'apt-get',
                   '-y',
                   'update',
                   attempts=2),
         mock.call('chroot',
                   'chroot',
                   'apt-get',
                   '-y',
                   'dist-upgrade',
                   attempts=2),
         mock.call('chroot',
                   'chroot',
                   'eatmydata',
                   'apt-get',
                   '-y',
                   'install',
                   'package1 package2',
                   attempts=2)
     ]
     self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
コード例 #2
0
ファイル: test_build_utils.py プロジェクト: openstack/bareon
 def test_run_apt_get(self, mock_exec):
     bu.run_apt_get('chroot', ['package1', 'package2'], attempts=2)
     mock_exec_expected_calls = [
         mock.call('chroot', 'chroot', 'apt-get', '-y', 'update',
                   attempts=2),
         mock.call('chroot', 'chroot', 'apt-get', '-y', 'dist-upgrade',
                   attempts=2),
         mock.call('chroot', 'chroot', 'apt-get', '-y', 'install',
                   'package1 package2', attempts=2)]
     self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
コード例 #3
0
ファイル: nailgun.py プロジェクト: openstack/bareon
    def do_build_image(self):
        """Building OS images

        Includes the following steps
        1) create temporary sparse files for all images (truncate)
        2) attach temporary files to loop devices (losetup)
        3) create file systems on these loop devices
        4) create temporary chroot directory
        5) install operating system (install_base_os)
        6) configure apt-get sources,and perform package install.
        7) configure OS (clean sources.list and preferences, etc.)
        8) umount loop devices
        9) resize file systems on loop devices
        10) shrink temporary sparse files (images)
        11) containerize (gzip) temporary sparse files
        12) move temporary gzipped files to their final location
        """
        LOG.info('--- Building image (do_build_image) ---')
        driver_os = self.driver.operating_system
        # TODO(kozhukalov): Implement metadata
        # as a pluggable data driver to avoid any fixed format.
        metadata = {}

        metadata['os'] = driver_os.to_dict()

        # TODO(kozhukalov): implement this using image metadata
        # we need to compare list of packages and repos
        LOG.info('*** Checking if image exists ***')
        if all([os.path.exists(img.uri.split('file://', 1)[1])
                for img in self.driver.image_scheme.images]):
            LOG.debug('All necessary images are available. '
                      'Nothing needs to be done.')
            return
        LOG.debug('At least one of the necessary images is unavailable. '
                  'Starting build process.')
        chroot = bu.mkdtemp_smart(
            CONF.image_build_dir, CONF.image_build_suffix)
        try:
            self.install_base_os(chroot)
            packages = driver_os.packages
            metadata['packages'] = packages

            self._set_apt_repos(
                chroot, driver_os.repos,
                proxies=driver_os.proxies.proxies,
                direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
            self._update_metadata_with_repos(
                metadata, driver_os.repos)

            LOG.debug('Installing packages using apt-get: %s',
                      ' '.join(packages))
            bu.run_apt_get(chroot, packages=packages,
                           attempts=CONF.fetch_packages_attempts)

            LOG.debug('Post-install OS configuration')
            root = driver_os.get_user_by_name('root')
            bu.do_post_inst(chroot,
                            hashed_root_password=root.hashed_password,
                            allow_unsigned_file=CONF.allow_unsigned_file,
                            force_ipv4_file=CONF.force_ipv4_file)

            LOG.debug('Making sure there are no running processes '
                      'inside chroot before trying to umount chroot')
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(
                        chroot, signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        'Stopping chrooted processes failed. '
                        'There are some processes running in chroot %s',
                        chroot)

            LOG.info('*** Finalizing image space ***')
            fu.umount_fs(os.path.join(chroot, 'proc'))
            # umounting all loop devices
            self.umount_target(chroot, pseudo=False)

            for image in self.driver.image_scheme.images:
                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(
                    image.target_device)

                if fs.type == 'ext4':
                    LOG.debug('Trying to re-enable journaling for ext4')
                    utils.execute('tune2fs', '-O', 'has_journal',
                                  str(fs.device))

                if image.target_device.name:
                    LOG.debug('Finally: detaching loop device: {0}'.format(
                        image.target_device.name))
                    try:
                        bu.deattach_loop(image.target_device.name)
                    except errors.ProcessExecutionError as e:
                        LOG.warning('Error occured while trying to detach '
                                    'loop device {0}. Error message: {1}'.
                                    format(image.target_device.name, e))

                LOG.debug('Shrinking temporary image file: %s',
                          image.img_tmp_file)
                bu.shrink_sparse_file(image.img_tmp_file)

                raw_size = os.path.getsize(image.img_tmp_file)
                raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)

                LOG.debug('Containerizing temporary image file: %s',
                          image.img_tmp_file)
                img_tmp_containerized = bu.containerize(
                    image.img_tmp_file, image.container,
                    chunk_size=CONF.data_chunk_size)
                img_containerized = image.uri.split('file://', 1)[1]

                # NOTE(kozhukalov): implement abstract publisher
                LOG.debug('Moving image file to the final location: %s',
                          img_containerized)
                shutil.move(img_tmp_containerized, img_containerized)

                container_size = os.path.getsize(img_containerized)
                container_md5 = utils.calculate_md5(
                    img_containerized, container_size)

                metadata.setdefault('images', []).append({
                    'raw_md5': raw_md5,
                    'raw_size': raw_size,
                    'raw_name': None,
                    'container_name': os.path.basename(img_containerized),
                    'container_md5': container_md5,
                    'container_size': container_size,
                    'container': image.container,
                    'format': image.format})

            # NOTE(kozhukalov): implement abstract publisher
            LOG.debug('Image metadata: %s', metadata)
            with open(self.driver.metadata_uri.split('file://', 1)[1],
                      'wt', encoding='utf-8') as f:
                yaml.safe_dump(metadata, stream=f)
            LOG.info('--- Building image END (do_build_image) ---')
        except Exception as exc:
            LOG.error('Failed to build image: %s', exc)
            raise
        finally:
            LOG.info('Cleanup chroot')
            self.destroy_chroot(chroot)
コード例 #4
0
ファイル: nailgun.py プロジェクト: openstack/bareon
    def do_mkbootstrap(self):
        """Building bootstrap image

        Currently supports only Ubuntu-Trusty
        Includes the following steps
        1) Allocate and configure debootstrap.
        2) Install packages
        3) Run user-post script(is defined)
        4) populate squashfs\init\vmlinuz files
        5) create metadata.yaml and pack thats all into tar.gz
        """
        LOG.info('--- Building bootstrap image (do_mkbootstrap) ---')
        driver_os = self.driver.operating_system
        # c_dir = output container directory, where all builded files will
        # be stored, before packaging into archive
        LOG.debug('Creating bootstrap container folder')
        c_dir = bu.mkdtemp_smart(CONF.image_build_dir,
                                 CONF.image_build_suffix + '_container')
        try:
            chroot = bu.mkdtemp_smart(
                CONF.image_build_dir, CONF.image_build_suffix)
            self.install_base_os(chroot)
            bs_scheme = self.driver.bootstrap_scheme
            # init modules, needed for bootstrap. Currently
            #  we support only one scheme initrd + rootfs + kernel
            initrd = filter(lambda x: x.name == 'initrd',
                            bs_scheme.modules)[0]
            rootfs = filter(lambda x: x.name == 'rootfs',
                            bs_scheme.modules)[0]
            metadata = {}
            metadata['os'] = driver_os.to_dict()
            packages = driver_os.packages
            metadata['packages'] = packages

            self._set_apt_repos(
                chroot, driver_os.repos,
                proxies=driver_os.proxies.proxies,
                direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
            self._update_metadata_with_repos(
                metadata, driver_os.repos)
            LOG.debug('Installing packages using apt-get: %s',
                      ' '.join(packages))
            # disable hosts/resolv files
            bu.propagate_host_resolv_conf(chroot)
            if hasattr(bs_scheme, 'certs') and bs_scheme.certs:
                bu.copy_update_certs(bs_scheme.certs, chroot)
            bu.run_apt_get(chroot, packages=packages,
                           attempts=CONF.fetch_packages_attempts)
            LOG.debug('Post-install OS configuration')
            if hasattr(bs_scheme, 'extra_files') and bs_scheme.extra_files:
                for extra in bs_scheme.extra_files:
                    bu.rsync_inject(extra, chroot)
            if (hasattr(bs_scheme, 'root_ssh_authorized_file') and
                    bs_scheme.root_ssh_authorized_file):
                LOG.debug('Put ssh auth file %s',
                          bs_scheme.root_ssh_authorized_file)
                auth_file = os.path.join(chroot, 'root/.ssh/authorized_keys')
                utils.makedirs_if_not_exists(os.path.dirname(
                    auth_file), mode=0o700)
                shutil.copy(
                    bs_scheme.root_ssh_authorized_file,
                    auth_file)
                os.chmod(auth_file, 0o700)
            # Allow user to drop and run script inside chroot:
            if (hasattr(bs_scheme, 'post_script_file') and
                    bs_scheme.post_script_file):
                bu.run_script_in_chroot(
                    chroot, bs_scheme.post_script_file)
            # Save runtime_uuid into bootstrap
            bu.dump_runtime_uuid(bs_scheme.uuid,
                                 os.path.join(chroot,
                                              'etc/nailgun-agent/config.yaml'))
            # NOTE(sslypushenko) Preferred names in LVM config should updated
            # due to point LVM to work only with /dev/mapper folder
            bu.override_lvm_config(
                chroot,
                {'devices': {
                    'preferred_names': CONF.mpath_lvm_preferred_names}},
                lvm_conf_path=CONF.lvm_conf_path)
            root = driver_os.get_user_by_name('root')
            bu.do_post_inst(chroot,
                            hashed_root_password=root.hashed_password,
                            allow_unsigned_file=CONF.allow_unsigned_file,
                            force_ipv4_file=CONF.force_ipv4_file)
            # restore disabled hosts/resolv files
            bu.restore_resolv_conf(chroot)
            metadata['all_packages'] = bu.get_installed_packages(chroot)
            # We need to recompress initramfs with new compression:
            bu.recompress_initramfs(
                chroot,
                compress=initrd.compress_format)
            # Bootstrap nodes load the kernel and initramfs via the network,
            # therefore remove the kernel and initramfs located in root
            # filesystem to make the image smaller (and save the network
            # bandwidth and the boot time)
            bu.copy_kernel_initramfs(chroot, c_dir, clean=True)
            LOG.debug('Making sure there are no running processes '
                      'inside chroot before trying to umount chroot')
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(
                        chroot, signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        'Stopping chrooted processes failed. '
                        'There are some processes running in chroot %s',
                        chroot)
            bu.run_mksquashfs(
                chroot, os.path.join(c_dir, os.path.basename(rootfs.uri)),
                rootfs.compress_format)
            self.dump_mkbootstrap_meta(metadata, c_dir, bs_scheme)
            output = bu.save_bs_container(self.driver.output, c_dir,
                                          bs_scheme.container.format)
            LOG.info('--- Building bootstrap image END (do_mkbootstrap) ---')
            return output
        except Exception as exc:
            LOG.error('Failed to build bootstrap image: %s', exc)
            raise
        finally:
            LOG.info('Cleanup chroot')
            self.destroy_chroot(chroot)
            try:
                shutil.rmtree(c_dir)
            except OSError:
                LOG.debug('Finally: directory %s seems does not exist '
                          'or can not be removed', c_dir)
コード例 #5
0
    def do_build_image(self):
        """Building OS images

        Includes the following steps
        1) create temporary sparse files for all images (truncate)
        2) attach temporary files to loop devices (losetup)
        3) create file systems on these loop devices
        4) create temporary chroot directory
        5) install operating system (install_base_os)
        6) configure apt-get sources,and perform package install.
        7) configure OS (clean sources.list and preferences, etc.)
        8) umount loop devices
        9) resize file systems on loop devices
        10) shrink temporary sparse files (images)
        11) containerize (gzip) temporary sparse files
        12) move temporary gzipped files to their final location
        """
        LOG.info('--- Building image (do_build_image) ---')
        driver_os = self.driver.operating_system
        # TODO(kozhukalov): Implement metadata
        # as a pluggable data driver to avoid any fixed format.
        metadata = {}

        metadata['os'] = driver_os.to_dict()

        # TODO(kozhukalov): implement this using image metadata
        # we need to compare list of packages and repos
        LOG.info('*** Checking if image exists ***')
        if all([
                os.path.exists(img.uri.split('file://', 1)[1])
                for img in self.driver.image_scheme.images
        ]):
            LOG.debug('All necessary images are available. '
                      'Nothing needs to be done.')
            return
        LOG.debug('At least one of the necessary images is unavailable. '
                  'Starting build process.')
        chroot = bu.mkdtemp_smart(CONF.image_build_dir,
                                  CONF.image_build_suffix)
        try:
            self.install_base_os(chroot)
            packages = driver_os.packages
            metadata['packages'] = packages

            self._set_apt_repos(
                chroot,
                driver_os.repos,
                proxies=driver_os.proxies.proxies,
                direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
            self._update_metadata_with_repos(metadata, driver_os.repos)

            LOG.debug('Installing packages using apt-get: %s',
                      ' '.join(packages))
            bu.run_apt_get(chroot,
                           packages=packages,
                           attempts=CONF.fetch_packages_attempts)

            LOG.debug('Post-install OS configuration')
            bu.do_post_inst(chroot,
                            allow_unsigned_file=CONF.allow_unsigned_file,
                            force_ipv4_file=CONF.force_ipv4_file)

            LOG.debug('Making sure there are no running processes '
                      'inside chroot before trying to umount chroot')
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(chroot,
                                                  signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        'Stopping chrooted processes failed. '
                        'There are some processes running in chroot %s',
                        chroot)

            LOG.info('*** Finalizing image space ***')
            fu.umount_fs(os.path.join(chroot, 'proc'))
            # umounting all loop devices
            self.umount_target(chroot, pseudo=False)

            for image in self.driver.image_scheme.images:
                # find fs with the same loop device object
                # as image.target_device
                fs = self.driver.partition_scheme.fs_by_device(
                    image.target_device)

                if fs.type == 'ext4':
                    LOG.debug('Trying to re-enable journaling for ext4')
                    utils.execute('tune2fs', '-O', 'has_journal',
                                  str(fs.device))

                if image.target_device.name:
                    LOG.debug('Finally: detaching loop device: {0}'.format(
                        image.target_device.name))
                    try:
                        bu.deattach_loop(image.target_device.name)
                    except errors.ProcessExecutionError as e:
                        LOG.warning(
                            'Error occured while trying to detach '
                            'loop device {0}. Error message: {1}'.format(
                                image.target_device.name, e))

                LOG.debug('Shrinking temporary image file: %s',
                          image.img_tmp_file)
                bu.shrink_sparse_file(image.img_tmp_file)

                raw_size = os.path.getsize(image.img_tmp_file)
                raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)

                LOG.debug('Containerizing temporary image file: %s',
                          image.img_tmp_file)
                img_tmp_containerized = bu.containerize(
                    image.img_tmp_file,
                    image.container,
                    chunk_size=CONF.data_chunk_size)
                img_containerized = image.uri.split('file://', 1)[1]

                # NOTE(kozhukalov): implement abstract publisher
                LOG.debug('Moving image file to the final location: %s',
                          img_containerized)
                shutil.move(img_tmp_containerized, img_containerized)

                container_size = os.path.getsize(img_containerized)
                container_md5 = utils.calculate_md5(img_containerized,
                                                    container_size)

                metadata.setdefault('images', []).append({
                    'raw_md5':
                    raw_md5,
                    'raw_size':
                    raw_size,
                    'raw_name':
                    None,
                    'container_name':
                    os.path.basename(img_containerized),
                    'container_md5':
                    container_md5,
                    'container_size':
                    container_size,
                    'container':
                    image.container,
                    'format':
                    image.format
                })

            # NOTE(kozhukalov): implement abstract publisher
            LOG.debug('Image metadata: %s', metadata)
            with open(self.driver.metadata_uri.split('file://', 1)[1],
                      'wt',
                      encoding='utf-8') as f:
                yaml.safe_dump(metadata, stream=f)
            LOG.info('--- Building image END (do_build_image) ---')
        except Exception as exc:
            LOG.error('Failed to build image: %s', exc)
            raise
        finally:
            LOG.info('Cleanup chroot')
            self.destroy_chroot(chroot)
コード例 #6
0
    def do_mkbootstrap(self):
        """Building bootstrap image

        Currently supports only Ubuntu-Trusty
        Includes the following steps
        1) Allocate and configure debootstrap.
        2) Install packages
        3) Run user-post script(is defined)
        4) populate squashfs\init\vmlinuz files
        5) create metadata.yaml and pack thats all into tar.gz
        """
        LOG.info('--- Building bootstrap image (do_mkbootstrap) ---')
        driver_os = self.driver.operating_system
        # c_dir = output container directory, where all builded files will
        # be stored, before packaging into archive
        LOG.debug('Creating bootstrap container folder')
        c_dir = bu.mkdtemp_smart(CONF.image_build_dir,
                                 CONF.image_build_suffix + '_container')
        try:
            chroot = bu.mkdtemp_smart(CONF.image_build_dir,
                                      CONF.image_build_suffix)
            self.install_base_os(chroot)
            bs_scheme = self.driver.bootstrap_scheme
            # init modules, needed for bootstrap. Currently
            #  we support only one scheme initrd + rootfs + kernel
            initrd = filter(lambda x: x.name == 'initrd', bs_scheme.modules)[0]
            rootfs = filter(lambda x: x.name == 'rootfs', bs_scheme.modules)[0]
            metadata = {}
            metadata['os'] = driver_os.to_dict()
            packages = driver_os.packages
            metadata['packages'] = packages

            self._set_apt_repos(
                chroot,
                driver_os.repos,
                proxies=driver_os.proxies.proxies,
                direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
            self._update_metadata_with_repos(metadata, driver_os.repos)
            LOG.debug('Installing packages using apt-get: %s',
                      ' '.join(packages))
            # disable hosts/resolv files
            bu.propagate_host_resolv_conf(chroot)
            if hasattr(bs_scheme, 'certs') and bs_scheme.certs:
                bu.copy_update_certs(bs_scheme.certs, chroot)
            bu.run_apt_get(chroot,
                           packages=packages,
                           attempts=CONF.fetch_packages_attempts)
            LOG.debug('Post-install OS configuration')
            if hasattr(bs_scheme, 'extra_files') and bs_scheme.extra_files:
                for extra in bs_scheme.extra_files:
                    bu.rsync_inject(extra, chroot)
            if (hasattr(bs_scheme, 'root_ssh_authorized_file')
                    and bs_scheme.root_ssh_authorized_file):
                LOG.debug('Put ssh auth file %s',
                          bs_scheme.root_ssh_authorized_file)
                auth_file = os.path.join(chroot, 'root/.ssh/authorized_keys')
                utils.makedirs_if_not_exists(os.path.dirname(auth_file),
                                             mode=0o700)
                shutil.copy(bs_scheme.root_ssh_authorized_file, auth_file)
                os.chmod(auth_file, 0o700)
            # Allow user to drop and run script inside chroot:
            if (hasattr(bs_scheme, 'post_script_file')
                    and bs_scheme.post_script_file):
                bu.run_script_in_chroot(chroot, bs_scheme.post_script_file)
            # Save runtime_uuid into bootstrap
            bu.dump_runtime_uuid(
                bs_scheme.uuid,
                os.path.join(chroot, 'etc/nailgun-agent/config.yaml'))
            bu.do_post_inst(chroot,
                            allow_unsigned_file=CONF.allow_unsigned_file,
                            force_ipv4_file=CONF.force_ipv4_file)
            # restore disabled hosts/resolv files
            bu.restore_resolv_conf(chroot)
            metadata['all_packages'] = bu.get_installed_packages(chroot)
            # We need to recompress initramfs with new compression:
            bu.recompress_initramfs(chroot, compress=initrd.compress_format)
            # Bootstrap nodes load the kernel and initramfs via the network,
            # therefore remove the kernel and initramfs located in root
            # filesystem to make the image smaller (and save the network
            # bandwidth and the boot time)
            bu.copy_kernel_initramfs(chroot, c_dir, clean=True)
            LOG.debug('Making sure there are no running processes '
                      'inside chroot before trying to umount chroot')
            if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
                if not bu.stop_chrooted_processes(chroot,
                                                  signal=signal.SIGKILL):
                    raise errors.UnexpectedProcessError(
                        'Stopping chrooted processes failed. '
                        'There are some processes running in chroot %s',
                        chroot)
            bu.run_mksquashfs(
                chroot, os.path.join(c_dir, os.path.basename(rootfs.uri)),
                rootfs.compress_format)
            self.dump_mkbootstrap_meta(metadata, c_dir, bs_scheme)
            output = bu.save_bs_container(self.driver.output, c_dir,
                                          bs_scheme.container.format)
            LOG.info('--- Building bootstrap image END (do_mkbootstrap) ---')
            return output
        except Exception as exc:
            LOG.error('Failed to build bootstrap image: %s', exc)
            raise
        finally:
            LOG.info('Cleanup chroot')
            self.destroy_chroot(chroot)
            try:
                shutil.rmtree(c_dir)
            except OSError:
                LOG.debug(
                    'Finally: directory %s seems does not exist '
                    'or can not be removed', c_dir)