Ejemplo n.º 1
0
    def bootstrap_os(self):
        """
        Bootstrap base OS image.
        """
        log.info("Bootstrapping new OS image")

        # directory is encoded into string to prevent unicode errors
        directory = tempfile.mkdtemp(dir=self.config.paths.workdir,
                                     prefix="upaas_bootstrap_")
        log.debug("Created temporary directory for bootstrap at "
                  "'%s'" % directory)

        for cmd in self.config.bootstrap.commands:
            cmd = cmd.replace("%workdir%", directory)
            try:
                commands.execute(cmd, timeout=self.config.bootstrap.timelimit,
                                 cwd=directory, env=self.config.bootstrap.env,
                                 strip_envs=True)
            except commands.CommandTimeout as e:
                log.error("Bootstrap was taking too long and it was killed")
                kill_and_remove_dir(directory)
                raise exceptions.OSBootstrapError(e)
            except commands.CommandFailed as e:
                log.error("Bootstrap command failed")
                kill_and_remove_dir(directory)
                raise exceptions.OSBootstrapError(e)
        log.info("All commands completed, installing packages")

        self.install_packages(directory, self.config.bootstrap.packages)
        log.info("Bootstrap done, packing image")

        archive_path = os.path.join(directory, "image.tar.gz")
        if not tar.pack_tar(directory, archive_path,
                            timeout=self.config.bootstrap.timelimit):
            kill_and_remove_dir(directory)
            raise exceptions.OSBootstrapError("Tar error")
        else:
            log.info("Image packed, uploading")

        try:
            self.storage.put(archive_path, distro.distro_image_filename())
        except Exception as e:
            log.error("Upload failed: %s" % e)
            raise

        log.info("Image uploaded")
        kill_and_remove_dir(directory)
        log.info("All done")
Ejemplo n.º 2
0
    def build_package(self, system_filename=None, interpreter_version=None,
                      current_revision=None, env=None):
        """
        Build a package

        :param system_filename: Use given file as base system, if None empty
                                system image will be used.
        :param interpreter_version: Use specific interpreter version, only used
                                    for fresh packages.
        :param current_revision: VCS revision id from current package.
        """
        if interpreter_version:
            self.interpreter_version = interpreter_version
            log.info("Using forced interpreter version: "
                     "%s" % interpreter_version)

        self.storage = load_handler(self.config.storage.handler,
                                    self.config.storage.settings)
        if system_filename and self.storage.exists(system_filename):
            log.info("Starting package build using package "
                     "%s" % system_filename)
            if current_revision:
                log.info("VCS revision from current package: "
                         "%s" % current_revision)
                self.current_revision = current_revision
        else:
            if system_filename:
                log.warning("Requested base package file not found, using "
                            "empty system image")
            self.envs['UPAAS_FRESH_PACKAGE'] = 'true'
            system_filename = None
            log.info("Starting package build using empty system image")
            if not self.has_valid_os_image():
                try:
                    self.bootstrap_os()
                except exceptions.OSBootstrapError as e:
                    self.system_error("Error during os bootstrap: %s" % e)
                except StorageError as e:
                    self.system_error("Error during uploading OS image: "
                                      "%s" % e)

        if not self.interpreter_version:
            self.user_error("Unsupported interpreter version")

        if env:
            log.info("Additional ENV: %s" % ', '.join(env.keys()))
            self.envs.update(env)

        self.actions.update(self.parse_actions(self.metadata))
        self.envs.update(self.parse_envs(self.metadata))
        self.os_packages += self.parse_packages(self.metadata)

        result = BuildResult()
        result.parent = system_filename
        result.interpreter_version = self.interpreter_version

        # directory is encoded into string to prevent unicode errors
        directory = tempfile.mkdtemp(dir=self.config.paths.workdir,
                                     prefix="upaas_package_")
        workdir = os.path.join(directory, "workdir")
        chroot_homedir = self.config.apps.home
        os.mkdir(workdir, 0o755)
        log.info("Working directory created at '%s'" % workdir)
        self.envs['HOME'] = chroot_homedir

        if not self.unpack_os(directory, workdir,
                              system_filename=system_filename):
            kill_and_remove_dir(directory)
            self.system_error("Unpacking OS image failed")
        log.info("OS image unpacked")
        result.progress = 10
        yield result

        log.info("Using interpreter %s, version %s" % (
            self.metadata.interpreter.type, self.interpreter_version))

        if not self.run_actions(self.builder_action_names, workdir):
            kill_and_remove_dir(directory)
            self.system_error("System actions failed")
        log.info("All builder actions executed")
        result.progress = 20
        yield result

        if not self.install_packages(workdir, self.os_packages):
            kill_and_remove_dir(directory)
            self.user_error("Failed to install OS packages")
        log.info("All packages installed")
        result.progress = 35
        yield result

        if not self.run_actions(self.interpreter_action_names, workdir, '/'):
            kill_and_remove_dir(directory)
            self.system_error("Interpreter actions failed")
        log.info("All interpreter actions executed")
        result.progress = 40
        yield result

        # TODO if building fails up to this point, then we can try retry it
        # on another builder (for a limited number of times)

        if system_filename:
            if not self.update(workdir, chroot_homedir):
                kill_and_remove_dir(directory)
                self.user_error("Updating repository failed")
        else:
            if not self.clone(workdir, chroot_homedir):
                kill_and_remove_dir(directory)
                self.user_error("Cloning repository failed")
        log.info("Application repository ready")
        result.progress = 45
        yield result

        result.vcs_revision = self.vcs_info(workdir, chroot_homedir)
        result.progress = 46
        yield result

        if not self.write_files(workdir, chroot_homedir):
            kill_and_remove_dir(directory)
            self.user_error("Creating files from metadata failed")
        log.info("Created all files from metadata")
        result.progress = 49
        yield result

        if not self.run_actions(self.app_action_names, workdir,
                                chroot_homedir):
            self.user_error("Application actions failed")
            kill_and_remove_dir(directory)
        log.info("All application actions executed")
        result.progress = 85
        yield result

        if not self.run_actions(self.finalize_action_names, workdir, '/'):
            kill_and_remove_dir(directory)
            self.system_error("Finalize actions failed")
        log.info("All final actions executed")
        result.progress = 88
        yield result

        if not self.chown_app_dir(workdir, chroot_homedir):
            kill_and_remove_dir(directory)
            self.system_error("Setting file ownership failed")
        log.info("Owner of application directory updated")
        result.progress = 89
        yield result

        if not self.umount_filesystems(workdir):
            kill_and_remove_dir(directory)
            self.system_error("Failed to unmount filesystems")
        result.progress = 90
        yield result

        package_path = os.path.join(directory, "package")
        if not tar.pack_tar(workdir, package_path):
            kill_and_remove_dir(directory)
            self.system_error("Creating package file failed")
        result.bytes = os.path.getsize(package_path)
        log.info("Application package created, "
                 "%s" % utils.bytes_to_human(result.bytes))
        result.progress = 93
        yield result

        checksum = calculate_file_sha256(package_path)
        log.info("Package checksum: %s" % checksum)
        result.progress = 96
        yield result

        try:
            self.storage.put(package_path, checksum)
        except StorageError as e:
            kill_and_remove_dir(directory)
            self.system_error("Package upload failed: %s" % e)

        kill_and_remove_dir(directory)

        result.progress = 100
        result.filename = checksum
        result.checksum = checksum
        yield result