Exemple #1
0
    def clear_caches(cls, emerge_config):
        """
        Clear Portage and Binary PMS caches.
        """
        emerge_settings, emerge_trees, _mtimedb = emerge_config
        clear_caches(emerge_trees)
        # clearing vartree.dbapi.cpcache doesn't seem to make a big diff
        root_tree = emerge_trees[emerge_settings["ROOT"]]
        vdb = root_tree["vartree"].dbapi
        for method_name in ("_clear_cache", ):
            method = getattr(vdb, method_name, None)
            if method is not None:
                method()
            else:
                print_error("vartree does not have a %s method anymore" %
                            (method_name, ))

        root_tree["porttree"].dbapi.close_caches()
        root_tree["porttree"].dbapi = portage.dbapi.porttree.portdbapi(
            root_tree["porttree"].settings)

        for x in range(10):
            count = gc.collect()
            if not count:
                break
Exemple #2
0
    def clear_caches(cls, emerge_config):
        """
        Clear Portage and Binary PMS caches.
        """
        emerge_settings, emerge_trees, _mtimedb = emerge_config
        clear_caches(emerge_trees)
        # clearing vartree.dbapi.cpcache doesn't seem to make a big diff
        root_tree = emerge_trees[emerge_settings["ROOT"]]
        vdb = root_tree["vartree"].dbapi
        for method_name in ("_clear_cache",):
            method = getattr(vdb, method_name, None)
            if method is not None:
                method()
            else:
                print_error(
                    "vartree does not have a %s method anymore" % (
                        method_name,))

        root_tree["porttree"].dbapi.close_caches()
        root_tree["porttree"].dbapi = portage.dbapi.porttree.portdbapi(
            root_tree["porttree"].settings)

        for x in range(10):
            count = gc.collect()
            if not count:
                break
Exemple #3
0
    def _run_builder(self, dirs_cleanup_queue):
        """
        This method is called by _run and executes the whole package build
        logic, including constraints validation given by argv parameters.
        NOTE: negative errors indicate warnings that can be skipped.
        """
        if self._packages:
            first_package = self._packages[0]
        else:
            first_package = "_empty_"

        log_dir = mkdtemp(prefix="matter_build.",
            suffix="." + first_package.replace("/", "_").lstrip("<>=~"))
        dirs_cleanup_queue.append(log_dir)

        emerge_settings, emerge_trees, mtimedb = self._emerge_config

        # reset settings to original state, variables will be reconfigured
        # while others may remain saved due to backup_changes().
        emerge_settings.unlock()
        emerge_settings.reset()
        emerge_settings.lock()

        # Setup stable/unstable keywords, must be done on
        # emerge_settings bacause the reference is spread everywhere
        # in emerge_trees.
        # This is not thread-safe, but Portage isn't either, so
        # who cares!
        # ACCEPT_KEYWORDS is not saved and reset every time by the
        # reset() call above.
        portdb = emerge_trees[emerge_settings["ROOT"]]["porttree"].dbapi

        self._setup_keywords(portdb, emerge_settings)

        portdb.freeze()
        vardb = emerge_trees[emerge_settings["ROOT"]]["vartree"].dbapi
        vardb.settings.unlock()
        vardb.settings["PORT_LOGDIR"] = log_dir
        vardb.settings.backup_changes("PORT_LOGDIR")
        vardb.settings.lock()

        # Load the most current variables from /etc/profile.env, which
        # has been re-generated by the env-update call in _run()
        emerge_settings.unlock()
        emerge_settings.reload()
        emerge_settings.regenerate()
        emerge_settings.lock()

        sets = self._get_sets_mod()  # can be None
        sets_conf = None
        if sets is not None:
            sets_conf = sets.load_default_config(
                emerge_settings,
                emerge_trees[emerge_settings["ROOT"]])

        packages = []
        # execute basic, pre-graph generation filters against each
        # package dependency in self._packages.
        # This is just fast pruning of obvious obviousness.
        for package in self._packages:
            expanded_pkgs = []

            # package sets support
            if package.startswith("@") and sets_conf:
                try:
                    set_pkgs = sets_conf.getSetAtoms(package[1:])
                    expanded_pkgs.extend(sorted(set_pkgs))
                except sets.PackageSetNotFound:
                    # make it fail, add set directly
                    expanded_pkgs.append(package)
            else:
                expanded_pkgs.append(package)

            for exp_pkg in expanded_pkgs:
                accepted = self._pre_graph_filters(
                    exp_pkg, portdb, vardb)
                for best_visible in accepted:
                    packages.append((exp_pkg, best_visible))

        if not packages:
            print_warning("No remaining packages in queue, aborting.")
            return 0

        # at this point we can go ahead building packages
        print_info("starting to build:")
        for package, best_visible in packages:
            print_info(": %s -> %s" % (
                    package, best_visible,))

        if not getcolor():
            portage.output.nocolor()

        # non interactive properties, this is not really required
        # accept-properties just sets os.environ...
        build_args = list(self._setup_build_args(self._params))
        build_args += ["=" + best_v for _x, best_v in packages]

        myaction, myopts, myfiles = parse_opts(build_args)
        adjust_configs(myopts, emerge_trees)
        apply_priorities(emerge_settings)

        spinner = stdout_spinner()
        if "--quiet" in myopts:
            spinner.update = spinner.update_basic
        elif "--nospinner" in myopts:
            spinner.update = spinner.update_basic
        if emerge_settings.get("TERM") == "dumb" or not is_stdout_a_tty():
            spinner.update = spinner.update_basic

        print_info("emerge args: %s" % (" ".join(build_args),))

        params = create_depgraph_params(myopts, myaction)
        success, graph, favorites = backtrack_depgraph(emerge_settings,
            emerge_trees, myopts, params, myaction, myfiles, spinner)

        if not success:
            # print issues to stdout and give up
            print_warning("dependencies calculation failed, aborting")
            graph.display_problems()

            # try to collect some info about the failure
            bt_config = (graph.get_backtrack_infos() or {}).get("config", {})
            for k, v in bt_config.items():
                if k == "needed_use_config_changes":
                    for tup in v:
                        try:
                            pkg, (new_use, new_changes) = tup
                        except (ValueError, TypeError):
                            print_error(
                                "unsupported needed_use_config_changes: %s" % (
                                    tup,))
                            continue
                        obj = self._missing_use_packages.setdefault(
                            "%s" % (pkg.cpv,), {})
                        obj["cp:slot"] = "%s" % (pkg.slot_atom,)
                        changes = obj.setdefault("changes", {})
                        changes.update(copy.deepcopy(new_changes))
                elif k == "needed_unstable_keywords":
                    for pkg in v:
                        self._needed_unstable_keywords.add("%s" % (pkg.cpv,))
                elif k == "needed_p_mask_changes":
                    for pkg in v:
                        self._needed_package_mask_changes.add(
                            "%s" % (pkg.cpv,))
                elif k == "needed_license_changes":
                    for pkg, lics in v:
                        obj = self._needed_license_changes.setdefault(
                            "%s" % (pkg.cpv,), set())
                        obj.update(lics)
                else:
                    print_warning("unsupported backtrack info: %s -> %s" % (
                            k, v,))

            return 0
        print_info("dependency graph generated successfully")

        real_queue = self._post_graph_filters(graph, vardb, portdb)
        if real_queue is None:
            # post-graph filters not passed, giving up
            return 0

        merge_queue = [x for x in real_queue if x.operation == "merge"]
        unmerge_queue = [x for x in real_queue if x.operation == "uninstall"]
        if merge_queue:
            print_info("about to build the following packages:")
            for pkg in merge_queue:
                print_info("  %s" % (pkg.cpv,))
        if unmerge_queue:
            print_info("about to uninstall the following packages:")
            for pkg in unmerge_queue:
                print_info("  %s" % (pkg.cpv,))

        if self._pretend:
            print_info("portage spawned with --pretend, done!")
            return 0

        # re-calling action_build(), deps are re-calculated though
        validate_ebuild_environment(emerge_trees)
        mergetask = Scheduler(emerge_settings, emerge_trees, mtimedb,
            myopts, spinner, favorites=favorites,
            graph_config=graph.schedulerGraph())
        del graph
        self.clear_caches(self._emerge_config)
        retval = mergetask.merge()

        not_merged = []
        real_queue_map = dict((pkg.cpv, pkg) for pkg in real_queue)
        failed_package = None
        if retval != 0:
            merge_list = mtimedb.get("resume", {}).get("mergelist")
            for _merge_type, _merge_root, merge_atom, _merge_act in merge_list:
                merge_atom = "%s" % (merge_atom,)
                if failed_package is None:
                    # we consider the first encountered package the one
                    # that failed. It makes sense since packages are built
                    # serially as of today.
                    # Also, the package object must be available in our
                    # package queue, so grab it from there.
                    failed_package = real_queue_map.get(merge_atom)
                not_merged.append(merge_atom)
                self._not_merged_packages.append(merge_atom)

        for pkg in real_queue:
            cpv = pkg.cpv
            if not cpv:
                print_warning("package: %s, has broken cpv: '%s', ignoring" % (
                        pkg, cpv,))
            elif cpv not in not_merged:
                if pkg.operation == "merge":
                    # add to build queue
                    print_info("package: %s, successfully built" % (cpv,))
                    self._built_packages.append("%s" % (cpv,))
                else:
                    # add to uninstall queue
                    print_info("package: %s, successfully uninstalled" % (cpv,))
                    self._uninstalled_packages.append("%s" % (cpv,))

        post_emerge(myaction, myopts, myfiles, emerge_settings["ROOT"],
            emerge_trees, mtimedb, retval)

        subprocess.call(["env-update"])

        if failed_package is not None:
            print_warning("failed package: %s::%s" % (failed_package.cpv,
                failed_package.repo,))

        if self._params["buildfail"] and (failed_package is not None):

            std_env = self._build_standard_environment(
                repository=self._params["repository"])
            std_env["MATTER_PACKAGE_NAMES"] = " ".join(self._packages)
            std_env["MATTER_PORTAGE_FAILED_PACKAGE_NAME"] = failed_package.cpv
            std_env["MATTER_PORTAGE_REPOSITORY"] = failed_package.repo
            # call pkgfail hook if defined
            std_env["MATTER_PORTAGE_BUILD_LOG_DIR"] = os.path.join(log_dir,
                "build")

            buildfail = self._params["buildfail"]
            print_info("spawning buildfail: %s" % (buildfail,))
            tmp_fd, tmp_path = mkstemp()
            with os.fdopen(tmp_fd, "wb") as tmp_f:
                with open(buildfail, "rb") as buildfail_f:
                    tmp_f.write(buildfail_f.read())
            try:
                # now execute
                os.chmod(tmp_path, 0o700)
                exit_st = subprocess.call([tmp_path], env = std_env)
                if exit_st != 0:
                    return exit_st
            finally:
                os.remove(tmp_path)

        print_info("portage spawned, return value: %d" % (retval,))
        return retval
Exemple #4
0
    def _pre_graph_filters(self, package, portdb, vardb):
        """
        Execute basic, pre-graph generation (dependencies calculation)
        filters against the package dependency to see if it's eligible
        for the graph.
        """
        allow_rebuild = self._params["rebuild"] == "yes"
        allow_not_installed = self._params["not-installed"] == "yes"
        allow_downgrade = self._params["downgrade"] == "yes"
        accepted = []

        # now determine what's the installed version.
        best_installed = portage.best(vardb.match(package, use_cache=0))
        if (not best_installed) and (not allow_not_installed):
            # package not installed
            print_error("package not installed: %s, ignoring this one" % (
                    package,))
            self._not_installed_packages.append(package)
            return accepted

        if (not best_installed) and allow_not_installed:
            print_warning(
                "%s not installed, but 'not-installed: yes' provided" % (
                    package,))

        best_visibles = []
        try:
            best_visibles += portdb.xmatch("match-visible", package)
        except portage.exception.InvalidAtom:
            print_error("cannot match: %s, invalid atom" % (package,))

        # map all the cpvs to their slots
        cpv_slot_map = {}
        for pkg in best_visibles:
            obj = cpv_slot_map.setdefault(pkg.slot, [])
            obj.append(pkg)

        # then pick the best for each slot
        del best_visibles[:]
        for slot, pkgs in cpv_slot_map.items():
            pkg = portage.best(pkgs)
            best_visibles.append(pkg)
        best_visibles.sort()  # deterministic is better

        if not best_visibles:
            # package not found, return error
            print_error("cannot match: %s, ignoring this one" % (package,))
            self._not_found_packages.append(package)
            return accepted

        print_info("matched: %s for %s" % (", ".join(best_visibles), package,))

        for best_visible in best_visibles:

            cp = best_visible.cp
            slot = best_visible.slot
            cp_slot = "%s:%s" % (cp, slot)

            # determine what's the installed version.
            # we know that among all the best_visibles, there is one that
            # is installed. The question is whether we got it now.
            best_installed = portage.best(vardb.match(cp_slot, use_cache=0))
            if (not best_installed) and (not allow_not_installed):
                # package not installed
                print_warning("%s not installed, skipping" % (cp_slot,))
                continue

            build_only = self._params["build-only"] == "yes"
            cmp_res = -1
            if best_installed:
                print_info("found installed: %s for %s" % (
                        best_installed, package,))
                # now compare
                # -1 if best_installed is older than best_visible
                # 1 if best_installed is newer than best_visible
                # 0 if they are equal
                cmp_res = portage.versions.pkgcmp(
                    portage.versions.pkgsplit(best_installed),
                    portage.versions.pkgsplit(best_visible))
            elif (not best_installed) and build_only:
                # package is not installed, and build-only
                # is provided. We assume that the package
                # is being built and added to repositories directly.
                # This means that we need to query binpms to know
                # about the current version.
                print_info("package is not installed, and 'build-only: yes'. "
                           "Asking the binpms about the package state.")
                best_available = self._binpms.best_available(cp_slot)
                print_info("found available: %s for %s" % (
                        best_available, cp_slot))
                if best_available:
                    cmp_res = portage.versions.pkgcmp(
                        portage.versions.pkgsplit(best_available),
                        portage.versions.pkgsplit(best_visible))

            is_rebuild = cmp_res == 0

            if (cmp_res == 1) and (not allow_downgrade):
                # downgrade in action and downgrade not allowed, aborting!
                print_warning(
                    "%s would be downgraded, %s to %s, ignoring" % (
                        cp_slot, best_installed, best_visible,))
                continue

            if is_rebuild and (not allow_rebuild):
                # rebuild in action and rebuild not allowed, aborting!
                print_warning(
                    "%s would be rebuilt to %s, ignoring" % (
                        cp_slot, best_visible,))
                continue

            # at this point we can go ahead accepting package in queue
            print_info("package: %s [%s], accepted in queue" % (
                    best_visible, cp_slot,))
            accepted.append(best_visible)

        return accepted
Exemple #5
0
    def _commit(self, spec, packages):
        """
        Commit packages that have been merged into the system.
        Overridden from BaseBinaryPMS.
        """
        repository = spec["repository"]
        spm = self._entropy.Spm()
        spm_atoms = set()
        exit_st = 0

        print_info("committing packages: %s, to repository: %s" % (
            ", ".join(sorted(packages)), repository,))

        # if we get here, something has been compiled
        # successfully
        for package in packages:
            try:
                spm_atom = spm.match_installed_package(package)
                spm_atoms.add(spm_atom)
            except KeyError:
                exit_st = 1
                print_warning(
                    "cannot find installed package: %s" % (
                        package,))
                continue

        if not spm_atoms:
            return exit_st

        print_info("about to commit:")
        spm_packages = sorted(spm_atoms)

        for atom in spm_packages:
            item_txt = atom

            # this is a spm atom
            spm_key = portage.dep.dep_getkey("=%s" % (atom,))
            try:
                spm_slot = spm.get_installed_package_metadata(
                    atom, "SLOT")
                spm_repo = spm.get_installed_package_metadata(
                    atom, "repository")
            except KeyError:
                spm_slot = None
                spm_repo = None

            etp_repo = None
            if spm_repo is not None:
                pkg_id, repo_id = self._entropy.atom_match(spm_key,
                    match_slot = spm_slot)
                if repo_id != 1:
                    repo_db = self._entropy.open_repository(repo_id)
                    etp_repo = repo_db.retrieveSpmRepository(pkg_id)

                    if (etp_repo is not None) and (etp_repo != spm_repo):
                        item_txt += ' [%s {%s=>%s}]' % ("warning",
                            etp_repo, spm_repo,)

            print_info(item_txt)

        # always stuff new configuration files here
        # if --gentle was specified, the uncommitted stuff here belongs
        # to our packages.
        # if --gentle was NOT specified, we just don't give a shit
        # Due to bug #2480 -- sometimes (app-misc/tracker)
        # _check_config_file_updates() doesn't return all the files
        subprocess.call("echo -5 | etc-update", shell = True)
        uncommitted = self._entropy._check_config_file_updates()
        if uncommitted:
            # ouch, wtf? better aborting
            print_error("tried to commit configuration file changes and failed")
            return 1

        print_info("about to compress:")

        store_dir = self._entropy._get_local_store_directory(repository)
        package_paths = []
        for atom in spm_packages:
            print_info(atom)
            try:
                pkg_list = spm.generate_package(atom, store_dir)
            except OSError:
                print_traceback()
                print_error("problem during package generation, aborting")
                return 1
            except Exception:
                print_traceback()
                print_error("problem during package generation (2), aborting")
                return 1
            package_paths.append(pkg_list)

        etp_pkg_files = [(pkg_list, False) for pkg_list in package_paths]
        # NOTE: any missing runtime dependency will be added
        # (beside those blacklisted), since this execution is not interactive
        try:
            package_ids = self._entropy.add_packages_to_repository(
                repository, etp_pkg_files, ask=False)
        except OnlineMirrorError as err:
            print_traceback()
            print_error("problem during package commit: %s" % (err,))
            return 1

        self._entropy.commit_repositories()

        if package_ids:
            self._entropy.dependencies_test(repository)

        return exit_st
Exemple #6
0
    def _commit(self, spec, packages):
        """
        Commit packages that have been merged into the system.
        Overridden from BaseBinaryPMS.
        """
        repository = spec["repository"]
        spm = self._entropy.Spm()
        spm_atoms = set()
        exit_st = 0

        print_info("committing packages: %s, to repository: %s" % (
            ", ".join(sorted(packages)),
            repository,
        ))

        # if we get here, something has been compiled
        # successfully
        for package in packages:
            try:
                spm_atom = spm.match_installed_package(package)
                spm_atoms.add(spm_atom)
            except KeyError:
                exit_st = 1
                print_warning("cannot find installed package: %s" %
                              (package, ))
                continue

        if not spm_atoms:
            return exit_st

        print_info("about to commit:")
        spm_packages = sorted(spm_atoms)

        for atom in spm_packages:
            item_txt = atom

            # this is a spm atom
            spm_key = portage.dep.dep_getkey("=%s" % (atom, ))
            try:
                spm_slot = spm.get_installed_package_metadata(atom, "SLOT")
                spm_repo = spm.get_installed_package_metadata(
                    atom, "repository")
            except KeyError:
                spm_slot = None
                spm_repo = None

            etp_repo = None
            if spm_repo is not None:
                pkg_id, repo_id = self._entropy.atom_match(spm_key,
                                                           match_slot=spm_slot)
                if repo_id != 1:
                    repo_db = self._entropy.open_repository(repo_id)
                    etp_repo = repo_db.retrieveSpmRepository(pkg_id)

                    if (etp_repo is not None) and (etp_repo != spm_repo):
                        item_txt += ' [%s {%s=>%s}]' % (
                            "warning",
                            etp_repo,
                            spm_repo,
                        )

            print_info(item_txt)

        # always stuff new configuration files here
        # if --gentle was specified, the uncommitted stuff here belongs
        # to our packages.
        # if --gentle was NOT specified, we just don't give a shit
        # Due to bug #2480 -- sometimes (app-misc/tracker)
        # _check_config_file_updates() doesn't return all the files
        subprocess.call("echo -5 | etc-update", shell=True)
        uncommitted = self._entropy._check_config_file_updates()
        if uncommitted:
            # ouch, wtf? better aborting
            print_error(
                "tried to commit configuration file changes and failed")
            return 1

        print_info("about to compress:")

        store_dir = self._entropy._get_local_store_directory(repository)
        package_paths = []
        for atom in spm_packages:
            print_info(atom)
            try:
                pkg_list = spm.generate_package(atom, store_dir)
            except OSError:
                print_traceback()
                print_error("problem during package generation, aborting")
                return 1
            except Exception:
                print_traceback()
                print_error("problem during package generation (2), aborting")
                return 1
            package_paths.append(pkg_list)

        etp_pkg_files = [(pkg_list, False) for pkg_list in package_paths]
        # NOTE: any missing runtime dependency will be added
        # (beside those blacklisted), since this execution is not interactive
        try:
            package_ids = self._entropy.add_packages_to_repository(
                repository, etp_pkg_files, ask=False)
        except OnlineMirrorError as err:
            print_traceback()
            print_error("problem during package commit: %s" % (err, ))
            return 1

        self._entropy.commit_repositories()

        if package_ids:
            self._entropy.dependencies_test(repository)

        return exit_st
Exemple #7
0
def matter_main(binary_pms, nsargs, cwd, specs):
    """
    Main application code run after all the resources setup.
    """

    try:
        binary_pms.validate_system()
    except BaseBinaryPMS.SystemValidationError as err:
        print_error("%s" % (err,))
        return 1

    print_info("matter loaded, starting to scan particles, pid: %s" % (
        os.getpid(),))

    def _teardown(_exit_st):
        if nsargs.post:
            _rc = PackageBuilder.teardown(
                nsargs.post, cwd, _exit_st)
            if _exit_st == 0 and _rc != 0:
                _exit_st = _rc
        return _exit_st

    # setup
    if nsargs.pre:
        _rc = PackageBuilder.setup(nsargs.pre, cwd)
        if _rc != 0:
            return _teardown(_rc)

    # sync portage
    if nsargs.sync:
        _rc = PackageBuilder.sync()
        if _rc != 0 and not nsargs.sync_best_effort:
            return _teardown(_rc)

    exit_st = 0
    completed = collections.deque()
    not_found = collections.deque()
    not_installed = collections.deque()
    not_merged = collections.deque()
    uninstalled = collections.deque()
    missing_use = {}
    unstable_keywords = set()
    pmask_changes = set()
    license_changes = {}
    tainted_repositories = set()
    spec_count = 0
    tot_spec = len(specs)
    preserved_libs = False
    emerge_config = binary_pms.load_emerge_config()

    for spec in specs:

        spec_count += 1
        keep_going = spec["keep-going"] == "yes"
        local_completed = []
        local_uninstalled = []

        tot_pkgs = len(spec["packages"])
        for pkg_count, packages in enumerate(spec["packages"], 1):

            builder = PackageBuilder(
                binary_pms, emerge_config, packages,
                spec, spec_count, tot_spec, pkg_count, tot_pkgs,
                nsargs.pretend)
            _rc = builder.run()

            not_found.extend(builder.get_not_found_packages())
            not_installed.extend(
                builder.get_not_installed_packages())
            not_merged.extend(
                builder.get_not_merged_packages())
            uninstalled = builder.get_uninstalled_packages()
            uninstalled.extend(uninstalled)
            local_uninstalled.extend(uninstalled)

            # Merge at least the first layer of dicts.
            for k, v in builder.get_missing_use_packages().items():
                obj = missing_use.setdefault(k, {})
                obj.update(v)

            unstable_keywords.update(
                builder.get_needed_unstable_keywords())
            pmask_changes.update(
                builder.get_needed_package_mask_changes())

            # We need to merge the two dicts, not just update()
            # or we can lose the full set of licenses associated
            # to a single cpv.
            for k, v in builder.get_needed_license_changes().items():
                obj = license_changes.setdefault(k, set())
                obj.update(v)

            preserved_libs = binary_pms.check_preserved_libraries(
                emerge_config)

            if preserved_libs and not nsargs.disable_preserved_libs:
                # abort, library breakages detected
                exit_st = 1
                print_error(
                    "preserved libraries detected, aborting")
                break

            # ignore _rc, we may have built pkgs even if _rc != 0
            built_packages = builder.get_built_packages()
            if built_packages:
                print_info("built packages, in queue: %s" % (
                        " ".join(built_packages),))
                local_completed.extend(
                    [x for x in built_packages \
                         if x not in local_completed])
                tainted_repositories.add(spec["repository"])

            # make some room
            print_info("")
            if _rc < 0:
                # ignore warning and go ahead
                continue
            else:
                exit_st = _rc
                if not keep_going:
                    break

        # call post-build cleanup operations
        if local_completed or local_uninstalled:
            PackageBuilder.post_build(spec, emerge_config)

        completed.extend([x for x in local_completed \
            if x not in completed])
        # portage calls setcwd()
        os.chdir(cwd)

        if preserved_libs and not nsargs.disable_preserved_libs:
            # completely abort
            break

        if local_completed and nsargs.commit:
            _rc = binary_pms.commit(
                spec,
                local_completed)
            if exit_st == 0 and _rc != 0:
                exit_st = _rc
                if not keep_going:
                    break

        PackageBuilder.clear_caches(emerge_config)

    if tainted_repositories and nsargs.push and nsargs.commit:
        if preserved_libs and nsargs.disable_preserved_libs:
            # cannot push anyway
            print_warning("Preserved libraries detected, cannot push !")
        elif not preserved_libs:
            for repository in tainted_repositories:
                _rc = binary_pms.push(repository)
                if exit_st == 0 and _rc != 0:
                    exit_st = _rc

    # print summary
    print_generic("")
    print_generic("Summary")
    print_generic("Packages built:\n  %s" % (
        "\n  ".join(sorted(completed)),))
    print_generic("Packages not built:\n  %s" % (
        "\n  ".join(sorted(not_merged)),))
    print_generic("Packages not found:\n  %s" % (
        "\n  ".join(sorted(not_found)),))
    print_generic("Packages not installed:\n  %s" % (
        "\n  ".join(sorted(not_installed)),))
    print_generic("Packages uninstalled:\n  %s" % (
        "\n  ".join(sorted(uninstalled)),))

    if missing_use:
        print_generic("Packages not built due to missing USE flags:")
        for atom in sorted(missing_use.keys()):
            use_data = missing_use[atom]
            use_l = []
            for use in sorted(use_data["changes"]):
                if use_data["changes"][use]:
                    use_l.append(use)
                else:
                    use_l.append("-" + use)
            print_generic("%s %s" % (
                    use_data["cp:slot"], " ".join(use_l)))
        print_generic("")

    if unstable_keywords:
        print_generic("Packages not built due to missing unstable keywords:")
        for atom in sorted(unstable_keywords):
            print_generic("%s" % (atom,))
        print_generic("")

    if pmask_changes:
        print_generic("Packages not built due to needed package.mask changes:")
        for atom in sorted(pmask_changes):
            print_generic("%s" % (atom,))
        print_generic("")

    print_generic("Preserved libs: %s" % (
        preserved_libs,))
    print_generic("")

    return _teardown(exit_st)
Exemple #8
0
def main():
    """
    Main App.
    """
    install_exception_handler()

    # disable color if standard output is not a TTY
    if not is_stdout_a_tty():
        nocolor()

    # Load Binary PMS modules
    import matter.binpms as _pms
    pms_dir = os.path.dirname(_pms.__file__)
    for thing in os.listdir(pms_dir):
        if thing.startswith("__init__.py"):
            continue

        thing = os.path.join(pms_dir, thing)
        if not os.path.isfile(thing):
            continue
        if not thing.endswith(".py"):
            continue

        name = os.path.basename(thing)
        name = name.rstrip(".py")
        package = "matter.binpms.%s" % (name,)

        try:
            importlib.import_module(package)  # they will then register
        except ImportError as err:
            pass
    avail_binpms = BaseBinaryPMS.available_pms

    matter_spec = MatterSpec()
    parser_data = matter_spec.data()
    matter_spec_params = ""
    for spec_key in sorted(parser_data.keys()):
        par = parser_data[spec_key]
        matter_spec_params += "%s: %s\n" % (
            purple(spec_key),
            darkgreen(par.get("desc", "N/A")),)

    _env_vars_help = """\

Environment variables for Package Builder module:
%s       =  repository identifier
%s    =  alternative command used to sync Portage
                              default: %s
%s   =  alternative command used to sync Portage overlays
                              default: %s

Environment variables passed to --post executables:
%s        = exit status from previous execution phases, useful for detecting
                             execution errors.

Matter Resources Lock file you can use to detect if matter is running:
%s (--blocking switch makes it acquire in blocking mode)

Matter .spec file supported parameters:
%s

Available Binary PMSs:
%s
""" % (
        purple("MATTER_REPOSITORY_ID"),
        purple("MATTER_PORTAGE_SYNC_CMD"),
        darkgreen(PackageBuilder.DEFAULT_PORTAGE_SYNC_CMD),
        purple("MATTER_OVERLAYS_SYNC_CMD"),
        darkgreen(PackageBuilder.DEFAULT_OVERLAYS_SYNC_CMD),
        purple("MATTER_EXIT_STATUS"),
        darkgreen(MatterResourceLock.LOCK_FILE_PATH),
        matter_spec_params,
        "\n".join(
        ["%s: %s" % (purple(k.NAME), darkgreen(k.__name__)) \
             for k in avail_binpms]),)

    parser = argparse.ArgumentParser(
        description="Automated Packages Builder",
        epilog=_env_vars_help,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    # * instead of + in order to support --sync only tasks
    parser.add_argument(
        "spec", nargs="+", metavar="<spec>", type=file,
        help="matter spec file")

    default_pms = avail_binpms[0]
    for k in avail_binpms:
        if k.DEFAULT:
            default_pms = k
            break
    parser.add_argument(
        "--pms", default=default_pms.NAME,
        help="specify an alternative Binary PMS (see --help for a list), "
        "current default: %s" % (default_pms.NAME,))

    parser.add_argument(
        "--blocking",
        help="when trying to acquire Binary PMS locks, "
        "block until success.",
        action="store_true")

    parser.add_argument("--commit",
        help="commit built packages to repository.",
        action="store_true")

    parser.add_argument(
        "--gentle",
        help="increase the system validation checks, be extremely "
        "careful wrt the current system status.",
        action="store_true")

    parser.add_argument("--pre", metavar="<exec>", type=file,
        help="executable to be called once for setup purposes.",
        default=None)

    parser.add_argument("--post", metavar="<exec>", type=file,
        help="executable to be called once for teardown purposes.",
        default=None)

    parser.add_argument(
        "--push",
        help="push Binary PMS package updates to online "
        "repository (only if --commit).",
        action="store_true")

    parser.add_argument(
        "--sync",
        help="sync Portage tree, and attached overlays, before starting.",
        action="store_true")

    parser.add_argument(
        "--sync-best-effort", default=False,
        help="sync Portage tree and attached overlays, as --sync, but do "
        "not exit if sync fails.",
        action="store_true")

    parser.add_argument(
        "--disable-preserved-libs",
        dest="disable_preserved_libs", default=False,
        help="disable prerserved libraries check.",
        action="store_true")

    parser.add_argument(
        "--pretend",
        dest="pretend", default=False,
        help="show what would be done without alterint the current system.",
        action="store_true")

    # extend parser arguments
    for k in avail_binpms:
        k.extend_parser(parser)

    try:
        nsargs = parser.parse_args(sys.argv[1:])
    except IOError as err:
        if err.errno == errno.ENOENT:
            print_error(err.strerror + ": " + err.filename)
            return 1
        raise

    if os.getuid() != 0:
        # root access required
        print_error("superuser access required")
        return 1

    # parse spec files
    specs = []
    for spec_f in nsargs.spec:
        spec = SpecParser(spec_f)
        data = spec.parse()
        if data:
            specs.append(data)

    if not specs:
        print_error("invalid spec files provided")
        return 1

    # O(n) determine what is the BinaryPMS to use
    klass = None
    for k in avail_binpms:
        if k.NAME == nsargs.pms:
            klass = k
            break
    if klass is None:
        print_error("invalid Binary PMS specified: %s" % (nsargs.pms,))
        return 1

    binary_pms = None
    exit_st = 0
    cwd = os.getcwd()
    try:
        try:
            binary_pms = klass(cwd, nsargs)
        except BaseBinaryPMS.BinaryPMSLoadError as err:
            # repository not available or not configured
            print_error("Cannot load Binary Package Manager: %s" % (err,))
            return 3

        print_info("Loaded Binary PMS: %s" % (klass.NAME,))

        # validate repository entries of spec metadata
        for spec in specs:
            try:
                binary_pms.validate_spec(spec)
            except BaseBinaryPMS.SpecParserError as err:
                print_error("%s" % (err,))
                return 1

        if nsargs.blocking:
            print_info("--blocking enabled, please wait for locks...")

        resource_lock = binary_pms.get_resource_lock(nsargs.blocking)
        with resource_lock:
            with MatterResourceLock(nsargs.blocking):
                exit_st = matter_main(binary_pms, nsargs, cwd, specs)

    except BaseBinaryResourceLock.NotAcquired:
        print_error("unable to acquire PMS Resources lock")
        return 42
    except MatterResourceLock.NotAcquired:
        print_error("unable to acquire Matter Resources lock")
        return 42
    except KeyboardInterrupt:
        print_error("Keyboard Interrupt, pid: %s" % (os.getpid(),))
        return 42
    finally:
        if binary_pms is not None:
            binary_pms.shutdown()

    print_warning("")
    print_warning("")
    print_warning("Tasks complete, exit status: %d" % (exit_st,))
    return exit_st
Exemple #9
0
    def _pre_graph_filters(self, package, portdb, vardb):
        """
        Execute basic, pre-graph generation (dependencies calculation)
        filters against the package dependency to see if it's eligible
        for the graph.
        """
        allow_rebuild = self._params["rebuild"] == "yes"
        allow_not_installed = self._params["not-installed"] == "yes"
        allow_downgrade = self._params["downgrade"] == "yes"

        try:
            best_visible = portdb.xmatch("bestmatch-visible", package)
        except portage.exception.InvalidAtom:
            print_error("cannot match: %s, invalid atom" % (package,))
            best_visible = None

        if not best_visible:
            # package not found, return error
            print_error("cannot match: %s, ignoring this one" % (package,))
            self._not_found_packages.append(package)
            return None

        print_info("matched: %s for %s" % (best_visible, package,))
        # now determine what's the installed version.
        best_installed = portage.best(vardb.match(package))
        if (not best_installed) and (not allow_not_installed):
            # package not installed
            print_error("package not installed: %s, ignoring this one" % (
                package,))
            self._not_installed_packages.append(package)
            return None

        if (not best_installed) and allow_not_installed:
            print_warning(
                "package not installed: "
                "%s, but 'not-installed: yes' provided" % (package,))

        build_only = self._params["build-only"] == "yes"
        cmp_res = -1
        if best_installed:
            print_info("found installed: %s for %s" % (
                    best_installed, package,))
            # now compare
            # -1 if best_installed is older than best_visible
            # 1 if best_installed is newer than best_visible
            # 0 if they are equal
            cmp_res = portage.versions.pkgcmp(
                portage.versions.pkgsplit(best_installed),
                portage.versions.pkgsplit(best_visible))
        elif (not best_installed) and build_only:
            # package is not installed, and build-only
            # is provided. We assume that the package
            # is being built and added to repositories directly.
            # This means that we need to query binpms to know
            # about the current version.
            print_info("package is not installed, and 'build-only: yes'. "
                       "Asking the binpms about the package state.")
            best_available = self._binpms.best_available(package)
            print_info("found available: %s for %s" % (
                    best_available, package))
            if best_available:
                cmp_res = portage.versions.pkgcmp(
                    portage.versions.pkgsplit(best_available),
                    portage.versions.pkgsplit(best_visible))

        is_rebuild = cmp_res == 0

        if (cmp_res == 1) and (not allow_downgrade):
            # downgrade in action and downgrade not allowed, aborting!
            print_warning(
                "package: %s, would be downgraded, %s to %s, ignoring" % (
                    package, best_installed, best_visible,))
            return None

        if is_rebuild and (not allow_rebuild):
            # rebuild in action and rebuild not allowed, aborting!
            print_warning(
                "package: %s, would be rebuilt to %s, ignoring" % (
                    package, best_visible,))
            return None

        # at this point we can go ahead accepting package in queue
        print_info("package: %s [%s], accepted in queue" % (
                best_visible, package,))
        return best_visible
Exemple #10
0
    def _run_builder(self, dirs_cleanup_queue):
        """
        This method is called by _run and executes the whole package build
        logic, including constraints validation given by argv parameters.
        NOTE: negative errors indicate warnings that can be skipped.
        """
        if self._packages:
            first_package = self._packages[0]
        else:
            first_package = "_empty_"

        log_dir = mkdtemp(prefix="matter_build.",
                          suffix="." +
                          first_package.replace("/", "_").lstrip("<>=~"))
        dirs_cleanup_queue.append(log_dir)

        emerge_settings, emerge_trees, mtimedb = self._emerge_config

        # reset settings to original state, variables will be reconfigured
        # while others may remain saved due to backup_changes().
        emerge_settings.unlock()
        emerge_settings.reset()
        emerge_settings.lock()

        # Setup stable/unstable keywords, must be done on
        # emerge_settings bacause the reference is spread everywhere
        # in emerge_trees.
        # This is not thread-safe, but Portage isn't either, so
        # who cares!
        # ACCEPT_KEYWORDS is not saved and reset every time by the
        # reset() call above.
        portdb = emerge_trees[emerge_settings["ROOT"]]["porttree"].dbapi

        self._setup_keywords(portdb, emerge_settings)

        portdb.freeze()
        vardb = emerge_trees[emerge_settings["ROOT"]]["vartree"].dbapi
        vardb.settings.unlock()
        vardb.settings["PORT_LOGDIR"] = log_dir
        vardb.settings.backup_changes("PORT_LOGDIR")
        vardb.settings.lock()

        # Load the most current variables from /etc/profile.env, which
        # has been re-generated by the env-update call in _run()
        emerge_settings.unlock()
        emerge_settings.reload()
        emerge_settings.regenerate()
        emerge_settings.lock()

        sets = self._get_sets_mod()  # can be None
        sets_conf = None
        if sets is not None:
            sets_conf = sets.load_default_config(
                emerge_settings, emerge_trees[emerge_settings["ROOT"]])

        packages = []
        # execute basic, pre-graph generation filters against each
        # package dependency in self._packages.
        # This is just fast pruning of obvious obviousness.
        for package in self._packages:
            expanded_pkgs = []

            # package sets support
            if package.startswith("@") and sets_conf:
                try:
                    set_pkgs = sets_conf.getSetAtoms(package[1:])
                    expanded_pkgs.extend(sorted(set_pkgs))
                except sets.PackageSetNotFound:
                    # make it fail, add set directly
                    expanded_pkgs.append(package)
            else:
                expanded_pkgs.append(package)

            for exp_pkg in expanded_pkgs:
                accepted = self._pre_graph_filters(exp_pkg, portdb, vardb)
                for best_visible in accepted:
                    packages.append((exp_pkg, best_visible))

        if not packages:
            print_warning("No remaining packages in queue, aborting.")
            return 0

        # at this point we can go ahead building packages
        print_info("starting to build:")
        for package, best_visible in packages:
            print_info(": %s -> %s" % (
                package,
                best_visible,
            ))

        if not getcolor():
            portage.output.nocolor()

        # non interactive properties, this is not really required
        # accept-properties just sets os.environ...
        build_args = list(self._setup_build_args(self._params))
        build_args += ["=" + best_v for _x, best_v in packages]

        myaction, myopts, myfiles = parse_opts(build_args)
        adjust_configs(myopts, emerge_trees)
        apply_priorities(emerge_settings)

        spinner = stdout_spinner()
        if "--quiet" in myopts:
            spinner.update = spinner.update_basic
        elif "--nospinner" in myopts:
            spinner.update = spinner.update_basic
        if emerge_settings.get("TERM") == "dumb" or not is_stdout_a_tty():
            spinner.update = spinner.update_basic

        print_info("emerge args: %s" % (" ".join(build_args), ))

        params = create_depgraph_params(myopts, myaction)
        success, graph, favorites = backtrack_depgraph(emerge_settings,
                                                       emerge_trees, myopts,
                                                       params, myaction,
                                                       myfiles, spinner)

        if not success:
            # print issues to stdout and give up
            print_warning("dependencies calculation failed, aborting")
            graph.display_problems()

            # try to collect some info about the failure
            bt_config = (graph.get_backtrack_infos() or {}).get("config", {})
            for k, v in bt_config.items():
                if k == "needed_use_config_changes":
                    for tup in v:
                        try:
                            pkg, (new_use, new_changes) = tup
                        except (ValueError, TypeError):
                            print_error(
                                "unsupported needed_use_config_changes: %s" %
                                (tup, ))
                            continue
                        obj = self._missing_use_packages.setdefault(
                            "%s" % (pkg.cpv, ), {})
                        obj["cp:slot"] = "%s" % (pkg.slot_atom, )
                        changes = obj.setdefault("changes", {})
                        changes.update(copy.deepcopy(new_changes))
                elif k == "needed_unstable_keywords":
                    for pkg in v:
                        self._needed_unstable_keywords.add("%s" % (pkg.cpv, ))
                elif k == "needed_p_mask_changes":
                    for pkg in v:
                        self._needed_package_mask_changes.add("%s" %
                                                              (pkg.cpv, ))
                elif k == "needed_license_changes":
                    for pkg, lics in v:
                        obj = self._needed_license_changes.setdefault(
                            "%s" % (pkg.cpv, ), set())
                        obj.update(lics)
                else:
                    print_warning("unsupported backtrack info: %s -> %s" % (
                        k,
                        v,
                    ))

            return 0
        print_info("dependency graph generated successfully")

        real_queue = self._post_graph_filters(graph, vardb, portdb)
        if real_queue is None:
            # post-graph filters not passed, giving up
            return 0

        merge_queue = [x for x in real_queue if x.operation == "merge"]
        unmerge_queue = [x for x in real_queue if x.operation == "uninstall"]
        if merge_queue:
            print_info("about to build the following packages:")
            for pkg in merge_queue:
                print_info("  %s" % (pkg.cpv, ))
        if unmerge_queue:
            print_info("about to uninstall the following packages:")
            for pkg in unmerge_queue:
                print_info("  %s" % (pkg.cpv, ))

        if self._pretend:
            print_info("portage spawned with --pretend, done!")
            return 0

        # re-calling action_build(), deps are re-calculated though
        validate_ebuild_environment(emerge_trees)
        mergetask = Scheduler(emerge_settings,
                              emerge_trees,
                              mtimedb,
                              myopts,
                              spinner,
                              favorites=favorites,
                              graph_config=graph.schedulerGraph())
        del graph
        self.clear_caches(self._emerge_config)
        retval = mergetask.merge()

        not_merged = []
        real_queue_map = dict((pkg.cpv, pkg) for pkg in real_queue)
        failed_package = None
        if retval != 0:
            merge_list = mtimedb.get("resume", {}).get("mergelist", [])
            for _merge_type, _merge_root, merge_atom, _merge_act in merge_list:
                merge_atom = "%s" % (merge_atom, )
                if failed_package is None:
                    # we consider the first encountered package the one
                    # that failed. It makes sense since packages are built
                    # serially as of today.
                    # Also, the package object must be available in our
                    # package queue, so grab it from there.
                    failed_package = real_queue_map.get(merge_atom)
                not_merged.append(merge_atom)
                self._not_merged_packages.append(merge_atom)

        for pkg in real_queue:
            cpv = pkg.cpv
            if not cpv:
                print_warning("package: %s, has broken cpv: '%s', ignoring" % (
                    pkg,
                    cpv,
                ))
            elif cpv not in not_merged:
                if pkg.operation == "merge":
                    # add to build queue
                    print_info("package: %s, successfully built" % (cpv, ))
                    self._built_packages.append("%s" % (cpv, ))
                else:
                    # add to uninstall queue
                    print_info("package: %s, successfully uninstalled" %
                               (cpv, ))
                    self._uninstalled_packages.append("%s" % (cpv, ))

        post_emerge(myaction, myopts, myfiles, emerge_settings["ROOT"],
                    emerge_trees, mtimedb, retval)

        subprocess.call(["env-update"])

        if failed_package is not None:
            print_warning("failed package: %s::%s" % (
                failed_package.cpv,
                failed_package.repo,
            ))

        if self._params["buildfail"] and (failed_package is not None):

            std_env = self._build_standard_environment(
                repository=self._params["repository"])
            std_env["MATTER_PACKAGE_NAMES"] = " ".join(self._packages)
            std_env["MATTER_PORTAGE_FAILED_PACKAGE_NAME"] = failed_package.cpv
            std_env["MATTER_PORTAGE_REPOSITORY"] = failed_package.repo
            # call pkgfail hook if defined
            std_env["MATTER_PORTAGE_BUILD_LOG_DIR"] = os.path.join(
                log_dir, "build")

            buildfail = self._params["buildfail"]
            print_info("spawning buildfail: %s" % (buildfail, ))
            tmp_fd, tmp_path = mkstemp()
            with os.fdopen(tmp_fd, "wb") as tmp_f:
                with open(buildfail, "rb") as buildfail_f:
                    tmp_f.write(buildfail_f.read())
            try:
                # now execute
                os.chmod(tmp_path, 0o700)
                exit_st = subprocess.call([tmp_path], env=std_env)
                if exit_st != 0:
                    return exit_st
            finally:
                os.remove(tmp_path)

        print_info("portage spawned, return value: %d" % (retval, ))
        return retval
Exemple #11
0
    def _pre_graph_filters(self, package, portdb, vardb):
        """
        Execute basic, pre-graph generation (dependencies calculation)
        filters against the package dependency to see if it's eligible
        for the graph.
        """
        allow_rebuild = self._params["rebuild"] == "yes"
        allow_not_installed = self._params["not-installed"] == "yes"
        allow_downgrade = self._params["downgrade"] == "yes"
        accepted = []

        # now determine what's the installed version.
        best_installed = portage.best(vardb.match(package, use_cache=0))
        if (not best_installed) and (not allow_not_installed):
            # package not installed
            print_error("package not installed: %s, ignoring this one" %
                        (package, ))
            self._not_installed_packages.append(package)
            return accepted

        if (not best_installed) and allow_not_installed:
            print_warning(
                "%s not installed, but 'not-installed: yes' provided" %
                (package, ))

        best_visibles = []
        try:
            best_visibles += portdb.xmatch("match-visible", package)
        except portage.exception.InvalidAtom:
            print_error("cannot match: %s, invalid atom" % (package, ))

        # map all the cpvs to their slots
        cpv_slot_map = {}
        for pkg in best_visibles:
            obj = cpv_slot_map.setdefault(pkg.slot, [])
            obj.append(pkg)

        # then pick the best for each slot
        del best_visibles[:]
        for slot, pkgs in cpv_slot_map.items():
            pkg = portage.best(pkgs)
            best_visibles.append(pkg)
        best_visibles.sort()  # deterministic is better

        if not best_visibles:
            # package not found, return error
            print_error("cannot match: %s, ignoring this one" % (package, ))
            self._not_found_packages.append(package)
            return accepted

        print_info("matched: %s for %s" % (
            ", ".join(best_visibles),
            package,
        ))

        for best_visible in best_visibles:

            cp = best_visible.cp
            slot = best_visible.slot
            cp_slot = "%s:%s" % (cp, slot)

            # determine what's the installed version.
            # we know that among all the best_visibles, there is one that
            # is installed. The question is whether we got it now.
            best_installed = portage.best(vardb.match(cp_slot, use_cache=0))
            if (not best_installed) and (not allow_not_installed):
                # package not installed
                print_warning("%s not installed, skipping" % (cp_slot, ))
                continue

            build_only = self._params["build-only"] == "yes"
            cmp_res = -1
            if best_installed:
                print_info("found installed: %s for %s" % (
                    best_installed,
                    package,
                ))
                # now compare
                # -1 if best_installed is older than best_visible
                # 1 if best_installed is newer than best_visible
                # 0 if they are equal
                cmp_res = portage.versions.pkgcmp(
                    portage.versions.pkgsplit(best_installed),
                    portage.versions.pkgsplit(best_visible))
            elif (not best_installed) and build_only:
                # package is not installed, and build-only
                # is provided. We assume that the package
                # is being built and added to repositories directly.
                # This means that we need to query binpms to know
                # about the current version.
                print_info("package is not installed, and 'build-only: yes'. "
                           "Asking the binpms about the package state.")
                best_available = self._binpms.best_available(cp_slot)
                print_info("found available: %s for %s" %
                           (best_available, cp_slot))
                if best_available:
                    cmp_res = portage.versions.pkgcmp(
                        portage.versions.pkgsplit(best_available),
                        portage.versions.pkgsplit(best_visible))

            is_rebuild = cmp_res == 0

            if (cmp_res == 1) and (not allow_downgrade):
                # downgrade in action and downgrade not allowed, aborting!
                print_warning("%s would be downgraded, %s to %s, ignoring" % (
                    cp_slot,
                    best_installed,
                    best_visible,
                ))
                continue

            if is_rebuild and (not allow_rebuild):
                # rebuild in action and rebuild not allowed, aborting!
                print_warning("%s would be rebuilt to %s, ignoring" % (
                    cp_slot,
                    best_visible,
                ))
                continue

            # at this point we can go ahead accepting package in queue
            print_info("package: %s [%s], accepted in queue" % (
                best_visible,
                cp_slot,
            ))
            accepted.append(best_visible)

        return accepted
Exemple #12
0
def matter_main(binary_pms, nsargs, cwd, specs):
    """
    Main application code run after all the resources setup.
    """

    try:
        binary_pms.validate_system()
    except BaseBinaryPMS.SystemValidationError as err:
        print_error("%s" % (err, ))
        return 1

    print_info("matter loaded, starting to scan particles, pid: %s" %
               (os.getpid(), ))

    def _teardown(_exit_st):
        if nsargs.post:
            _rc = PackageBuilder.teardown(nsargs.post, cwd, _exit_st)
            if _exit_st == 0 and _rc != 0:
                _exit_st = _rc
        return _exit_st

    # setup
    if nsargs.pre:
        _rc = PackageBuilder.setup(nsargs.pre, cwd)
        if _rc != 0:
            return _teardown(_rc)

    # sync portage
    if nsargs.sync:
        _rc = PackageBuilder.sync()
        if _rc != 0 and not nsargs.sync_best_effort:
            return _teardown(_rc)

    exit_st = 0
    completed = collections.deque()
    not_found = collections.deque()
    not_installed = collections.deque()
    not_merged = collections.deque()
    uninstalled = collections.deque()
    missing_use = {}
    unstable_keywords = set()
    pmask_changes = set()
    license_changes = {}
    tainted_repositories = set()
    spec_count = 0
    tot_spec = len(specs)
    preserved_libs = False
    emerge_config = binary_pms.load_emerge_config()

    for spec in specs:

        spec_count += 1
        keep_going = spec["keep-going"] == "yes"
        local_completed = []
        local_uninstalled = []

        tot_pkgs = len(spec["packages"])
        for pkg_count, packages in enumerate(spec["packages"], 1):

            builder = PackageBuilder(binary_pms, emerge_config, packages, spec,
                                     spec_count, tot_spec, pkg_count, tot_pkgs,
                                     nsargs.pretend)
            _rc = builder.run()

            not_found.extend(builder.get_not_found_packages())
            not_installed.extend(builder.get_not_installed_packages())
            not_merged.extend(builder.get_not_merged_packages())
            uninstalled = builder.get_uninstalled_packages()
            uninstalled.extend(uninstalled)
            local_uninstalled.extend(uninstalled)

            # Merge at least the first layer of dicts.
            for k, v in builder.get_missing_use_packages().items():
                obj = missing_use.setdefault(k, {})
                obj.update(v)

            unstable_keywords.update(builder.get_needed_unstable_keywords())
            pmask_changes.update(builder.get_needed_package_mask_changes())

            # We need to merge the two dicts, not just update()
            # or we can lose the full set of licenses associated
            # to a single cpv.
            for k, v in builder.get_needed_license_changes().items():
                obj = license_changes.setdefault(k, set())
                obj.update(v)

            preserved_libs = binary_pms.check_preserved_libraries(
                emerge_config)

            if preserved_libs and not nsargs.disable_preserved_libs:
                # abort, library breakages detected
                exit_st = 1
                print_error("preserved libraries detected, aborting")
                break

            # ignore _rc, we may have built pkgs even if _rc != 0
            built_packages = builder.get_built_packages()
            if built_packages:
                print_info("built packages, in queue: %s" %
                           (" ".join(built_packages), ))
                local_completed.extend(
                    [x for x in built_packages \
                         if x not in local_completed])
                tainted_repositories.add(spec["repository"])

            # make some room
            print_info("")
            if _rc < 0:
                # ignore warning and go ahead
                continue
            else:
                exit_st = _rc
                if not keep_going:
                    break

        # call post-build cleanup operations
        if local_completed or local_uninstalled:
            PackageBuilder.post_build(spec, emerge_config)

        completed.extend([x for x in local_completed \
            if x not in completed])
        # portage calls setcwd()
        os.chdir(cwd)

        if preserved_libs and not nsargs.disable_preserved_libs:
            # completely abort
            break

        if local_completed and nsargs.commit:
            _rc = binary_pms.commit(spec, local_completed)
            if exit_st == 0 and _rc != 0:
                exit_st = _rc
                if not keep_going:
                    break

        PackageBuilder.clear_caches(emerge_config)

    if tainted_repositories and nsargs.push and nsargs.commit:
        if preserved_libs and nsargs.disable_preserved_libs:
            # cannot push anyway
            print_warning("Preserved libraries detected, cannot push !")
        elif not preserved_libs:
            for repository in tainted_repositories:
                _rc = binary_pms.push(repository)
                if exit_st == 0 and _rc != 0:
                    exit_st = _rc

    # print summary
    print_generic("")
    print_generic("Summary")
    print_generic("Packages built:\n  %s" % ("\n  ".join(sorted(completed)), ))
    print_generic("Packages not built:\n  %s" %
                  ("\n  ".join(sorted(not_merged)), ))
    print_generic("Packages not found:\n  %s" %
                  ("\n  ".join(sorted(not_found)), ))
    print_generic("Packages not installed:\n  %s" %
                  ("\n  ".join(sorted(not_installed)), ))
    print_generic("Packages uninstalled:\n  %s" %
                  ("\n  ".join(sorted(uninstalled)), ))

    if missing_use:
        print_generic("Packages not built due to missing USE flags:")
        for atom in sorted(missing_use.keys()):
            use_data = missing_use[atom]
            use_l = []
            for use in sorted(use_data["changes"]):
                if use_data["changes"][use]:
                    use_l.append(use)
                else:
                    use_l.append("-" + use)
            print_generic("%s %s" % (use_data["cp:slot"], " ".join(use_l)))
        print_generic("")

    if unstable_keywords:
        print_generic("Packages not built due to missing unstable keywords:")
        for atom in sorted(unstable_keywords):
            print_generic("%s" % (atom, ))
        print_generic("")

    if pmask_changes:
        print_generic("Packages not built due to needed package.mask changes:")
        for atom in sorted(pmask_changes):
            print_generic("%s" % (atom, ))
        print_generic("")

    print_generic("Preserved libs: %s" % (preserved_libs, ))
    print_generic("")

    return _teardown(exit_st)
Exemple #13
0
def main():
    """
    Main App.
    """
    install_exception_handler()

    # disable color if standard output is not a TTY
    if not is_stdout_a_tty():
        nocolor()

    # Load Binary PMS modules
    import matter.binpms as _pms
    pms_dir = os.path.dirname(_pms.__file__)
    for thing in os.listdir(pms_dir):
        if thing.startswith("__init__.py"):
            continue

        thing = os.path.join(pms_dir, thing)
        if not os.path.isfile(thing):
            continue
        if not thing.endswith(".py"):
            continue

        name = os.path.basename(thing)
        name = name.rstrip(".py")
        package = "matter.binpms.%s" % (name, )

        try:
            importlib.import_module(package)  # they will then register
        except ImportError as err:
            pass
    avail_binpms = BaseBinaryPMS.available_pms

    matter_spec = MatterSpec()
    parser_data = matter_spec.data()
    matter_spec_params = ""
    for spec_key in sorted(parser_data.keys()):
        par = parser_data[spec_key]
        matter_spec_params += "%s: %s\n" % (
            purple(spec_key),
            darkgreen(par.get("desc", "N/A")),
        )

    _env_vars_help = """\

Environment variables for Package Builder module:
%s       =  repository identifier
%s    =  alternative command used to sync Portage
                              default: %s
%s   =  alternative command used to sync Portage overlays
                              default: %s

Environment variables passed to --post executables:
%s        = exit status from previous execution phases, useful for detecting
                             execution errors.

Matter Resources Lock file you can use to detect if matter is running:
%s (--blocking switch makes it acquire in blocking mode)

Matter .spec file supported parameters:
%s

Available Binary PMSs:
%s
""" % (
        purple("MATTER_REPOSITORY_ID"),
        purple("MATTER_PORTAGE_SYNC_CMD"),
        darkgreen(PackageBuilder.DEFAULT_PORTAGE_SYNC_CMD),
        purple("MATTER_OVERLAYS_SYNC_CMD"),
        darkgreen(PackageBuilder.DEFAULT_OVERLAYS_SYNC_CMD),
        purple("MATTER_EXIT_STATUS"),
        darkgreen(MatterResourceLock.LOCK_FILE_PATH),
        matter_spec_params,
        "\n".join(
        ["%s: %s" % (purple(k.NAME), darkgreen(k.__name__)) \
             for k in avail_binpms]),)

    parser = argparse.ArgumentParser(
        description="Automated Packages Builder",
        epilog=_env_vars_help,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    # * instead of + in order to support --sync only tasks
    parser.add_argument("spec",
                        nargs="+",
                        metavar="<spec>",
                        type=file,
                        help="matter spec file")

    default_pms = avail_binpms[0]
    for k in avail_binpms:
        if k.DEFAULT:
            default_pms = k
            break
    parser.add_argument(
        "--pms",
        default=default_pms.NAME,
        help="specify an alternative Binary PMS (see --help for a list), "
        "current default: %s" % (default_pms.NAME, ))

    parser.add_argument("--blocking",
                        help="when trying to acquire Binary PMS locks, "
                        "block until success.",
                        action="store_true")

    parser.add_argument("--commit",
                        help="commit built packages to repository.",
                        action="store_true")

    parser.add_argument(
        "--gentle",
        help="increase the system validation checks, be extremely "
        "careful wrt the current system status.",
        action="store_true")

    parser.add_argument(
        "--pre",
        metavar="<exec>",
        type=file,
        help="executable to be called once for setup purposes.",
        default=None)

    parser.add_argument(
        "--post",
        metavar="<exec>",
        type=file,
        help="executable to be called once for teardown purposes.",
        default=None)

    parser.add_argument("--push",
                        help="push Binary PMS package updates to online "
                        "repository (only if --commit).",
                        action="store_true")

    parser.add_argument(
        "--sync",
        help="sync Portage tree, and attached overlays, before starting.",
        action="store_true")

    parser.add_argument(
        "--sync-best-effort",
        default=False,
        help="sync Portage tree and attached overlays, as --sync, but do "
        "not exit if sync fails.",
        action="store_true")

    parser.add_argument("--disable-preserved-libs",
                        dest="disable_preserved_libs",
                        default=False,
                        help="disable prerserved libraries check.",
                        action="store_true")

    parser.add_argument(
        "--pretend",
        dest="pretend",
        default=False,
        help="show what would be done without alterint the current system.",
        action="store_true")

    # extend parser arguments
    for k in avail_binpms:
        k.extend_parser(parser)

    try:
        nsargs = parser.parse_args(sys.argv[1:])
    except IOError as err:
        if err.errno == errno.ENOENT:
            print_error(err.strerror + ": " + err.filename)
            return 1
        raise

    if os.getuid() != 0:
        # root access required
        print_error("superuser access required")
        return 1

    # parse spec files
    specs = []
    for spec_f in nsargs.spec:
        spec = SpecParser(spec_f)
        data = spec.parse()
        if data:
            specs.append(data)

    if not specs:
        print_error("invalid spec files provided")
        return 1

    # O(n) determine what is the BinaryPMS to use
    klass = None
    for k in avail_binpms:
        if k.NAME == nsargs.pms:
            klass = k
            break
    if klass is None:
        print_error("invalid Binary PMS specified: %s" % (nsargs.pms, ))
        return 1

    binary_pms = None
    exit_st = 0
    cwd = os.getcwd()
    try:
        try:
            binary_pms = klass(cwd, nsargs)
        except BaseBinaryPMS.BinaryPMSLoadError as err:
            # repository not available or not configured
            print_error("Cannot load Binary Package Manager: %s" % (err, ))
            return 3

        print_info("Loaded Binary PMS: %s" % (klass.NAME, ))

        # validate repository entries of spec metadata
        for spec in specs:
            try:
                binary_pms.validate_spec(spec)
            except BaseBinaryPMS.SpecParserError as err:
                print_error("%s" % (err, ))
                return 1

        if nsargs.blocking:
            print_info("--blocking enabled, please wait for locks...")

        resource_lock = binary_pms.get_resource_lock(nsargs.blocking)
        with resource_lock:
            with MatterResourceLock(nsargs.blocking):
                exit_st = matter_main(binary_pms, nsargs, cwd, specs)

    except BaseBinaryResourceLock.NotAcquired:
        print_error("unable to acquire PMS Resources lock")
        return 42
    except MatterResourceLock.NotAcquired:
        print_error("unable to acquire Matter Resources lock")
        return 42
    except KeyboardInterrupt:
        print_error("Keyboard Interrupt, pid: %s" % (os.getpid(), ))
        return 42
    finally:
        if binary_pms is not None:
            binary_pms.shutdown()

    print_warning("")
    print_warning("")
    print_warning("Tasks complete, exit status: %d" % (exit_st, ))
    return exit_st