Example #1
0
    def visit_variants(self, func, variants=None, **kwargs):
        """Iterate over variants and call a function on each."""
        if variants:
            present_variants = range(self.package.num_variants)
            invalid_variants = set(variants) - set(present_variants)
            if invalid_variants:
                raise BuildError(
                    "The package does not contain the variants: %s" %
                    ", ".join(str(x) for x in sorted(invalid_variants)))

        # iterate over variants
        results = []
        num_visited = 0

        for variant in self.package.iter_variants():
            if variants and variant.index not in variants:
                self._print_header("Skipping variant %s (%s)..." %
                                   (variant.index, self._n_of_m(variant)))
                continue

            # visit the variant
            result = func(variant, **kwargs)
            results.append(result)
            num_visited += 1

        return num_visited, results
Example #2
0
    def _build_variant_base(self,
                            variant,
                            build_type,
                            install_path=None,
                            clean=False,
                            install=False,
                            **kwargs):
        # create build/install paths
        install_path = install_path or self.package.config.local_packages_path
        variant_install_path = self.get_package_install_path(install_path)
        variant_build_path = self.build_path

        if variant.subpath:
            variant_build_path = os.path.join(variant_build_path,
                                              variant.subpath)
            variant_install_path = os.path.join(variant_install_path,
                                                variant.subpath)

        # create directories (build, install)
        if clean and os.path.exists(variant_build_path):
            shutil.rmtree(variant_build_path)

        safe_makedirs(variant_build_path)

        if install:
            # inform package repo that a variant is about to be built/installed
            pkg_repo = package_repository_manager.get_repository(install_path)
            pkg_repo.pre_variant_install(variant.resource)

            if not os.path.exists(variant_install_path):
                safe_makedirs(variant_install_path)

        # create build environment
        context, rxt_filepath = self.create_build_context(
            variant=variant,
            build_type=build_type,
            build_path=variant_build_path)

        # run build system
        build_system_name = self.build_system.name()
        self._print("\nInvoking %s build system...", build_system_name)

        build_result = self.build_system.build(
            context=context,
            variant=variant,
            build_path=variant_build_path,
            install_path=variant_install_path,
            install=install,
            build_type=build_type)

        if not build_result.get("success"):
            raise BuildError("The %s build system failed." % build_system_name)

        if install:
            # install some files for debugging purposes
            extra_files = build_result.get("extra_files", []) + [rxt_filepath]
            for file_ in extra_files:
                copy_or_replace(file_, variant_install_path)

        return build_result
Example #3
0
    def _build_variant_base(self,
                            variant,
                            build_type,
                            install_path=None,
                            clean=False,
                            install=False,
                            **kwargs):
        # create build/install paths
        install_path = install_path or self.package.config.local_packages_path
        variant_install_path = self.get_package_install_path(install_path)
        variant_build_path = self.build_path
        if variant.subpath:
            variant_build_path = os.path.join(variant_build_path,
                                              variant.subpath)
            variant_install_path = os.path.join(variant_install_path,
                                                variant.subpath)

        if clean and os.path.exists(variant_build_path):
            shutil.rmtree(variant_build_path)
        if not os.path.exists(variant_build_path):
            os.makedirs(variant_build_path)

        if install and not os.path.exists(variant_install_path):
            os.makedirs(variant_install_path)

        # create build environment
        context, rxt_filepath = self.create_build_context(
            variant=variant,
            build_type=build_type,
            build_path=variant_build_path)

        # run build system
        build_system_name = self.build_system.name()
        self._print("\nInvoking %s build system...", build_system_name)
        build_result = self.build_system.build(
            context=context,
            variant=variant,
            build_path=variant_build_path,
            install_path=variant_install_path,
            install=install,
            build_type=build_type)

        if not build_result.get("success"):
            raise BuildError("The %s build system failed" % build_system_name)

        if install:
            # install some files for debugging purposes
            extra_files = build_result.get("extra_files", []) + [rxt_filepath]
            for file_ in extra_files:
                shutil.copy(file_, variant_install_path)

        return build_result
Example #4
0
def _cmd(context, command):
    cmd_str = ' '.join(quote(x) for x in command)
    _log("running: %s" % cmd_str)

    if context is None:
        p = popen(command)
    else:
        p = context.execute_shell(command=command, block=False)

    p.wait()

    if p.returncode:
        raise BuildError("Failed to download source with pip: %s" % cmd_str)
Example #5
0
def find_pip_from_context(python_version, pip_version=None):
    """Find pip from rez context.

    Args:
        python_version (str or `Version`): Python version to use
        pip_version (str or `Version`): Version of pip to use, or latest.

    Returns:
        3-tuple:
        - str: Python executable or None if we fell back to system pip.
        - str: Pip version or None if we fell back to system pip.
        - `ResolvedContext`: Context containing pip, or None if we fell back
          to system pip.
    """
    target = "python"
    package_request = []

    if python_version:
        ver = Version(str(python_version))
        python_major_minor_ver = ver.trim(2)
    else:
        # use latest major.minor
        package = get_latest_package("python")
        if package:
            python_major_minor_ver = package.version.trim(2)
        else:
            raise BuildError("Found no python package.")

    python_package = "python-%s" % str(python_major_minor_ver)

    package_request.append(python_package)

    if pip_version:
        target = "pip"
        if pip_version == "latest":
            package_request.append("pip")
        else:
            package_request.append("pip-%s" % str(pip_version))

    print_info("Trying to use pip from %s package", target)

    try:
        context = ResolvedContext(package_request)
    except (PackageFamilyNotFoundError, PackageNotFoundError):
        print_debug("No rez package called %s found", target)
        return None, None, None

    py_exe_name = "python"
    if platform_.name != "windows":
        # Python < 2 on Windows doesn't have versionned executable.
        py_exe_name += str(python_major_minor_ver.trim(1))

    py_exe = context.which(py_exe_name)

    proc = context.execute_command(
        # -E and -s are used to isolate the environment as much as possible.
        # See python --help for more details. We absolutely don't want to get
        # pip from the user home.
        [
            py_exe, "-E", "-s", "-c",
            "import pip, sys; sys.stdout.write(pip.__version__)"
        ],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True)
    out, err = proc.communicate()
    if proc.returncode:
        print_debug("Failed to get pip from package %s", target)
        print_debug(out)
        print_debug(err)
        return None, None, None

    pip_version = out.strip()

    variant = context.get_resolved_package(target)
    package = variant.parent
    print_info("Found pip-%s inside %s. Will use it with %s", pip_version,
               package.uri, py_exe)

    return py_exe, pip_version, context
Example #6
0
    def _build_variant_base(self,
                            variant,
                            build_type,
                            install_path=None,
                            clean=False,
                            install=False,
                            **kwargs):
        # create build/install paths
        install_path = install_path or self.package.config.local_packages_path
        package_install_path = self.get_package_install_path(install_path)
        variant_build_path = self.build_path

        if variant.index is None:
            variant_install_path = package_install_path
        else:
            subpath = variant._non_shortlinked_subpath
            variant_build_path = os.path.join(variant_build_path, subpath)
            variant_install_path = os.path.join(package_install_path, subpath)

        # create directories (build, install)
        if clean and os.path.exists(variant_build_path):
            self._rmtree(variant_build_path)

        safe_makedirs(variant_build_path)

        # find last dir of installation path that exists, and possibly make it
        # writable during variant installation
        #
        last_dir = get_existing_path(variant_install_path,
                                     topmost_path=install_path)
        if last_dir:
            ctxt = make_path_writable(last_dir)
        else:
            ctxt = with_noop()

        with ctxt:
            if install:
                # inform package repo that a variant is about to be built/installed
                pkg_repo = package_repository_manager.get_repository(
                    install_path)
                pkg_repo.pre_variant_install(variant.resource)

                if not os.path.exists(variant_install_path):
                    safe_makedirs(variant_install_path)

                # if hashed variants are enabled, create the variant shortlink
                if variant.parent.hashed_variants:
                    try:
                        # create the dir containing all shortlinks
                        base_shortlinks_path = os.path.join(
                            package_install_path,
                            variant.parent.config.variant_shortlinks_dirname)

                        safe_makedirs(base_shortlinks_path)

                        # create the shortlink
                        rel_variant_path = os.path.relpath(
                            variant_install_path, base_shortlinks_path)
                        create_unique_base26_symlink(base_shortlinks_path,
                                                     rel_variant_path)

                    except Exception as e:
                        # Treat any error as warning - lack of shortlink is not
                        # a breaking issue, it just means the variant root path
                        # will be long.
                        #
                        print_warning(
                            "Error creating variant shortlink for %s: %s: %s",
                            variant_install_path, e.__class__.__name__, e)

            # Re-evaluate the variant, so that variables such as 'building' and
            # 'build_variant_index' are set, and any early-bound package attribs
            # are re-evaluated wrt these vars. This is done so that attribs such as
            # 'requires' can change depending on whether a build is occurring or not.
            #
            # Note that this re-evaluated variant is ONLY used here, for the purposes
            # of creating the build context. The variant that is actually installed
            # is the one evaluated where 'building' is False.
            #
            re_evaluated_package = variant.parent.get_reevaluated({
                "building":
                True,
                "build_variant_index":
                variant.index or 0,
                "build_variant_requires":
                variant.variant_requires
            })
            re_evaluated_variant = re_evaluated_package.get_variant(
                variant.index)

            # create build environment (also creates build.rxt file)
            context, rxt_filepath = self.create_build_context(
                variant=re_evaluated_variant,
                build_type=build_type,
                build_path=variant_build_path)

            # list of extra files (build.rxt etc) that are installed if an
            # installation is taking place
            #
            extra_install_files = [rxt_filepath]

            # create variant.json file. This identifies which variant this is.
            # This is important for hashed variants, where it is not obvious
            # which variant is in which root path. The file is there for
            # debugging purposes only.
            #
            if variant.index is not None:
                data = {
                    "index": variant.index,
                    "data": variant.parent.data["variants"][variant.index]
                }

                filepath = os.path.join(variant_build_path, "variant.json")
                extra_install_files.append(filepath)

                with open(filepath, 'w') as f:
                    json.dump(data, f, indent=2)

            # run build system
            build_system_name = self.build_system.name()
            self._print("\nInvoking %s build system...", build_system_name)

            build_result = self.build_system.build(
                context=context,
                variant=variant,
                build_path=variant_build_path,
                install_path=variant_install_path,
                install=install,
                build_type=build_type)

            if not build_result.get("success"):
                # delete the possibly partially installed variant payload
                if install:
                    self._rmtree(variant_install_path)

                raise BuildError("The %s build system failed." %
                                 build_system_name)

            if install:
                # add some installation details to build result
                build_result.update({
                    "package_install_path":
                    package_install_path,
                    "variant_install_path":
                    variant_install_path
                })

                # the build system can also specify extra files that need to
                # be installed
                filepaths = build_result.get("extra_files")
                if filepaths:
                    extra_install_files.extend(filepaths)

                # install extra files
                for file_ in extra_install_files:
                    copy_or_replace(file_, variant_install_path)

                # Install include modules. Note that this doesn't need to be done
                # multiple times, but for subsequent variants it has no effect.
                #
                self._install_include_modules(install_path)

            return build_result
Example #7
0
    def _build_variant_base(self, variant, build_type, install_path=None,
                            clean=False, install=False, **kwargs):
        # create build/install paths
        install_path = install_path or self.package.config.local_packages_path
        variant_install_path = self.get_package_install_path(install_path)
        variant_build_path = self.build_path

        if variant.subpath:
            variant_build_path = os.path.join(variant_build_path, variant.subpath)
            variant_install_path = os.path.join(variant_install_path, variant.subpath)

        # create directories (build, install)
        if clean and os.path.exists(variant_build_path):
            shutil.rmtree(variant_build_path)

        safe_makedirs(variant_build_path)

        # find last dir of installation path that exists, and possibly make it
        # writable during variant installation
        #
        last_dir = get_existing_path(variant_install_path,
                                     topmost_path=install_path)
        if last_dir:
            ctxt = make_path_writable(last_dir)
        else:
            ctxt = with_noop()

        with ctxt:
            if install:
                # inform package repo that a variant is about to be built/installed
                pkg_repo = package_repository_manager.get_repository(install_path)
                pkg_repo.pre_variant_install(variant.resource)

                if not os.path.exists(variant_install_path):
                    safe_makedirs(variant_install_path)

            # Re-evaluate the variant, so that variables such as 'building' and
            # 'build_variant_index' are set, and any early-bound package attribs
            # are re-evaluated wrt these vars. This is done so that attribs such as
            # 'requires' can change depending on whether a build is occurring or not.
            #
            # Note that this re-evaluated variant is ONLY used here, for the purposes
            # of creating the build context. The variant that is actually installed
            # is the one evaluated where 'building' is False.
            #
            re_evaluated_package = variant.parent.get_reevaluated({
                "building": True,
                "build_variant_index": variant.index or 0,
                "build_variant_requires": variant.variant_requires
            })
            re_evaluated_variant = re_evaluated_package.get_variant(variant.index)

            # create build environment
            context, rxt_filepath = self.create_build_context(
                variant=re_evaluated_variant,
                build_type=build_type,
                build_path=variant_build_path)

            # run build system
            build_system_name = self.build_system.name()
            self._print("\nInvoking %s build system...", build_system_name)

            build_result = self.build_system.build(
                context=context,
                variant=variant,
                build_path=variant_build_path,
                install_path=variant_install_path,
                install=install,
                build_type=build_type)

            if not build_result.get("success"):
                raise BuildError("The %s build system failed." % build_system_name)

            if install:
                # install some files for debugging purposes
                extra_files = build_result.get("extra_files", [])
                if rxt_filepath:
                    extra_files = extra_files + [rxt_filepath]

                for file_ in extra_files:
                    copy_or_replace(file_, variant_install_path)

                # Install include modules. Note that this doesn't need to be done
                # multiple times, but for subsequent variants it has no effect.
                #
                self._install_include_modules(install_path)

            return build_result