Beispiel #1
0
def pip_install_package(source_name,
                        pip_version=None,
                        python_version=None,
                        mode=InstallMode.min_deps,
                        release=False,
                        prefix=None,
                        extra_args=None):
    """Install a pip-compatible python package as a rez package.
    Args:
        source_name (str): Name of package or archive/url containing the pip
            package source. This is the same as the arg you would pass to
            the 'pip install' command.
        pip_version (str or `Version`): Version of pip to use to perform the
            install, uses latest if None.
        python_version (str or `Version`): Python version to use to perform the
            install, and subsequently have the resulting rez package depend on.
        mode (`InstallMode`): Installation mode, determines how dependencies are
            managed.
        release (bool): If True, install as a released package; otherwise, it
            will be installed as a local package.
        extra_args (List[str]): Additional options to the pip install command.

    Returns:
        2-tuple:
            List of `Variant`: Installed variants;
            List of `Variant`: Skipped variants (already installed).
    """
    installed_variants = []
    skipped_variants = []

    py_exe, context = find_pip(pip_version, python_version)
    print_info("Installing %r with pip taken from %r", source_name, py_exe)

    # TODO: should check if packages_path is writable before continuing with pip
    #
    if prefix is not None:
        packages_path = prefix
    else:
        packages_path = (config.release_packages_path
                         if release else config.local_packages_path)

    targetpath = mkdtemp(suffix="-rez", prefix="pip-")

    if context and config.debug("package_release"):
        buf = StringIO()
        print("\n\npackage download environment:", file=buf)
        context.print_info(buf)
        _log(buf.getvalue())

    # Build pip commandline
    cmd = [py_exe, "-m", "pip", "install"]

    _extra_args = extra_args or config.pip_extra_args or []

    if "--no-use-pep517" not in _extra_args:
        cmd.append("--use-pep517")

    if not _option_present(_extra_args, "-t", "--target"):
        cmd.append("--target=%s" % targetpath)

    if mode == InstallMode.no_deps and "--no-deps" not in _extra_args:
        cmd.append("--no-deps")

    cmd.extend(_extra_args)
    cmd.append(source_name)

    # run pip
    #
    # Note: https://github.com/pypa/pip/pull/3934. If/when this PR is merged,
    # it will allow explicit control of where to put bin files.
    #
    _cmd(context=context, command=cmd)

    # determine version of python in use
    if context is None:
        # since we had to use system pip, we have to assume system python version
        py_ver_str = '.'.join(map(str, sys.version_info))
        py_ver = Version(py_ver_str)
    else:
        python_variant = context.get_resolved_package("python")
        py_ver = python_variant.version

    # Collect resulting python packages using distlib
    distribution_path = DistributionPath([targetpath])
    distributions = list(distribution_path.get_distributions())
    dist_names = [x.name for x in distributions]

    def log_append_pkg_variants(pkg_maker):
        template = '{action} [{package.qualified_name}] {package.uri}{suffix}'
        actions_variants = [
            (
                print_info,
                'Installed',
                installed_variants,
                pkg_maker.installed_variants or [],
            ),
            (
                print_debug,
                'Skipped',
                skipped_variants,
                pkg_maker.skipped_variants or [],
            ),
        ]
        for print_, action, variants, pkg_variants in actions_variants:
            for variant in pkg_variants:
                variants.append(variant)
                package = variant.parent
                suffix = (' (%s)' % variant.subpath) if variant.subpath else ''
                print_(template.format(**locals()))

    # get list of package and dependencies
    for distribution in distributions:
        # convert pip requirements into rez requirements
        rez_requires = get_rez_requirements(installed_dist=distribution,
                                            python_version=py_ver,
                                            name_casings=dist_names)

        # log the pip -> rez requirements translation, for debugging
        _log("Pip to rez requirements translation information for " +
             distribution.name_and_version + ":\n" +
             pformat({
                 "pip": {
                     "run_requires": map(str, distribution.run_requires)
                 },
                 "rez": rez_requires
             }))

        # determine where pip files need to be copied into rez package
        src_dst_lut = _get_distribution_files_mapping(distribution, targetpath)

        # build tools list
        tools = []
        for relpath in src_dst_lut.values():
            dir_, filename = os.path.split(relpath)
            if dir_ == "bin":
                tools.append(filename)

        # Sanity warning to see if any files will be copied
        if not src_dst_lut:
            message = 'No source files exist for {}!'
            if not _verbose:
                message += '\nTry again with rez-pip --verbose ...'
            print_warning(message.format(distribution.name_and_version))

        def make_root(variant, path):
            """Using distlib to iterate over all installed files of the current
            distribution to copy files to the target directory of the rez package
            variant
            """
            for rel_src, rel_dest in src_dst_lut.items():
                src = os.path.join(targetpath, rel_src)
                dest = os.path.join(path, rel_dest)

                if not os.path.exists(os.path.dirname(dest)):
                    os.makedirs(os.path.dirname(dest))

                shutil.copyfile(src, dest)

                if _is_exe(src):
                    shutil.copystat(src, dest)

        # create the rez package
        name = pip_to_rez_package_name(distribution.name)
        version = pip_to_rez_version(distribution.version)
        requires = rez_requires["requires"]
        variant_requires = rez_requires["variant_requires"]
        metadata = rez_requires["metadata"]

        with make_package(name, packages_path, make_root=make_root) as pkg:
            # basics (version etc)
            pkg.version = version

            if distribution.metadata.summary:
                pkg.description = distribution.metadata.summary

            # requirements and variants
            if requires:
                pkg.requires = requires

            if variant_requires:
                pkg.variants = [variant_requires]

            # commands
            commands = []
            commands.append("env.PYTHONPATH.append('{root}/python')")

            if tools:
                pkg.tools = tools
                commands.append("env.PATH.append('{root}/bin')")

            pkg.commands = '\n'.join(commands)

            # Make the package use hashed variants. This is required because we
            # can't control what ends up in its variants, and that can easily
            # include problematic chars (>, +, ! etc).
            # TODO: https://github.com/nerdvegas/rez/issues/672
            #
            pkg.hashed_variants = True

            # add some custom attributes to retain pip-related info
            pkg.pip_name = distribution.name_and_version
            pkg.from_pip = True
            pkg.is_pure_python = metadata["is_pure_python"]

            distribution_metadata = distribution.metadata.todict()

            help_ = []

            if "home_page" in distribution_metadata:
                help_.append(["Home Page", distribution_metadata["home_page"]])

            if "download_url" in distribution_metadata:
                help_.append(
                    ["Source Code", distribution_metadata["download_url"]])

            if help_:
                pkg.help = help_

            if "author" in distribution_metadata:
                author = distribution_metadata["author"]

                if "author_email" in distribution_metadata:
                    author += ' ' + distribution_metadata["author_email"]

                pkg.authors = [author]

        log_append_pkg_variants(pkg)

    # cleanup
    shutil.rmtree(targetpath)

    # print summary
    #
    if installed_variants:
        print_info("%d packages were installed.", len(installed_variants))
    else:
        print_warning("NO packages were installed.")
    if skipped_variants:
        print_warning(
            "%d packages were already installed.",
            len(skipped_variants),
        )

    return installed_variants, skipped_variants
Beispiel #2
0
Datei: pip.py Projekt: shiyi9/rez
def pip_install_package(source_name,
                        pip_version=None,
                        python_version=None,
                        mode=InstallMode.min_deps,
                        release=False):
    """Install a pip-compatible python package as a rez package.
    Args:
        source_name (str): Name of package or archive/url containing the pip
            package source. This is the same as the arg you would pass to
            the 'pip install' command.
        pip_version (str or `Version`): Version of pip to use to perform the
            install, uses latest if None.
        python_version (str or `Version`): Python version to use to perform the
            install, and subsequently have the resulting rez package depend on.
        mode (`InstallMode`): Installation mode, determines how dependencies are
            managed.
        release (bool): If True, install as a released package; otherwise, it
            will be installed as a local package.

    Returns:
        2-tuple:
            List of `Variant`: Installed variants;
            List of `Variant`: Skipped variants (already installed).
    """
    installed_variants = []
    skipped_variants = []

    py_exe, context = find_pip(pip_version, python_version)

    # TODO: should check if packages_path is writable before continuing with pip
    #
    packages_path = (config.release_packages_path
                     if release else config.local_packages_path)

    tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
    stagingdir = os.path.join(tmpdir, "rez_staging")
    stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])

    destpath = os.path.join(stagingdir, "python")
    # TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
    binpath = os.path.join(stagingdir, "bin")

    if context and config.debug("package_release"):
        buf = StringIO()
        print("\n\npackage download environment:", file=buf)
        context.print_info(buf)
        _log(buf.getvalue())

    # Build pip commandline
    cmd = [
        py_exe, "-m", "pip", "install", "--use-pep517",
        "--target=%s" % destpath
    ]

    if mode == InstallMode.no_deps:
        cmd.append("--no-deps")
    cmd.append(source_name)

    _cmd(context=context, command=cmd)
    _system = System()

    # determine version of python in use
    if context is None:
        # since we had to use system pip, we have to assume system python version
        py_ver_str = '.'.join(map(str, sys.version_info))
        py_ver = Version(py_ver_str)
    else:
        python_variant = context.get_resolved_package("python")
        py_ver = python_variant.version

    # moving bin folder to expected relative location as per wheel RECORD files
    staged_binpath = os.path.join(destpath, "bin")
    if os.path.isdir(staged_binpath):
        shutil.move(os.path.join(destpath, "bin"), binpath)

    # Collect resulting python packages using distlib
    distribution_path = DistributionPath([destpath])
    distributions = list(distribution_path.get_distributions())
    dist_names = [x.name for x in distributions]

    # get list of package and dependencies
    for distribution in distributions:
        # convert pip requirements into rez requirements
        rez_requires = get_rez_requirements(installed_dist=distribution,
                                            python_version=py_ver,
                                            name_casings=dist_names)

        # log the pip -> rez translation, for debugging
        _log("Pip to rez translation information for " +
             distribution.name_and_version + ":\n" +
             pformat({
                 "pip": {
                     "run_requires": map(str, distribution.run_requires)
                 },
                 "rez": rez_requires
             }))

        # iterate over installed files and determine dest filepaths
        tools = []
        src_dst_lut = {}

        for installed_file in distribution.list_installed_files():
            # distlib expects the script files to be located in ../../bin/
            # when in fact ../bin seems to be the resulting path after the
            # installation as such we need to point the bin files to the
            # expected location to match wheel RECORD files
            installed_filepath = os.path.normpath(installed_file[0])
            bin_prefix = os.path.join('..', '..', 'bin') + os.sep

            if installed_filepath.startswith(bin_prefix):
                # account for extra parentdir as explained above
                installed = os.path.join(destpath, '_', installed_filepath)
            else:
                installed = os.path.join(destpath, installed_filepath)

            source_file = os.path.normpath(installed)

            if os.path.exists(source_file):
                destination_file = os.path.relpath(source_file, stagingdir)
                exe = False

                if is_exe(source_file) and destination_file.startswith("bin" +
                                                                       os.sep):
                    _file = os.path.basename(destination_file)
                    tools.append(_file)
                    exe = True

                src_dst_lut[source_file] = [destination_file, exe]
            else:
                _log("Source file does not exist: " + source_file + "!")

        def make_root(variant, path):
            """Using distlib to iterate over all installed files of the current
            distribution to copy files to the target directory of the rez package
            variant
            """
            for source_file, data in src_dst_lut.items():
                destination_file, exe = data
                destination_file = os.path.normpath(
                    os.path.join(path, destination_file))

                if not os.path.exists(os.path.dirname(destination_file)):
                    os.makedirs(os.path.dirname(destination_file))

                shutil.copyfile(source_file, destination_file)
                if exe:
                    shutil.copystat(source_file, destination_file)

        # create the rez package
        name = pip_to_rez_package_name(distribution.name)
        version = pip_to_rez_version(distribution.version)
        requires = rez_requires["requires"]
        variant_requires = rez_requires["variant_requires"]
        metadata = rez_requires["metadata"]

        with make_package(name, packages_path, make_root=make_root) as pkg:
            # basics (version etc)
            pkg.version = version

            if distribution.metadata.summary:
                pkg.description = distribution.metadata.summary

            # requirements and variants
            if requires:
                pkg.requires = requires

            if variant_requires:
                pkg.variants = [variant_requires]

            # commands
            commands = []
            commands.append("env.PYTHONPATH.append('{root}/python')")

            if tools:
                pkg.tools = tools
                commands.append("env.PATH.append('{root}/bin')")

            pkg.commands = '\n'.join(commands)

            # Make the package use hashed variants. This is required because we
            # can't control what ends up in its variants, and that can easily
            # include problematic chars (>, +, ! etc).
            # TODO: https://github.com/nerdvegas/rez/issues/672
            #
            pkg.hashed_variants = True

            # add some custom attributes to retain pip-related info
            pkg.pip_name = distribution.name_and_version
            pkg.from_pip = True
            pkg.is_pure_python = metadata["is_pure_python"]

        installed_variants.extend(pkg.installed_variants or [])
        skipped_variants.extend(pkg.skipped_variants or [])

    # cleanup
    shutil.rmtree(tmpdir)

    return installed_variants, skipped_variants