def _get_dependencies(requirement, distributions): def get_distrubution_name(pip_name): pip_to_rez_name = pip_name.lower().replace("-", "_") for dist in distributions: _name, _ = parse_name_and_version(dist.name_and_version) if _name.replace("-", "_") == pip_to_rez_name: return dist.name.replace("-", "_") result = [] requirements = [requirement] if isinstance(requirement, basestring) else requirement["requires"] for package in requirements: if "(" in package: try: name, version = parse_name_and_version(package) version = version.replace("==", "") name = get_distrubution_name(name) except DistlibException: n, vs = package.split(" (") vs = vs[:-1] versions = [] for v in vs.split(","): package = "%s (%s)" % (n, v) name, version = parse_name_and_version(package) version = version.replace("==", "") versions.append(version) version = "".join(versions) name = get_distrubution_name(name) result.append("-".join([name, version])) else: name = get_distrubution_name(package) result.append(name) return result
def get_distrubution_name(pip_name): pip_to_rez_name = pip_name.lower().replace("-", "_") for dist in distributions: _name, _ = parse_name_and_version(dist.name_and_version) if _name.replace("-", "_") == pip_to_rez_name: return dist.name.replace("-", "_") return pip_to_rez_name
def _get_dependencies(requirement, distributions): def get_distribution_name(pip_name): pip_to_rez_name = pip_name.lower().replace("-", "_") for dist in distributions: _name, _ = parse_name_and_version(dist.name_and_version) if _name.replace("-", "_") == pip_to_rez_name: return dist.name.replace("-", "_") result = [] requirements = ([requirement] if isinstance(requirement, basestring) else requirement["requires"]) for package in requirements: if "(" in package: try: name, version = parse_name_and_version(package) version = version.replace("==", "") name = get_distribution_name(name) except DistlibException: # check if package contains extraneous environment info and remove it # see environment markers: https://www.python.org/dev/peps/pep-0508/#environment-markers if ";" in package: package = package.split(";")[0].strip() n, vs = package.split(' (') vs = vs[:-1] versions = [] for v in vs.split(','): package = "%s (%s)" % (n, v) name, version = parse_name_and_version(package) version = version.replace("==", "") versions.append(version) version = "".join(versions) name = get_distribution_name(name) result.append("-".join([name, version])) else: name = get_distribution_name(package) result.append(name) return result
def _get_dependencies(requirement, distributions): def get_distribution_name(pip_name): pip_to_rez_name = pip_name.lower().replace("-", "_") for dist in distributions: _name, _ = parse_name_and_version(dist.name_and_version) if _name.replace("-", "_") == pip_to_rez_name: return dist.name.replace("-", "_") result = [] requirements = ([requirement] if isinstance(requirement, basestring) else requirement["requires"]) for package in requirements: if "(" in package: try: name, version = parse_name_and_version(package) version = version.replace("==", "") name = get_distribution_name(name) except DistlibException: n, vs = package.split(' (') vs = vs[:-1] versions = [] for v in vs.split(','): package = "%s (%s)" % (n, v) name, version = parse_name_and_version(package) version = version.replace("==", "") versions.append(version) version = "".join(versions) name = get_distribution_name(name) result.append("-".join([name, version])) else: name = get_distribution_name(package) result.append(name) return result
def pip_to_rez_package_name(distribution): """Convert a distribution name to a rez compatible name. The rez package name can't be simply set to the dist name, because some pip packages have hyphen in the name. In rez this is not a valid package name (it would be interpreted as the start of the version). Example: my-pkg-1.2 is 'my', version 'pkg-1.2'. Arguments: distribution (Distribution): The distribution whose name to convert. """ name, _ = parse_name_and_version(distribution.name_and_version) name = distribution.name[0:len(name)].replace("-", "_") return name
def get_distribution_name(pip_name): pip_to_rez_name = pip_name.lower().replace("-", "_") for dist in distributions: _name, _ = parse_name_and_version(dist.name_and_version) if _name.replace("-", "_") == pip_to_rez_name: return dist.name.replace("-", "_")
def pip_install_package(source_name, python_version=None, release=False, no_deps=False, prefix=None, auto_variants=True, variants=None): """Install a pip-compatible python package as a rez package. Args: source_name (str): Name of package or archive/url containing the pip package source. This is the same as the arg you would pass to the 'pip install' command. python_version (str or `Version`): Python version to use to perform the install, and subsequently have the resulting rez package depend on. prefix (str, optional): Override install location to here, similar to `rez build --prefix` no_deps (bool, optional): The `pip --no-deps` argument auto_variants (bool, optional): Compute variants from the PyPI classifiers portion of setup() release (bool): If True, install as a released package; otherwise, it will be installed as a local package. prefix (str, optional): Override release path with this absolute path Returns: 2-tuple: List of `Variant`: Installed variants; List of `Variant`: Skipped variants (already installed). """ installed_variants = [] skipped_variants = [] if prefix is not None: config.release_packages_path = prefix # TODO: should check if packages_path is writable # before continuing with pip # packages_path = (config.release_packages_path if release else config.local_packages_path) tmpdir = mkdtemp(suffix="-rez", prefix="pip-") stagingdir = os.path.join(tmpdir, "rez_staging") stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep]) destpath = os.path.join(stagingdir, "python") python_exe, context = find_python(python_version) if context and config.debug("package_release"): buf = StringIO() print >> buf, "\n\npackage download environment:" context.print_info(buf) _log(buf.getvalue()) # Build pip commandline cmd = [ python_exe, "-m", "pip", "install", "--target", destpath, # Only ever consider wheels, anything else is ancient "--use-pep517", # Handle case where the Python distribution used alongside # pip already has a package installed in its `site-packages/` dir. "--ignore-installed", ] if no_deps: # Delegate the installation of dependencies to the user # This is important, as each dependency may have different # requirements of its own, and variants to go with it. cmd.append("--no-deps") cmd.append(source_name) _cmd(context=context, command=cmd) # Collect resulting python packages using distlib distribution_path = DistributionPath([destpath]) distributions = [d for d in distribution_path.get_distributions()] for distribution in distribution_path.get_distributions(): requirements = [] if distribution.metadata.run_requires: # Handle requirements. Currently handles # conditional environment based # requirements and normal requirements # TODO: Handle optional requirements? for requirement in distribution.metadata.run_requires: if "environment" in requirement: if interpret(requirement["environment"]): requirements.extend(_get_dependencies( requirement, distributions)) elif "extra" in requirement: # Currently ignoring optional requirements pass else: requirements.extend(_get_dependencies( requirement, distributions)) tools = [] src_dst_lut = {} files = distribution.list_installed_files() for installed_file in files: source_file = os.path.join(destpath, installed_file[0]) source_file = os.path.normpath(source_file) if os.path.exists(source_file): destination_file = source_file.split(stagingsep)[1] exe = False starts_with_bin = destination_file.startswith( "%s%s" % ("bin", os.path.sep) ) if is_exe(source_file) and starts_with_bin: _, _file = os.path.split(destination_file) tools.append(_file) exe = True data = [destination_file, exe] src_dst_lut[source_file] = data else: _log("Source file does not exist: " + source_file + "!") def make_root(variant, path): """Using distlib to iterate over all installed files of the current distribution to copy files to the target directory of the rez package variant """ for source_file, data in src_dst_lut.items(): destination_file, exe = data destination_file = os.path.join(path, destination_file) destination_file = os.path.normpath(destination_file) if not os.path.exists(os.path.dirname(destination_file)): os.makedirs(os.path.dirname(destination_file)) shutil.copyfile(source_file, destination_file) if exe: shutil.copystat(source_file, destination_file) name, _ = parse_name_and_version(distribution.name_and_version) name = distribution.name[0:len(name)].replace("-", "_") # determine variant requirements variants_ = variants or [] if (not variants_) and auto_variants: variants_.extend(wheel_to_variants(distribution)) if variants_: print_info("'%s' - Automatically detected variants: %s" % ( name, ", ".join(variants_)) ) with make_package(name, packages_path, make_root=make_root) as pkg: pkg.version = distribution.version if distribution.metadata.summary: pkg.description = distribution.metadata.summary if variants_: pkg.variants = [variants_] if requirements: pkg.requires = requirements commands = [] commands.append("env.PYTHONPATH.append('{root}/python')") if tools: pkg.tools = tools commands.append("env.PATH.append('{root}/bin')") pkg.commands = '\n'.join(commands) installed_variants.extend(pkg.installed_variants or []) skipped_variants.extend(pkg.skipped_variants or []) # cleanup shutil.rmtree(tmpdir) return installed_variants, skipped_variants
def pip_install_package(source_name, pip_version=None, python_version=None, mode=InstallMode.min_deps, release=False): """Install a pip-compatible python package as a rez package. Args: source_name (str): Name of package or archive/url containing the pip package source. This is the same as the arg you would pass to the 'pip install' command. pip_version (str or `Version`): Version of pip to use to perform the install, uses latest if None. python_version (str or `Version`): Python version to use to perform the install, and subsequently have the resulting rez package depend on. mode (`InstallMode`): Installation mode, determines how dependencies are managed. release (bool): If True, install as a released package; otherwise, it will be installed as a local package. Returns: 2-tuple: List of `Variant`: Installed variants; List of `Variant`: Skipped variants (already installed). """ installed_variants = [] skipped_variants = [] pip_exe, context = find_pip(pip_version, python_version) # TODO: should check if packages_path is writable before continuing with pip # packages_path = (config.release_packages_path if release else config.local_packages_path) tmpdir = mkdtemp(suffix="-rez", prefix="pip-") stagingdir = os.path.join(tmpdir, "rez_staging") stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep]) destpath = os.path.join(stagingdir, "python") binpath = os.path.join(stagingdir, "bin") incpath = os.path.join(stagingdir, "include") datapath = stagingdir if context and config.debug("package_release"): buf = StringIO() print >> buf, "\n\npackage download environment:" context.print_info(buf) _log(buf.getvalue()) # Build pip commandline cmd = [pip_exe, "install", "--install-option=--install-lib=%s" % destpath, "--install-option=--install-scripts=%s" % binpath, "--install-option=--install-headers=%s" % incpath, "--install-option=--install-data=%s" % datapath] if mode == InstallMode.no_deps: cmd.append("--no-deps") cmd.append(source_name) _cmd(context=context, command=cmd) _system = System() # Collect resulting python packages using distlib distribution_path = DistributionPath([destpath], include_egg=True) distributions = [d for d in distribution_path.get_distributions()] for distribution in distribution_path.get_distributions(): requirements = [] if distribution.metadata.run_requires: # Handle requirements. Currently handles conditional environment based # requirements and normal requirements # TODO: Handle optional requirements? for requirement in distribution.metadata.run_requires: if "environment" in requirement: if interpret(requirement["environment"]): requirements.extend(_get_dependencies(requirement, distributions)) elif "extra" in requirement: # Currently ignoring optional requirements pass else: requirements.extend(_get_dependencies(requirement, distributions)) tools = [] src_dst_lut = {} for installed_file in distribution.list_installed_files(allow_fail=True): source_file = os.path.normpath(os.path.join(destpath, installed_file[0])) if os.path.exists(source_file): destination_file = installed_file[0].split(stagingsep)[1] exe = False if is_exe(source_file) and \ destination_file.startswith("%s%s" % ("bin", os.path.sep)): _, _file = os.path.split(destination_file) tools.append(_file) exe = True data = [destination_file, exe] src_dst_lut[source_file] = data else: _log("Source file does not exist: " + source_file + "!") def make_root(variant, path): """Using distlib to iterate over all installed files of the current distribution to copy files to the target directory of the rez package variant """ for source_file, data in src_dst_lut.items(): destination_file, exe = data destination_file = os.path.normpath(os.path.join(path, destination_file)) if not os.path.exists(os.path.dirname(destination_file)): os.makedirs(os.path.dirname(destination_file)) shutil.copyfile(source_file, destination_file) if exe: shutil.copystat(source_file, destination_file) # determine variant requirements # TODO detect if platform/arch/os necessary, no if pure python variant_reqs = [] variant_reqs.append("platform-%s" % _system.platform) variant_reqs.append("arch-%s" % _system.arch) variant_reqs.append("os-%s" % _system.os) if context is None: # since we had to use system pip, we have to assume system python version py_ver = '.'.join(map(str, sys.version_info[:2])) else: python_variant = context.get_resolved_package("python") py_ver = python_variant.version.trim(2) variant_reqs.append("python-%s" % py_ver) name, _ = parse_name_and_version(distribution.name_and_version) name = distribution.name[0:len(name)].replace("-", "_") with make_package(name, packages_path, make_root=make_root) as pkg: pkg.version = distribution.version if distribution.metadata.summary: pkg.description = distribution.metadata.summary pkg.variants = [variant_reqs] if requirements: pkg.requires = requirements commands = [] commands.append("env.PYTHONPATH.append('{root}/python')") if tools: pkg.tools = tools commands.append("env.PATH.append('{root}/bin')") pkg.commands = '\n'.join(commands) installed_variants.extend(pkg.installed_variants or []) skipped_variants.extend(pkg.skipped_variants or []) # cleanup shutil.rmtree(tmpdir) return installed_variants, skipped_variants