def _get_dependencies(requirement, distributions):
def get_distrubution_name(pip_name):
pip_to_rez_name = pip_name.lower().replace("-", "_")
for dist in distributions:
_name, _ = parse_name_and_version(dist.name_and_version)
if _name.replace("-", "_") == pip_to_rez_name:
return dist.name.replace("-", "_")
result = []
requirements = [requirement] if isinstance(requirement, basestring) else requirement["requires"]
for package in requirements:
if "(" in package:
try:
name, version = parse_name_and_version(package)
version = version.replace("==", "")
name = get_distrubution_name(name)
except DistlibException:
n, vs = package.split(" (")
vs = vs[:-1]
versions = []
for v in vs.split(","):
package = "%s (%s)" % (n, v)
name, version = parse_name_and_version(package)
version = version.replace("==", "")
versions.append(version)
version = "".join(versions)
name = get_distrubution_name(name)
result.append("-".join([name, version]))
else:
name = get_distrubution_name(package)
result.append(name)
return result
def get_distrubution_name(pip_name):
pip_to_rez_name = pip_name.lower().replace("-", "_")
for dist in distributions:
_name, _ = parse_name_and_version(dist.name_and_version)
if _name.replace("-", "_") == pip_to_rez_name:
return dist.name.replace("-", "_")
return pip_to_rez_name
def _get_dependencies(requirement, distributions):
def get_distribution_name(pip_name):
pip_to_rez_name = pip_name.lower().replace("-", "_")
for dist in distributions:
_name, _ = parse_name_and_version(dist.name_and_version)
if _name.replace("-", "_") == pip_to_rez_name:
return dist.name.replace("-", "_")
result = []
requirements = ([requirement] if isinstance(requirement, basestring) else
requirement["requires"])
for package in requirements:
if "(" in package:
try:
name, version = parse_name_and_version(package)
version = version.replace("==", "")
name = get_distribution_name(name)
except DistlibException:
# check if package contains extraneous environment info and remove it
# see environment markers: https://www.python.org/dev/peps/pep-0508/#environment-markers
if ";" in package:
package = package.split(";")[0].strip()
n, vs = package.split(' (')
vs = vs[:-1]
versions = []
for v in vs.split(','):
package = "%s (%s)" % (n, v)
name, version = parse_name_and_version(package)
version = version.replace("==", "")
versions.append(version)
version = "".join(versions)
name = get_distribution_name(name)
result.append("-".join([name, version]))
else:
name = get_distribution_name(package)
result.append(name)
return result
def _get_dependencies(requirement, distributions):
def get_distribution_name(pip_name):
pip_to_rez_name = pip_name.lower().replace("-", "_")
for dist in distributions:
_name, _ = parse_name_and_version(dist.name_and_version)
if _name.replace("-", "_") == pip_to_rez_name:
return dist.name.replace("-", "_")
result = []
requirements = ([requirement] if isinstance(requirement, basestring) else
requirement["requires"])
for package in requirements:
if "(" in package:
try:
name, version = parse_name_and_version(package)
version = version.replace("==", "")
name = get_distribution_name(name)
except DistlibException:
n, vs = package.split(' (')
vs = vs[:-1]
versions = []
for v in vs.split(','):
package = "%s (%s)" % (n, v)
name, version = parse_name_and_version(package)
version = version.replace("==", "")
versions.append(version)
version = "".join(versions)
name = get_distribution_name(name)
result.append("-".join([name, version]))
else:
name = get_distribution_name(package)
result.append(name)
return result
def pip_to_rez_package_name(distribution):
"""Convert a distribution name to a rez compatible name.
The rez package name can't be simply set to the dist name, because some
pip packages have hyphen in the name. In rez this is not a valid package
name (it would be interpreted as the start of the version).
Example: my-pkg-1.2 is 'my', version 'pkg-1.2'.
Arguments:
distribution (Distribution): The distribution whose name to convert.
"""
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
return name
def get_distribution_name(pip_name):
pip_to_rez_name = pip_name.lower().replace("-", "_")
for dist in distributions:
_name, _ = parse_name_and_version(dist.name_and_version)
if _name.replace("-", "_") == pip_to_rez_name:
return dist.name.replace("-", "_")
def pip_install_package(source_name, python_version=None,
release=False, no_deps=False,
prefix=None, auto_variants=True,
variants=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
prefix (str, optional): Override install location to here,
similar to `rez build --prefix`
no_deps (bool, optional): The `pip --no-deps` argument
auto_variants (bool, optional): Compute variants from the PyPI
classifiers portion of setup()
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
prefix (str, optional): Override release path with this absolute path
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
if prefix is not None:
config.release_packages_path = prefix
# TODO: should check if packages_path is writable
# before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
python_exe, context = find_python(python_version)
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [
python_exe, "-m", "pip", "install",
"--target", destpath,
# Only ever consider wheels, anything else is ancient
"--use-pep517",
# Handle case where the Python distribution used alongside
# pip already has a package installed in its `site-packages/` dir.
"--ignore-installed",
]
if no_deps:
# Delegate the installation of dependencies to the user
# This is important, as each dependency may have different
# requirements of its own, and variants to go with it.
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles
# conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(
requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(
requirement, distributions))
tools = []
src_dst_lut = {}
files = distribution.list_installed_files()
for installed_file in files:
source_file = os.path.join(destpath, installed_file[0])
source_file = os.path.normpath(source_file)
if os.path.exists(source_file):
destination_file = source_file.split(stagingsep)[1]
exe = False
starts_with_bin = destination_file.startswith(
"%s%s" % ("bin", os.path.sep)
)
if is_exe(source_file) and starts_with_bin:
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez
package variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.join(path, destination_file)
destination_file = os.path.normpath(destination_file)
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
# determine variant requirements
variants_ = variants or []
if (not variants_) and auto_variants:
variants_.extend(wheel_to_variants(distribution))
if variants_:
print_info("'%s' - Automatically detected variants: %s" % (
name, ", ".join(variants_))
)
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
if variants_:
pkg.variants = [variants_]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def pip_install_package(source_name, pip_version=None, python_version=None,
mode=InstallMode.min_deps, release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
pip_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
binpath = os.path.join(stagingdir, "bin")
incpath = os.path.join(stagingdir, "include")
datapath = stagingdir
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [pip_exe, "install",
"--install-option=--install-lib=%s" % destpath,
"--install-option=--install-scripts=%s" % binpath,
"--install-option=--install-headers=%s" % incpath,
"--install-option=--install-data=%s" % datapath]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath], include_egg=True)
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(requirement, distributions))
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files(allow_fail=True):
source_file = os.path.normpath(os.path.join(destpath, installed_file[0]))
if os.path.exists(source_file):
destination_file = installed_file[0].split(stagingsep)[1]
exe = False
if is_exe(source_file) and \
destination_file.startswith("%s%s" % ("bin", os.path.sep)):
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# determine variant requirements
# TODO detect if platform/arch/os necessary, no if pure python
variant_reqs = []
variant_reqs.append("platform-%s" % _system.platform)
variant_reqs.append("arch-%s" % _system.arch)
variant_reqs.append("os-%s" % _system.os)
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver = '.'.join(map(str, sys.version_info[:2]))
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version.trim(2)
variant_reqs.append("python-%s" % py_ver)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
pkg.variants = [variant_reqs]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
