def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False,
prefix=None,
extra_args=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
extra_args (List[str]): Additional options to the pip install command.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
py_exe, context = find_pip(pip_version, python_version)
print_info("Installing %r with pip taken from %r", source_name, py_exe)
# TODO: should check if packages_path is writable before continuing with pip
#
if prefix is not None:
packages_path = prefix
else:
packages_path = (config.release_packages_path
if release else config.local_packages_path)
targetpath = mkdtemp(suffix="-rez", prefix="pip-")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [py_exe, "-m", "pip", "install"]
_extra_args = extra_args or config.pip_extra_args or []
if "--no-use-pep517" not in _extra_args:
cmd.append("--use-pep517")
if not _option_present(_extra_args, "-t", "--target"):
cmd.append("--target=%s" % targetpath)
if mode == InstallMode.no_deps and "--no-deps" not in _extra_args:
cmd.append("--no-deps")
cmd.extend(_extra_args)
cmd.append(source_name)
# run pip
#
# Note: https://github.com/pypa/pip/pull/3934. If/when this PR is merged,
# it will allow explicit control of where to put bin files.
#
_cmd(context=context, command=cmd)
# determine version of python in use
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver_str = '.'.join(map(str, sys.version_info))
py_ver = Version(py_ver_str)
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version
# Collect resulting python packages using distlib
distribution_path = DistributionPath([targetpath])
distributions = list(distribution_path.get_distributions())
dist_names = [x.name for x in distributions]
def log_append_pkg_variants(pkg_maker):
template = '{action} [{package.qualified_name}] {package.uri}{suffix}'
actions_variants = [
(
print_info,
'Installed',
installed_variants,
pkg_maker.installed_variants or [],
),
(
print_debug,
'Skipped',
skipped_variants,
pkg_maker.skipped_variants or [],
),
]
for print_, action, variants, pkg_variants in actions_variants:
for variant in pkg_variants:
variants.append(variant)
package = variant.parent
suffix = (' (%s)' % variant.subpath) if variant.subpath else ''
print_(template.format(**locals()))
# get list of package and dependencies
for distribution in distributions:
# convert pip requirements into rez requirements
rez_requires = get_rez_requirements(installed_dist=distribution,
python_version=py_ver,
name_casings=dist_names)
# log the pip -> rez requirements translation, for debugging
_log("Pip to rez requirements translation information for " +
distribution.name_and_version + ":\n" +
pformat({
"pip": {
"run_requires": map(str, distribution.run_requires)
},
"rez": rez_requires
}))
# determine where pip files need to be copied into rez package
src_dst_lut = _get_distribution_files_mapping(distribution, targetpath)
# build tools list
tools = []
for relpath in src_dst_lut.values():
dir_, filename = os.path.split(relpath)
if dir_ == "bin":
tools.append(filename)
# Sanity warning to see if any files will be copied
if not src_dst_lut:
message = 'No source files exist for {}!'
if not _verbose:
message += '\nTry again with rez-pip --verbose ...'
print_warning(message.format(distribution.name_and_version))
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for rel_src, rel_dest in src_dst_lut.items():
src = os.path.join(targetpath, rel_src)
dest = os.path.join(path, rel_dest)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
shutil.copyfile(src, dest)
if _is_exe(src):
shutil.copystat(src, dest)
# create the rez package
name = pip_to_rez_package_name(distribution.name)
version = pip_to_rez_version(distribution.version)
requires = rez_requires["requires"]
variant_requires = rez_requires["variant_requires"]
metadata = rez_requires["metadata"]
with make_package(name, packages_path, make_root=make_root) as pkg:
# basics (version etc)
pkg.version = version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
# requirements and variants
if requires:
pkg.requires = requires
if variant_requires:
pkg.variants = [variant_requires]
# commands
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
# Make the package use hashed variants. This is required because we
# can't control what ends up in its variants, and that can easily
# include problematic chars (>, +, ! etc).
# TODO: https://github.com/nerdvegas/rez/issues/672
#
pkg.hashed_variants = True
# add some custom attributes to retain pip-related info
pkg.pip_name = distribution.name_and_version
pkg.from_pip = True
pkg.is_pure_python = metadata["is_pure_python"]
distribution_metadata = distribution.metadata.todict()
help_ = []
if "home_page" in distribution_metadata:
help_.append(["Home Page", distribution_metadata["home_page"]])
if "download_url" in distribution_metadata:
help_.append(
["Source Code", distribution_metadata["download_url"]])
if help_:
pkg.help = help_
if "author" in distribution_metadata:
author = distribution_metadata["author"]
if "author_email" in distribution_metadata:
author += ' ' + distribution_metadata["author_email"]
pkg.authors = [author]
log_append_pkg_variants(pkg)
# cleanup
shutil.rmtree(targetpath)
# print summary
#
if installed_variants:
print_info("%d packages were installed.", len(installed_variants))
else:
print_warning("NO packages were installed.")
if skipped_variants:
print_warning(
"%d packages were already installed.",
len(skipped_variants),
)
return installed_variants, skipped_variants
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
pip_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path
if release else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
# TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
binpath = os.path.join(stagingdir, "bin")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [pip_exe, "install", "--use-pep517", "--target=%s" % destpath]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
def pure_python_package(installed_dist):
true_table = {"true": True, "false": False}
packages = pkg_resources.find_distributions(destpath)
dist = next(
(package
for package in packages if package.key == installed_dist.key),
None)
wheel_data = dist.get_metadata('WHEEL')
# see https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
wheel_data = Parser().parsestr(wheel_data)
# see https://www.python.org/dev/peps/pep-0427/#what-s-the-deal-with-purelib-vs-platlib
return true_table[wheel_data["Root-Is-Purelib"]]
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = [d for d in distribution_path.get_distributions()]
# moving bin folder to expected relative location as per wheel RECORD files
staged_binpath = os.path.join(destpath, "bin")
if os.path.isdir(staged_binpath):
shutil.move(os.path.join(destpath, "bin"), binpath)
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(
_get_dependencies(requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(
_get_dependencies(requirement, distributions))
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files():
# distlib expects the script files to be located in ../../bin/
# when in fact ../bin seems to be the resulting path after the
# installation as such we need to point the bin files to the
# expected location to match wheel RECORD files
installed_filepath = os.path.normpath(installed_file[0])
bin_prefix = os.path.join('..', '..', 'bin') + os.sep
if installed_filepath.startswith(bin_prefix):
# account for extra parentdir as explained above
installed = os.path.join(destpath, '_', installed_filepath)
else:
installed = os.path.join(destpath, installed_filepath)
source_file = os.path.normpath(installed)
if os.path.exists(source_file):
destination_file = os.path.relpath(source_file, stagingdir)
exe = False
if is_exe(source_file) and destination_file.startswith("bin" +
os.sep):
_file = os.path.basename(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(
os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# determine variant requirements
variant_reqs = []
pure = pure_python_package(distribution)
if not pure:
variant_reqs.append("platform-%s" % _system.platform)
variant_reqs.append("arch-%s" % _system.arch)
# Add the python version requirement. Note that we specify python to
# minor version because of environment markers - these often mean that
# you cannot use a loose python requirement (ie major version only)
# because then the package requirements would not be correct for all
# versions of python within that major version.
#
# This is not perfect. It means that often we will overspecify the required
# python version; and theoretically there could be markers that specify
# python down to the patch version. However, accurately varianting on
# python based on markers may be overly complicated, and may also
# result in odd varianting cases.
#
# https://www.python.org/dev/peps/pep-0508/#environment-markers
#
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver = '.'.join(map(str, sys.version_info[:2]))
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version.trim(2)
variant_reqs.append("python-%s" % py_ver)
name = pip_to_rez_package_name(distribution)
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = pip_to_rez_version(distribution.version)
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
pkg.variants = [variant_reqs]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def pip_install_package(source_name, pip_version=None, python_version=None,
mode=InstallMode.min_deps, release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
pip_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
binpath = os.path.join(stagingdir, "bin")
incpath = os.path.join(stagingdir, "include")
datapath = stagingdir
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [pip_exe, "install",
"--install-option=--install-lib=%s" % destpath,
"--install-option=--install-scripts=%s" % binpath,
"--install-option=--install-headers=%s" % incpath,
"--install-option=--install-data=%s" % datapath]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath], include_egg=True)
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(requirement, distributions))
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files(allow_fail=True):
source_file = os.path.normpath(os.path.join(destpath, installed_file[0]))
if os.path.exists(source_file):
destination_file = installed_file[0].split(stagingsep)[1]
exe = False
if is_exe(source_file) and \
destination_file.startswith("%s%s" % ("bin", os.path.sep)):
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# determine variant requirements
# TODO detect if platform/arch/os necessary, no if pure python
variant_reqs = []
variant_reqs.append("platform-%s" % _system.platform)
variant_reqs.append("arch-%s" % _system.arch)
variant_reqs.append("os-%s" % _system.os)
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver = '.'.join(map(str, sys.version_info[:2]))
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version.trim(2)
variant_reqs.append("python-%s" % py_ver)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
pkg.variants = [variant_reqs]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def pip_install_package(source_name, python_version=None,
release=False, no_deps=False,
prefix=None, auto_variants=True,
variants=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
prefix (str, optional): Override install location to here,
similar to `rez build --prefix`
no_deps (bool, optional): The `pip --no-deps` argument
auto_variants (bool, optional): Compute variants from the PyPI
classifiers portion of setup()
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
prefix (str, optional): Override release path with this absolute path
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
if prefix is not None:
config.release_packages_path = prefix
# TODO: should check if packages_path is writable
# before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
python_exe, context = find_python(python_version)
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [
python_exe, "-m", "pip", "install",
"--target", destpath,
# Only ever consider wheels, anything else is ancient
"--use-pep517",
# Handle case where the Python distribution used alongside
# pip already has a package installed in its `site-packages/` dir.
"--ignore-installed",
]
if no_deps:
# Delegate the installation of dependencies to the user
# This is important, as each dependency may have different
# requirements of its own, and variants to go with it.
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles
# conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(
requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(
requirement, distributions))
tools = []
src_dst_lut = {}
files = distribution.list_installed_files()
for installed_file in files:
source_file = os.path.join(destpath, installed_file[0])
source_file = os.path.normpath(source_file)
if os.path.exists(source_file):
destination_file = source_file.split(stagingsep)[1]
exe = False
starts_with_bin = destination_file.startswith(
"%s%s" % ("bin", os.path.sep)
)
if is_exe(source_file) and starts_with_bin:
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez
package variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.join(path, destination_file)
destination_file = os.path.normpath(destination_file)
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
# determine variant requirements
variants_ = variants or []
if (not variants_) and auto_variants:
variants_.extend(wheel_to_variants(distribution))
if variants_:
print_info("'%s' - Automatically detected variants: %s" % (
name, ", ".join(variants_))
)
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
if variants_:
pkg.variants = [variants_]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
py_exe, context = find_pip(pip_version, python_version)
# TODO: should check if packages_path is writable before continuing with pip
#
packages_path = (config.release_packages_path
if release else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
# TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
binpath = os.path.join(stagingdir, "bin")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [
py_exe, "-m", "pip", "install", "--use-pep517",
"--target=%s" % destpath
]
if mode == InstallMode.no_deps:
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
_system = System()
# determine version of python in use
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver_str = '.'.join(map(str, sys.version_info))
py_ver = Version(py_ver_str)
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version
# moving bin folder to expected relative location as per wheel RECORD files
staged_binpath = os.path.join(destpath, "bin")
if os.path.isdir(staged_binpath):
shutil.move(os.path.join(destpath, "bin"), binpath)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = list(distribution_path.get_distributions())
dist_names = [x.name for x in distributions]
# get list of package and dependencies
for distribution in distributions:
# convert pip requirements into rez requirements
rez_requires = get_rez_requirements(installed_dist=distribution,
python_version=py_ver,
name_casings=dist_names)
# log the pip -> rez translation, for debugging
_log("Pip to rez translation information for " +
distribution.name_and_version + ":\n" +
pformat({
"pip": {
"run_requires": map(str, distribution.run_requires)
},
"rez": rez_requires
}))
# iterate over installed files and determine dest filepaths
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files():
# distlib expects the script files to be located in ../../bin/
# when in fact ../bin seems to be the resulting path after the
# installation as such we need to point the bin files to the
# expected location to match wheel RECORD files
installed_filepath = os.path.normpath(installed_file[0])
bin_prefix = os.path.join('..', '..', 'bin') + os.sep
if installed_filepath.startswith(bin_prefix):
# account for extra parentdir as explained above
installed = os.path.join(destpath, '_', installed_filepath)
else:
installed = os.path.join(destpath, installed_filepath)
source_file = os.path.normpath(installed)
if os.path.exists(source_file):
destination_file = os.path.relpath(source_file, stagingdir)
exe = False
if is_exe(source_file) and destination_file.startswith("bin" +
os.sep):
_file = os.path.basename(destination_file)
tools.append(_file)
exe = True
src_dst_lut[source_file] = [destination_file, exe]
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(
os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# create the rez package
name = pip_to_rez_package_name(distribution.name)
version = pip_to_rez_version(distribution.version)
requires = rez_requires["requires"]
variant_requires = rez_requires["variant_requires"]
metadata = rez_requires["metadata"]
with make_package(name, packages_path, make_root=make_root) as pkg:
# basics (version etc)
pkg.version = version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
# requirements and variants
if requires:
pkg.requires = requires
if variant_requires:
pkg.variants = [variant_requires]
# commands
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
# Make the package use hashed variants. This is required because we
# can't control what ends up in its variants, and that can easily
# include problematic chars (>, +, ! etc).
# TODO: https://github.com/nerdvegas/rez/issues/672
#
pkg.hashed_variants = True
# add some custom attributes to retain pip-related info
pkg.pip_name = distribution.name_and_version
pkg.from_pip = True
pkg.is_pure_python = metadata["is_pure_python"]
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
