def bundle_context(context,
dest_dir,
force=False,
skip_non_relocatable=False,
quiet=False,
verbose=False):
"""Bundle a context and its variants into a relocatable dir.
This creates a copy of a context with its variants retargeted to a local
package repository containing only the variants the context uses. The
generated file structure looks like so:
/dest_dir/
/context.rxt
/packages/
/foo/1.1.1/package.py
/...(payload)...
/bah/4.5.6/package.py
/...(payload)...
Args:
context (`ResolvedContext`): Context to bundle
dest_dir (str): Destination directory. Must not exist.
force (bool): If True, relocate package even if non-relocatable. Use at
your own risk. Overrides `skip_non_relocatable`.
skip_non_relocatable (bool): If True, leave non-relocatable packages
unchanged. Normally this will raise a `PackageCopyError`.
quiet (bool): Suppress all output
verbose (bool): Verbose mode (quiet will override)
"""
if quiet:
verbose = False
if force:
skip_non_relocatable = False
if os.path.exists(dest_dir):
raise ContextBundleError("Dest dir must not exist: %s" % dest_dir)
if not quiet:
label = context.load_path or "context"
print_info("Bundling %s into %s...", label, dest_dir)
os.mkdir(dest_dir)
_init_bundle(dest_dir)
relocated_package_names = _copy_variants(
context=context,
bundle_dir=dest_dir,
force=force,
skip_non_relocatable=skip_non_relocatable,
verbose=verbose)
rxt_filepath = _write_retargeted_context(
context=context,
bundle_dir=dest_dir,
relocated_package_names=relocated_package_names)
if verbose:
print_info("Context bundled to %s", rxt_filepath)
def remove_packages_ignored_since(days,
paths=None,
dry_run=False,
verbose=False):
"""Remove packages ignored for >= specified number of days.
Args:
days (int): Remove packages ignored >= this many days
paths (list of str, optional): Paths to search for packages, defaults
to `config.packages_path`.
dry_run: Dry run mode
verbose (bool): Verbose mode
Returns:
int: Number of packages removed. In dry-run mode, returns the number of
packages that _would_ be removed.
"""
num_removed = 0
for path in (paths or config.packages_path):
repo = package_repository_manager.get_repository(path)
if verbose:
print_info("Searching %s...", repo)
num_removed += repo.remove_ignored_since(days=days,
dry_run=dry_run,
verbose=verbose)
return num_removed
def create_context(pip_version=None, python_version=None):
"""Create a context containing the specific pip and python.
Args:
pip_version (str or `Version`): Version of pip to use, or latest if None.
python_version (str or `Version`): Python version to use, or latest if
None.
Returns:
`ResolvedContext`: Context containing pip and python.
"""
# determine pip pkg to use for install, and python variants to install on
if pip_version:
pip_req = "pip-%s" % str(pip_version)
else:
pip_req = "pip"
if python_version:
ver = Version(str(python_version))
major_minor_ver = ver.trim(2)
py_req = "python-%s" % str(major_minor_ver)
else:
# use latest major.minor
package = get_latest_package("python")
if package:
major_minor_ver = package.version.trim(2)
else:
# no python package. We're gonna fail, let's just choose current
# python version (and fail at context creation time)
major_minor_ver = ".".join(map(str, sys.version_info[:2]))
py_req = "python-%s" % str(major_minor_ver)
# use pip + latest python to perform pip download operations
request = [pip_req, py_req]
with convert_errors(
from_=(PackageFamilyNotFoundError, PackageNotFoundError),
to=BuildError,
msg="Cannot run - pip or python rez " "package is not present",
):
context = ResolvedContext(request)
# print pip package used to perform the install
pip_variant = context.get_resolved_package("pip")
pip_package = pip_variant.parent
print_info("Using %s (%s)" % (pip_package.qualified_name, pip_variant.uri))
return context
def create_context(pip_version=None, python_version=None):
"""Create a context containing the specific pip and python.
Args:
pip_version (str or `Version`): Version of pip to use, or latest if None.
python_version (str or `Version`): Python version to use, or latest if
None.
Returns:
`ResolvedContext`: Context containing pip and python.
"""
# determine pip pkg to use for install, and python variants to install on
if pip_version:
pip_req = "pip-%s" % str(pip_version)
else:
pip_req = "pip"
if python_version:
ver = Version(str(python_version))
major_minor_ver = ver.trim(2)
py_req = "python-%s" % str(major_minor_ver)
else:
# use latest major.minor
package = get_latest_package("python")
if package:
major_minor_ver = package.version.trim(2)
else:
# no python package. We're gonna fail, let's just choose current
# python version (and fail at context creation time)
major_minor_ver = '.'.join(map(str, sys.version_info[:2]))
py_req = "python-%s" % str(major_minor_ver)
# use pip + latest python to perform pip download operations
request = [pip_req, py_req]
with convert_errors(from_=(PackageFamilyNotFoundError,
PackageNotFoundError),
to=BuildError,
msg="Cannot run - pip or python rez "
"package is not present"):
context = ResolvedContext(request)
# print pip package used to perform the install
pip_variant = context.get_resolved_package("pip")
pip_package = pip_variant.parent
print_info("Using %s (%s)" % (pip_package.qualified_name, pip_variant.uri))
return context
def test_print(self):
"""Test valid msg and nargs combinations for print_*."""
for msg in ("Hello", "Hello %s", "Hello %s %s"):
logging_.print_debug(msg)
logging_.print_info(msg)
logging_.print_warning(msg)
logging_.print_error(msg)
logging_.print_critical(msg)
for nargs in ([], ["foo"], ["foo", "bar"]):
logging_.print_debug(msg, *nargs)
logging_.print_info(msg, *nargs)
logging_.print_warning(msg, *nargs)
logging_.print_error(msg, *nargs)
logging_.print_critical(msg, *nargs)
def test_print(self):
"""Test valid msg and nargs combinations for print_*."""
for msg in ("Hello", "Hello %s", "Hello %s %s"):
logging_.print_debug(msg)
logging_.print_info(msg)
logging_.print_warning(msg)
logging_.print_error(msg)
logging_.print_critical(msg)
for nargs in ([], ["foo"], ["foo", "bar"]):
logging_.print_debug(msg, *nargs)
logging_.print_info(msg, *nargs)
logging_.print_warning(msg, *nargs)
logging_.print_error(msg, *nargs)
logging_.print_critical(msg, *nargs)
def find_pip(pip_version=None, python_version=None):
"""Find pip.
Will revert to native pip installed with rez, if a pip rez package cannot
be found. In this case, None is returned.
Args:
pip_version (str or `Version`): Version of pip to use, or latest if None.
python_version (str or `Version`): Python version to use, or latest if
None.
Returns:
2-tuple:
- str: Python executable.
- `ResolvedContext`: Context containing pip, or None if we fell back
to system pip.
"""
py_exe = "python"
context = None
# find pip, fall back to system if rez pip package not found
try:
context = create_context(pip_version, python_version)
py_exe = context.which("python")
except BuildError:
# fall back on system pip
py_exe = sys.executable
print_info("Using %s -m pip", py_exe)
# check version, must be >=19
if context:
proc = context.execute_command(
[py_exe, "-c", "import pip; print pip.__version__"],
stdout=subprocess.PIPE)
out, _ = proc.communicate()
pip_version = out.strip()
else:
import pip
pip_version = pip.__version__
pip_major = pip_version.split('.')[0]
if int(pip_major) < 19:
raise RezSystemError("pip >= 19 is required! Please update your pip.")
return py_exe, context
def remove_variant(pkgcache, uri, opts):
from rez.packages import get_variant_from_uri
from rez.utils.logging_ import print_info, print_warning, print_error
from rez.package_cache import PackageCache
print_info("Removing variant %r from package cache at %s:", uri, pkgcache.path)
variant = get_variant_from_uri(uri)
if variant is None:
print("No such variant: %s" % uri, file=sys.stderr)
sys.exit(1)
status = pkgcache.remove_variant(variant)
if status == PackageCache.VARIANT_NOT_FOUND:
print_error("No such variant found in cache")
elif status == PackageCache.VARIANT_COPYING:
print_warning("Another process is currently caching this variant")
else:
print_info("Variant successfully removed")
def bundle(self):
if os.path.exists(self.dest_dir):
raise ContextBundleError("Dest dir must not exist: %s" % self.dest_dir)
if not self.quiet:
label = self.context.load_path or "context"
print_info("Bundling %s into %s...", label, self.dest_dir)
# initialize the bundle
self._init_bundle()
# copy the variants from the context into the bundle
relocated_package_names = self._copy_variants()
# write a copy of the context, with refs changed to bundled variants
self._write_retargeted_context(relocated_package_names)
# apply patching to retarget dynamic linker to bundled packages
if self.patch_libs:
self._patch_libs()
# finalize the bundle
self._finalize_bundle()
def add_variant(pkgcache, uri, opts):
from rez.packages import get_variant_from_uri
from rez.utils.logging_ import print_info, print_warning
from rez.package_cache import PackageCache
print_info("Adding variant %r to package cache at %s:", uri, pkgcache.path)
variant = get_variant_from_uri(uri)
if variant is None:
print("No such variant: %s" % uri, file=sys.stderr)
sys.exit(1)
destpath, status = pkgcache.add_variant(variant, force=opts.force)
if status == PackageCache.VARIANT_FOUND:
print_info("Already exists: %s", destpath)
elif status == PackageCache.VARIANT_COPYING:
print_warning("Another process is currently copying to: %s", destpath)
else:
print_info("Successfully cached to: %s", destpath)
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False,
prefix=None,
extra_args=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
extra_args (List[str]): Additional options to the pip install command.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
py_exe, context = find_pip(pip_version, python_version)
print_info("Installing %r with pip taken from %r", source_name, py_exe)
# TODO: should check if packages_path is writable before continuing with pip
#
if prefix is not None:
packages_path = prefix
else:
packages_path = (config.release_packages_path
if release else config.local_packages_path)
targetpath = mkdtemp(suffix="-rez", prefix="pip-")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [py_exe, "-m", "pip", "install"]
_extra_args = extra_args or config.pip_extra_args or []
if "--no-use-pep517" not in _extra_args:
cmd.append("--use-pep517")
if not _option_present(_extra_args, "-t", "--target"):
cmd.append("--target=%s" % targetpath)
if mode == InstallMode.no_deps and "--no-deps" not in _extra_args:
cmd.append("--no-deps")
cmd.extend(_extra_args)
cmd.append(source_name)
# run pip
#
# Note: https://github.com/pypa/pip/pull/3934. If/when this PR is merged,
# it will allow explicit control of where to put bin files.
#
_cmd(context=context, command=cmd)
# determine version of python in use
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver_str = '.'.join(map(str, sys.version_info))
py_ver = Version(py_ver_str)
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version
# Collect resulting python packages using distlib
distribution_path = DistributionPath([targetpath])
distributions = list(distribution_path.get_distributions())
dist_names = [x.name for x in distributions]
def log_append_pkg_variants(pkg_maker):
template = '{action} [{package.qualified_name}] {package.uri}{suffix}'
actions_variants = [
(
print_info,
'Installed',
installed_variants,
pkg_maker.installed_variants or [],
),
(
print_debug,
'Skipped',
skipped_variants,
pkg_maker.skipped_variants or [],
),
]
for print_, action, variants, pkg_variants in actions_variants:
for variant in pkg_variants:
variants.append(variant)
package = variant.parent
suffix = (' (%s)' % variant.subpath) if variant.subpath else ''
print_(template.format(**locals()))
# get list of package and dependencies
for distribution in distributions:
# convert pip requirements into rez requirements
rez_requires = get_rez_requirements(installed_dist=distribution,
python_version=py_ver,
name_casings=dist_names)
# log the pip -> rez requirements translation, for debugging
_log("Pip to rez requirements translation information for " +
distribution.name_and_version + ":\n" +
pformat({
"pip": {
"run_requires": map(str, distribution.run_requires)
},
"rez": rez_requires
}))
# determine where pip files need to be copied into rez package
src_dst_lut = _get_distribution_files_mapping(distribution, targetpath)
# build tools list
tools = []
for relpath in src_dst_lut.values():
dir_, filename = os.path.split(relpath)
if dir_ == "bin":
tools.append(filename)
# Sanity warning to see if any files will be copied
if not src_dst_lut:
message = 'No source files exist for {}!'
if not _verbose:
message += '\nTry again with rez-pip --verbose ...'
print_warning(message.format(distribution.name_and_version))
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for rel_src, rel_dest in src_dst_lut.items():
src = os.path.join(targetpath, rel_src)
dest = os.path.join(path, rel_dest)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
shutil.copyfile(src, dest)
if _is_exe(src):
shutil.copystat(src, dest)
# create the rez package
name = pip_to_rez_package_name(distribution.name)
version = pip_to_rez_version(distribution.version)
requires = rez_requires["requires"]
variant_requires = rez_requires["variant_requires"]
metadata = rez_requires["metadata"]
with make_package(name, packages_path, make_root=make_root) as pkg:
# basics (version etc)
pkg.version = version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
# requirements and variants
if requires:
pkg.requires = requires
if variant_requires:
pkg.variants = [variant_requires]
# commands
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
# Make the package use hashed variants. This is required because we
# can't control what ends up in its variants, and that can easily
# include problematic chars (>, +, ! etc).
# TODO: https://github.com/nerdvegas/rez/issues/672
#
pkg.hashed_variants = True
# add some custom attributes to retain pip-related info
pkg.pip_name = distribution.name_and_version
pkg.from_pip = True
pkg.is_pure_python = metadata["is_pure_python"]
distribution_metadata = distribution.metadata.todict()
help_ = []
if "home_page" in distribution_metadata:
help_.append(["Home Page", distribution_metadata["home_page"]])
if "download_url" in distribution_metadata:
help_.append(
["Source Code", distribution_metadata["download_url"]])
if help_:
pkg.help = help_
if "author" in distribution_metadata:
author = distribution_metadata["author"]
if "author_email" in distribution_metadata:
author += ' ' + distribution_metadata["author_email"]
pkg.authors = [author]
log_append_pkg_variants(pkg)
# cleanup
shutil.rmtree(targetpath)
# print summary
#
if installed_variants:
print_info("%d packages were installed.", len(installed_variants))
else:
print_warning("NO packages were installed.")
if skipped_variants:
print_warning(
"%d packages were already installed.",
len(skipped_variants),
)
return installed_variants, skipped_variants
def find_pip_from_context(python_version, pip_version=None):
"""Find pip from rez context.
Args:
python_version (str or `Version`): Python version to use
pip_version (str or `Version`): Version of pip to use, or latest.
Returns:
3-tuple:
- str: Python executable or None if we fell back to system pip.
- str: Pip version or None if we fell back to system pip.
- `ResolvedContext`: Context containing pip, or None if we fell back
to system pip.
"""
target = "python"
package_request = []
if python_version:
ver = Version(str(python_version))
python_major_minor_ver = ver.trim(2)
else:
# use latest major.minor
package = get_latest_package("python")
if package:
python_major_minor_ver = package.version.trim(2)
else:
raise BuildError("Found no python package.")
python_package = "python-%s" % str(python_major_minor_ver)
package_request.append(python_package)
if pip_version:
target = "pip"
if pip_version == "latest":
package_request.append("pip")
else:
package_request.append("pip-%s" % str(pip_version))
print_info("Trying to use pip from %s package", target)
try:
context = ResolvedContext(package_request)
except (PackageFamilyNotFoundError, PackageNotFoundError):
print_debug("No rez package called %s found", target)
return None, None, None
py_exe_name = "python"
if platform_.name != "windows":
# Python < 2 on Windows doesn't have versionned executable.
py_exe_name += str(python_major_minor_ver.trim(1))
py_exe = context.which(py_exe_name)
proc = context.execute_command(
# -E and -s are used to isolate the environment as much as possible.
# See python --help for more details. We absolutely don't want to get
# pip from the user home.
[
py_exe, "-E", "-s", "-c",
"import pip, sys; sys.stdout.write(pip.__version__)"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
out, err = proc.communicate()
if proc.returncode:
print_debug("Failed to get pip from package %s", target)
print_debug(out)
print_debug(err)
return None, None, None
pip_version = out.strip()
variant = context.get_resolved_package(target)
package = variant.parent
print_info("Found pip-%s inside %s. Will use it with %s", pip_version,
package.uri, py_exe)
return py_exe, pip_version, context
def _get_preprocessed(self, data):
"""
Returns:
(DeveloperPackage, new_data) 2-tuple IFF the preprocess function
changed the package; otherwise None.
"""
from rez.serialise import process_python_objects
from rez.utils.data_utils import get_dict_diff
from copy import deepcopy
with add_sys_paths(config.package_definition_build_python_paths):
preprocess = getattr(self, "preprocess", None)
if preprocess:
preprocess_func = preprocess.func
print_info("Applying preprocess from package.py")
else:
# load globally configured preprocess function
dotted = self.config.package_preprocess_function
if not dotted:
return None
if '.' not in dotted:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. Package "
"preprocessing has not been applied.")
return None
name, funcname = dotted.rsplit('.', 1)
try:
module = __import__(name=name, fromlist=[funcname])
except Exception as e:
print_error(
"Failed to load preprocessing function '%s': %s" %
(dotted, str(e)))
return None
setattr(module, "InvalidPackageError", InvalidPackageError)
preprocess_func = getattr(module, funcname)
if not preprocess_func or not isfunction(isfunction):
print_error("Function '%s' not found" % dotted)
return None
print_info("Applying preprocess function %s" % dotted)
preprocessed_data = deepcopy(data)
# apply preprocessing
try:
preprocess_func(this=self, data=preprocessed_data)
except InvalidPackageError:
raise
except Exception as e:
print_error("Failed to apply preprocess: %s: %s" %
(e.__class__.__name__, str(e)))
return None
# if preprocess added functions, these need to be converted to
# SourceCode instances
preprocessed_data = process_python_objects(preprocessed_data)
if preprocessed_data == data:
return None
# recreate package from modified package data
package = create_package(self.name,
preprocessed_data,
package_cls=self.__class__)
# print summary of changed package attributes
added, removed, changed = get_dict_diff(data, preprocessed_data)
lines = ["Package attributes were changed in preprocessing:"]
if added:
lines.append("Added attributes: %s" % ['.'.join(x) for x in added])
if removed:
lines.append("Removed attributes: %s" %
['.'.join(x) for x in removed])
if changed:
lines.append("Changed attributes: %s" %
['.'.join(x) for x in changed])
txt = '\n'.join(lines)
print_info(txt)
return package, preprocessed_data
def _verbose_info(self, msg, *nargs):
if self.verbose:
print_info(msg, *nargs)
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
follow_symlinks=False, overrides=None, verbose=False):
# Get payload path of source variant. For some types (eg from a "memory"
# type repo) there may not be a root.
#
variant_root = getattr(src_variant, "root", None)
if not variant_root:
raise PackageCopyError(
"Cannot copy source variant %s - it is a type of variant that "
"does not have a root." % src_variant.uri
)
if not os.path.isdir(variant_root):
raise PackageCopyError(
"Cannot copy source variant %s - its root does not appear to "
"be present on disk (%s)." % src_variant.uri, variant_root
)
dest_variant_name = overrides.get("name") or src_variant.name
dest_variant_version = overrides.get("version") or src_variant.version
# determine variant installation path
dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_variant_name,
package_version=dest_variant_version
)
is_varianted = (src_variant.index is not None)
src_variant_subpath = None
if is_varianted:
src_variant_subpath = src_variant._non_shortlinked_subpath
variant_install_path = os.path.join(
dest_pkg_payload_path, src_variant_subpath)
else:
variant_install_path = dest_pkg_payload_path
# get ready for copy/symlinking
copy_func = partial(replacing_copy,
follow_symlinks=follow_symlinks)
if shallow:
maybe_symlink = replacing_symlink
else:
maybe_symlink = copy_func
# possibly make install path temporarily writable
last_dir = get_existing_path(
variant_install_path,
topmost_path=os.path.dirname(dest_pkg_payload_path))
if last_dir and config.make_package_temporarily_writable:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
# copy the variant payload
with ctxt:
safe_makedirs(variant_install_path)
# determine files not to copy
skip_files = []
if is_varianted and not src_variant.parent.hashed_variants:
# Detect overlapped variants. This is the case where one variant subpath
# might be A, and another is A/B. We must ensure that A/B is not created
# as a symlink during shallow install of variant A - that would then
# cause A/B payload to be installed back into original package, possibly
# corrupting it.
#
# Here we detect this case, and create a list of dirs not to copy/link,
# because they are in fact a subpath dir for another variant.
#
# Note that for hashed variants, we don't do this check because overlapped
# variants are not possible.
#
skip_files.extend(_get_overlapped_variant_dirs(src_variant))
else:
# just skip package definition file
for name in config.plugins.package_repository.filesystem.package_filenames:
for fmt in (FileFormat.py, FileFormat.yaml):
filename = name + '.' + fmt.extension
skip_files.append(filename)
# copy/link all topmost files within the variant root
for name in os.listdir(variant_root):
if name in skip_files:
filepath = os.path.join(variant_root, name)
if verbose and is_varianted:
print_info(
"Did not copy %s - this is part of an overlapping "
"variant's root path.", filepath
)
continue
src_path = os.path.join(variant_root, name)
dest_path = os.path.join(variant_install_path, name)
if os.path.islink(src_path):
copy_func(src_path, dest_path)
else:
maybe_symlink(src_path, dest_path)
# copy permissions of source variant dirs onto dest
src_package = src_variant.parent
src_pkg_repo = src_package.repository
src_pkg_payload_path = src_pkg_repo.get_package_payload_path(
package_name=src_package.name,
package_version=src_package.version
)
shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path)
subpath = src_variant_subpath
while subpath:
src_path = os.path.join(src_pkg_payload_path, subpath)
dest_path = os.path.join(dest_pkg_payload_path, subpath)
shutil.copystat(src_path, dest_path)
subpath = os.path.dirname(subpath)
# create the variant shortlink
if src_variant.parent.hashed_variants:
try:
# base _v dir
base_shortlinks_path = os.path.join(
dest_pkg_payload_path,
src_package.config.variant_shortlinks_dirname
)
safe_makedirs(base_shortlinks_path)
# shortlink
rel_variant_path = os.path.relpath(
variant_install_path, base_shortlinks_path)
create_unique_base26_symlink(
base_shortlinks_path, rel_variant_path)
except Exception as e:
# Treat any error as warning - lack of shortlink is not
# a breaking issue, it just means the variant root path
# will be long.
#
print_warning(
"Error creating variant shortlink for %s: %s: %s",
variant_install_path, e.__class__.__name__, e
)
def pip_install_package(source_name, python_version=None,
release=False, no_deps=False,
prefix=None, auto_variants=True,
variants=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
prefix (str, optional): Override install location to here,
similar to `rez build --prefix`
no_deps (bool, optional): The `pip --no-deps` argument
auto_variants (bool, optional): Compute variants from the PyPI
classifiers portion of setup()
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
prefix (str, optional): Override release path with this absolute path
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
if prefix is not None:
config.release_packages_path = prefix
# TODO: should check if packages_path is writable
# before continuing with pip
#
packages_path = (config.release_packages_path if release
else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
python_exe, context = find_python(python_version)
if context and config.debug("package_release"):
buf = StringIO()
print >> buf, "\n\npackage download environment:"
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [
python_exe, "-m", "pip", "install",
"--target", destpath,
# Only ever consider wheels, anything else is ancient
"--use-pep517",
# Handle case where the Python distribution used alongside
# pip already has a package installed in its `site-packages/` dir.
"--ignore-installed",
]
if no_deps:
# Delegate the installation of dependencies to the user
# This is important, as each dependency may have different
# requirements of its own, and variants to go with it.
cmd.append("--no-deps")
cmd.append(source_name)
_cmd(context=context, command=cmd)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = [d for d in distribution_path.get_distributions()]
for distribution in distribution_path.get_distributions():
requirements = []
if distribution.metadata.run_requires:
# Handle requirements. Currently handles
# conditional environment based
# requirements and normal requirements
# TODO: Handle optional requirements?
for requirement in distribution.metadata.run_requires:
if "environment" in requirement:
if interpret(requirement["environment"]):
requirements.extend(_get_dependencies(
requirement, distributions))
elif "extra" in requirement:
# Currently ignoring optional requirements
pass
else:
requirements.extend(_get_dependencies(
requirement, distributions))
tools = []
src_dst_lut = {}
files = distribution.list_installed_files()
for installed_file in files:
source_file = os.path.join(destpath, installed_file[0])
source_file = os.path.normpath(source_file)
if os.path.exists(source_file):
destination_file = source_file.split(stagingsep)[1]
exe = False
starts_with_bin = destination_file.startswith(
"%s%s" % ("bin", os.path.sep)
)
if is_exe(source_file) and starts_with_bin:
_, _file = os.path.split(destination_file)
tools.append(_file)
exe = True
data = [destination_file, exe]
src_dst_lut[source_file] = data
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez
package variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.join(path, destination_file)
destination_file = os.path.normpath(destination_file)
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
name, _ = parse_name_and_version(distribution.name_and_version)
name = distribution.name[0:len(name)].replace("-", "_")
# determine variant requirements
variants_ = variants or []
if (not variants_) and auto_variants:
variants_.extend(wheel_to_variants(distribution))
if variants_:
print_info("'%s' - Automatically detected variants: %s" % (
name, ", ".join(variants_))
)
with make_package(name, packages_path, make_root=make_root) as pkg:
pkg.version = distribution.version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
if variants_:
pkg.variants = [variants_]
if requirements:
pkg.requires = requirements
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def run_test(self, test_name):
"""Run a test.
Runs the test in its correct environment. Note that if tests share the
same requirements, the contexts will be reused.
Args:
test_name (str): Name of test to run.
Returns:
int: Exit code of first failed test, or 0 if none failed. If the first
test to fail did so because it was not able to run (eg its
environment could not be configured), -1 is returned.
"""
package = self.get_package()
exitcode = 0
if test_name not in self.get_test_names():
raise PackageTestError("Test '%s' not found in package %s" %
(test_name, package.uri))
if self.use_current_env:
if package is None:
self._add_test_result(
test_name, None, "skipped",
"The current environment does not contain a package "
"matching the request")
return
current_context = ResolvedContext.get_current()
current_variant = current_context.get_resolved_package(
package.name)
target_variants = [current_variant]
else:
target_variants = self._get_target_variants(test_name)
for variant in target_variants:
# get test info for this variant. If None, that just means that this
# variant doesn't provide this test. That's ok - 'tests' might be
# implemented as a late function attribute that provides some tests
# for some variants and not others
#
test_info = self._get_test_info(test_name, variant)
if not test_info:
self._add_test_result(
test_name, variant, "skipped",
"The test is not declared in this variant")
continue
command = test_info["command"]
requires = test_info["requires"]
on_variants = test_info["on_variants"]
# show progress
if self.verbose > 1:
self._print_header("\nRunning test: %s\nPackage: %s\n%s\n",
test_name, variant.uri, '-' * 80)
elif self.verbose:
self._print_header("\nRunning test: %s\n%s\n", test_name,
'-' * 80)
# apply variant selection filter if specified
if isinstance(on_variants, dict):
filter_type = on_variants["type"]
func = getattr(self, "_on_variant_" + filter_type)
do_test = func(variant, on_variants)
if not do_test:
reason = (
"Test skipped as specified by on_variants '%s' filter"
% filter_type)
print_info(reason)
self._add_test_result(test_name, variant, "skipped",
reason)
continue
# add requirements to force the current variant to be resolved.
# TODO this is not perfect, and will need to be updated when
# explicit variant selection is added to rez (this is a new
# feature). Until then, there's no guarantee that we'll resolve to
# the variant we want, so we take that into account here.
#
requires.extend(map(str, variant.variant_requires))
# create test runtime env
exc = None
try:
context = self._get_context(requires)
except RezError as e:
exc = e
fail_reason = None
if exc is not None:
fail_reason = "The test environment failed to resolve: %s" % exc
elif context is None:
fail_reason = "The current environment does not meet test requirements"
elif not context.success:
fail_reason = "The test environment failed to resolve"
if fail_reason:
self._add_test_result(test_name, variant, "failed",
fail_reason)
print_error(fail_reason)
if not exitcode:
exitcode = -1
if self.stop_on_fail:
self.stopped_on_fail = True
return exitcode
continue
# check that this has actually resolved the variant we want
resolved_variant = context.get_resolved_package(package.name)
assert resolved_variant
if resolved_variant.handle != variant.handle:
print_warning(
"Could not resolve environment for this variant (%s). This "
"is a known issue and will be fixed once 'explicit variant "
"selection' is added to rez.", variant.uri)
self._add_test_result(
test_name, variant, "skipped",
"Could not resolve to variant (known issue)")
continue
# expand refs like {root} in commands
if isinstance(command, basestring):
command = variant.format(command)
else:
command = map(variant.format, command)
# run the test in the context
if self.verbose:
if self.verbose > 1:
context.print_info(self.stdout)
print('')
if isinstance(command, basestring):
cmd_str = command
else:
cmd_str = ' '.join(map(quote, command))
self._print_header("Running test command: %s", cmd_str)
if self.dry_run:
self._add_test_result(test_name, variant, "skipped",
"Dry run mode")
continue
def _pre_test_commands(executor):
# run package.py:pre_test_commands() if present
pre_test_commands = getattr(variant, "pre_test_commands")
if not pre_test_commands:
return
test_ns = {"name": test_name}
with executor.reset_globals():
executor.bind("this", variant)
executor.bind("test", RO_AttrDictWrapper(test_ns))
executor.execute_code(pre_test_commands)
retcode, _, _ = context.execute_shell(
command=command,
actions_callback=_pre_test_commands,
stdout=self.stdout,
stderr=self.stderr,
block=True)
if retcode:
print_warning("Test command exited with code %d", retcode)
self._add_test_result(
test_name, variant, "failed",
"Test failed with exit code %d" % retcode)
if not exitcode:
exitcode = retcode
if self.stop_on_fail:
self.stopped_on_fail = True
return exitcode
continue
# test passed
self._add_test_result(test_name, variant, "success",
"Test succeeded")
# just test against one variant in this case
if on_variants is False:
break
return exitcode
def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
follow_symlinks=False, overrides=None, verbose=False):
# Get payload path of source variant. For some types (eg from a "memory"
# type repo) there may not be a root.
#
variant_root = getattr(src_variant, "root", None)
if not variant_root:
raise PackageCopyError(
"Cannot copy source variant %s - it is a type of variant that "
"does not have a root.", src_variant.uri
)
if not os.path.isdir(variant_root):
raise PackageCopyError(
"Cannot copy source variant %s - its root does not appear to "
"be present on disk (%s).", src_variant.uri, variant_root
)
dest_variant_name = overrides.get("name") or src_variant.name
dest_variant_version = overrides.get("version") or src_variant.version
# determine variant installation path
dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_variant_name,
package_version=dest_variant_version
)
if src_variant.subpath:
variant_install_path = os.path.join(dest_pkg_payload_path,
src_variant.subpath)
else:
variant_install_path = dest_pkg_payload_path
# get ready for copy/symlinking
copy_func = partial(replacing_copy,
follow_symlinks=follow_symlinks)
if shallow:
maybe_symlink = replacing_symlink
else:
maybe_symlink = copy_func
# possibly make install path temporarily writable
last_dir = get_existing_path(
variant_install_path,
topmost_path=os.path.dirname(dest_pkg_payload_path))
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
# copy the variant payload
with ctxt:
safe_makedirs(variant_install_path)
# determine files not to copy
skip_files = []
if src_variant.subpath:
# Detect overlapped variants. This is the case where one variant subpath
# might be A, and another is A/B. We must ensure that A/B is not created
# as a symlink during shallow install of variant A - that would then
# cause A/B payload to be installed back into original package, possibly
# corrupting it.
#
# Here we detect this case, and create a list of dirs not to copy/link,
# because they are in fact a subpath dir for another variant.
#
skip_files.extend(_get_overlapped_variant_dirs(src_variant))
else:
# just skip package definition file
for name in config.plugins.package_repository.filesystem.package_filenames:
for fmt in (FileFormat.py, FileFormat.yaml):
filename = name + '.' + fmt.extension
skip_files.append(filename)
# copy/link all topmost files within the variant root
for name in os.listdir(variant_root):
if name in skip_files:
filepath = os.path.join(variant_root, name)
if verbose:
if src_variant.subpath:
msg = ("Did not copy %s - this is part of an "
"overlapping variant's root path.")
else:
msg = "Did not copy package definition file %s"
print_info(msg, filepath)
continue
src_path = os.path.join(variant_root, name)
dest_path = os.path.join(variant_install_path, name)
if os.path.islink(src_path):
copy_func(src_path, dest_path)
else:
maybe_symlink(src_path, dest_path)
# copy permissions of source variant dirs onto dest
src_package = src_variant.parent
src_pkg_repo = src_package.repository
src_pkg_payload_path = src_pkg_repo.get_package_payload_path(
package_name=src_package.name,
package_version=src_package.version
)
shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path)
subpath = src_variant.subpath
while subpath:
src_path = os.path.join(src_pkg_payload_path, subpath)
dest_path = os.path.join(dest_pkg_payload_path, subpath)
shutil.copystat(src_path, dest_path)
subpath = os.path.dirname(subpath)
def pip_install_package(source_name,
pip_version=None,
python_version=None,
mode=InstallMode.min_deps,
release=False,
prefix=None,
extra_args=None):
"""Install a pip-compatible python package as a rez package.
Args:
source_name (str): Name of package or archive/url containing the pip
package source. This is the same as the arg you would pass to
the 'pip install' command.
pip_version (str or `Version`): Version of pip to use to perform the
install, uses latest if None.
python_version (str or `Version`): Python version to use to perform the
install, and subsequently have the resulting rez package depend on.
mode (`InstallMode`): Installation mode, determines how dependencies are
managed.
release (bool): If True, install as a released package; otherwise, it
will be installed as a local package.
extra_args (List[str]): Additional options to the pip install command.
Returns:
2-tuple:
List of `Variant`: Installed variants;
List of `Variant`: Skipped variants (already installed).
"""
installed_variants = []
skipped_variants = []
py_exe, context = find_pip(pip_version, python_version)
print_info("Installing %r with pip taken from %r", source_name, py_exe)
# TODO: should check if packages_path is writable before continuing with pip
#
if prefix is not None:
packages_path = prefix
else:
packages_path = (config.release_packages_path
if release else config.local_packages_path)
tmpdir = mkdtemp(suffix="-rez", prefix="pip-")
stagingdir = os.path.join(tmpdir, "rez_staging")
stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
destpath = os.path.join(stagingdir, "python")
# TODO use binpath once https://github.com/pypa/pip/pull/3934 is approved
binpath = os.path.join(stagingdir, "bin")
if context and config.debug("package_release"):
buf = StringIO()
print("\n\npackage download environment:", file=buf)
context.print_info(buf)
_log(buf.getvalue())
# Build pip commandline
cmd = [py_exe, "-m", "pip", "install"]
_extra_args = extra_args or config.pip_extra_args or []
if "--no-use-pep517" not in _extra_args:
cmd.append("--use-pep517")
if not _option_present(_extra_args, "-t", "--target"):
cmd.append("--target=%s" % destpath)
if mode == InstallMode.no_deps and "--no-deps" not in _extra_args:
cmd.append("--no-deps")
cmd.extend(_extra_args)
cmd.append(source_name)
# run pip
_cmd(context=context, command=cmd)
# determine version of python in use
if context is None:
# since we had to use system pip, we have to assume system python version
py_ver_str = '.'.join(map(str, sys.version_info))
py_ver = Version(py_ver_str)
else:
python_variant = context.get_resolved_package("python")
py_ver = python_variant.version
# moving bin folder to expected relative location as per wheel RECORD files
staged_binpath = os.path.join(destpath, "bin")
if os.path.isdir(staged_binpath):
shutil.move(os.path.join(destpath, "bin"), binpath)
# Collect resulting python packages using distlib
distribution_path = DistributionPath([destpath])
distributions = list(distribution_path.get_distributions())
dist_names = [x.name for x in distributions]
# get list of package and dependencies
for distribution in distributions:
# convert pip requirements into rez requirements
rez_requires = get_rez_requirements(installed_dist=distribution,
python_version=py_ver,
name_casings=dist_names)
# log the pip -> rez translation, for debugging
_log("Pip to rez translation information for " +
distribution.name_and_version + ":\n" +
pformat({
"pip": {
"run_requires": map(str, distribution.run_requires)
},
"rez": rez_requires
}))
# iterate over installed files and determine dest filepaths
tools = []
src_dst_lut = {}
for installed_file in distribution.list_installed_files():
# distlib expects the script files to be located in ../../bin/
# when in fact ../bin seems to be the resulting path after the
# installation as such we need to point the bin files to the
# expected location to match wheel RECORD files
installed_filepath = os.path.normpath(installed_file[0])
bin_prefix = os.path.join('..', '..', 'bin') + os.sep
if installed_filepath.startswith(bin_prefix):
# account for extra parentdir as explained above
installed = os.path.join(destpath, '_', installed_filepath)
else:
installed = os.path.join(destpath, installed_filepath)
source_file = os.path.normpath(installed)
if os.path.exists(source_file):
destination_file = os.path.relpath(source_file, stagingdir)
exe = False
if is_exe(source_file) and destination_file.startswith("bin" +
os.sep):
_file = os.path.basename(destination_file)
tools.append(_file)
exe = True
src_dst_lut[source_file] = [destination_file, exe]
else:
_log("Source file does not exist: " + source_file + "!")
def make_root(variant, path):
"""Using distlib to iterate over all installed files of the current
distribution to copy files to the target directory of the rez package
variant
"""
for source_file, data in src_dst_lut.items():
destination_file, exe = data
destination_file = os.path.normpath(
os.path.join(path, destination_file))
if not os.path.exists(os.path.dirname(destination_file)):
os.makedirs(os.path.dirname(destination_file))
shutil.copyfile(source_file, destination_file)
if exe:
shutil.copystat(source_file, destination_file)
# create the rez package
name = pip_to_rez_package_name(distribution.name)
version = pip_to_rez_version(distribution.version)
requires = rez_requires["requires"]
variant_requires = rez_requires["variant_requires"]
metadata = rez_requires["metadata"]
with make_package(name, packages_path, make_root=make_root) as pkg:
# basics (version etc)
pkg.version = version
if distribution.metadata.summary:
pkg.description = distribution.metadata.summary
# requirements and variants
if requires:
pkg.requires = requires
if variant_requires:
pkg.variants = [variant_requires]
# commands
commands = []
commands.append("env.PYTHONPATH.append('{root}/python')")
if tools:
pkg.tools = tools
commands.append("env.PATH.append('{root}/bin')")
pkg.commands = '\n'.join(commands)
# Make the package use hashed variants. This is required because we
# can't control what ends up in its variants, and that can easily
# include problematic chars (>, +, ! etc).
# TODO: https://github.com/nerdvegas/rez/issues/672
#
pkg.hashed_variants = True
# add some custom attributes to retain pip-related info
pkg.pip_name = distribution.name_and_version
pkg.from_pip = True
pkg.is_pure_python = metadata["is_pure_python"]
installed_variants.extend(pkg.installed_variants or [])
skipped_variants.extend(pkg.skipped_variants or [])
# cleanup
shutil.rmtree(tmpdir)
return installed_variants, skipped_variants
def _info(msg, *nargs):
if verbose:
print_info(msg, *nargs)
def copy_package(package,
dest_repository,
variants=None,
shallow=False,
dest_name=None,
dest_version=None,
overwrite=False,
force=False,
follow_symlinks=False,
dry_run=False,
keep_timestamp=False,
skip_payload=False,
overrides=None,
verbose=False):
"""Copy a package from one package repository to another.
This copies the package definition and payload. The package can also be
re-named and/or re-versioned using the `dest_name` and `dest_version` args.
The result is a dict describing which package variants were and were not
copied. For example:
{
"copied": [
(`Variant`, `Variant`)
],
"skipped": [
(`Variant`, `Variant`)
]
}
Each 2-tuple in the 'copied' or 'skipped' list contains the source and
destination variant respectively. In the 'skipped' list, the source variant
is the variant that was NOT copied, and the dest variant is the existing
target variant that caused the source not to be copied. Skipped variants
will only be present when `overwrite` is False.
Note:
Whether or not a package can be copied is determined by its 'relocatable'
attribute (see the `default_relocatable` config setting for more details).
An attempt to copy a non-relocatable package will fail. You can override
this behaviour with the `force` argument.
Args:
package (`Package`): Package to copy.
dest_repository (`PackageRepository` or str): The package repository, or
a package repository path, to copy the package into.
variants (list of int): Indexes of variants to build, or all if None.
shallow (bool): If True, symlinks of each variant's root directory are
created, rather than the payload being copied.
dest_name (str): If provided, copy the package to a new package name.
dest_version (str or `Version`): If provided, copy the package to a new
version.
overwrite (bool): Overwrite variants if they already exist in the
destination package. In this case, the existing payload is removed
before the new payload is copied.
force (bool): Copy the package regardless of its relocatable attribute.
Use at your own risk (there is no guarantee the resulting package
will be functional).
follow_symlinks (bool): Follow symlinks when copying package payload,
rather than copying the symlinks themselves.
keep_timestamp (bool): By default, a newly copied package will get a
new timestamp (because that's when it was added to the target repo).
By setting this option to True, the original package's timestamp
is kept intact. Note that this will have no effect if variant(s)
are copied into an existing package.
skip_payload (bool): If True, do not copy the package payload.
overrides (dict): See `PackageRepository.install_variant`.
verbose (bool): Verbose mode.
dry_run (bool): Dry run mode. Dest variants in the result will be None
in this case.
Returns:
Dict: See comments above.
"""
copied = []
skipped = []
def finalize():
return {"copied": copied, "skipped": skipped}
# check that package is relocatable
if not force and not skip_payload and not package.is_relocatable:
raise PackageCopyError("Cannot copy non-relocatable package: %s" %
package.uri)
if isinstance(dest_repository, basestring):
repo_path = dest_repository
dest_pkg_repo = package_repository_manager.get_repository(repo_path)
else:
dest_pkg_repo = dest_repository
# cannot copy package over the top of itself
if package.repository == dest_pkg_repo and \
(dest_name is None or dest_name == package.name) and \
(dest_version is None or str(dest_version) == str(package.version)):
raise PackageCopyError("Cannot copy package over itself: %s." %
package.uri)
# determine variants to potentially install
src_variants = []
for variant in package.iter_variants():
if variants is None or variant.index in variants:
src_variants.append(variant)
if not src_variants:
return finalize()
# Construct overrides.
#
overrides = (overrides or {}).copy()
if dest_name:
overrides["name"] = dest_name
if dest_version:
overrides["version"] = dest_version
# Find variants that already exist in the dest package, and remove them
# from the copy candidates if overwriting is disabled.
#
new_src_variants = []
for src_variant in src_variants:
existing_variant = dest_pkg_repo.install_variant(src_variant.resource,
overrides=overrides,
dry_run=True)
if existing_variant:
if overwrite:
if verbose:
print_info("Source variant %s will overwrite %s",
src_variant.uri, existing_variant.uri)
else:
if verbose:
print_info(
"Skipping source variant %s - already exists in "
"destination package at %s", src_variant.uri,
existing_variant.uri)
skipped.append((src_variant, existing_variant))
continue
new_src_variants.append(src_variant)
src_variants = new_src_variants
# Install each variant and associated payload.
#
for i, src_variant in enumerate(src_variants):
if verbose:
print_info("Copying source variant %s into repository %s...",
src_variant.uri, str(dest_pkg_repo))
if dry_run:
dest_variant = None
else:
if not skip_payload:
# Perform pre-install steps. For eg, a "building" marker file is
# created in the filesystem pkg repo, so that the package dir
# (which doesn't have variants copied into it yet) is not picked
# up as a valid package.
#
dest_pkg_repo.pre_variant_install(src_variant.resource)
# copy include modules before the first variant install
if i == 0:
_copy_package_include_modules(src_variant.parent,
dest_pkg_repo,
overrides=overrides)
# copy the variant's payload
_copy_variant_payload(src_variant=src_variant,
dest_pkg_repo=dest_pkg_repo,
shallow=shallow,
follow_symlinks=follow_symlinks,
overrides=overrides,
verbose=verbose)
# construct overrides
overrides_ = overrides.copy()
if not keep_timestamp and "timestamp" not in overrides:
overrides_["timestamp"] = int(time.time())
# install the variant into the package definition
dest_variant = dest_pkg_repo.install_variant(
variant_resource=src_variant.resource, overrides=overrides_)
if verbose:
print_info("Copied source variant %s to target variant %s",
src_variant, dest_variant)
copied.append((src_variant, dest_variant))
return finalize()
def _copy_variant_payload(src_variant,
dest_pkg_repo,
shallow=False,
follow_symlinks=False,
overrides=None,
verbose=False):
# Get payload path of source variant. For some types (eg from a "memory"
# type repo) there may not be a root.
#
variant_root = getattr(src_variant, "root", None)
if not variant_root:
raise PackageCopyError(
"Cannot copy source variant %s - it is a type of variant that "
"does not have a root.", src_variant.uri)
if not os.path.isdir(variant_root):
raise PackageCopyError(
"Cannot copy source variant %s - its root does not appear to "
"be present on disk (%s).", src_variant.uri, variant_root)
dest_variant_name = overrides.get("name") or src_variant.name
dest_variant_version = overrides.get("version") or src_variant.version
# determine variant installation path
dest_pkg_payload_path = dest_pkg_repo.get_package_payload_path(
package_name=dest_variant_name, package_version=dest_variant_version)
if src_variant.subpath:
variant_install_path = os.path.join(dest_pkg_payload_path,
src_variant.subpath)
else:
variant_install_path = dest_pkg_payload_path
# get ready for copy/symlinking
copy_func = partial(replacing_copy, follow_symlinks=follow_symlinks)
if shallow:
maybe_symlink = replacing_symlink
else:
maybe_symlink = copy_func
# possibly make install path temporarily writable
last_dir = get_existing_path(
variant_install_path,
topmost_path=os.path.dirname(dest_pkg_payload_path))
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
# copy the variant payload
with ctxt:
safe_makedirs(variant_install_path)
# determine files not to copy
skip_files = []
if src_variant.subpath:
# Detect overlapped variants. This is the case where one variant subpath
# might be A, and another is A/B. We must ensure that A/B is not created
# as a symlink during shallow install of variant A - that would then
# cause A/B payload to be installed back into original package, possibly
# corrupting it.
#
# Here we detect this case, and create a list of dirs not to copy/link,
# because they are in fact a subpath dir for another variant.
#
skip_files.extend(_get_overlapped_variant_dirs(src_variant))
else:
# just skip package definition file
for name in config.plugins.package_repository.filesystem.package_filenames:
for fmt in (FileFormat.py, FileFormat.yaml):
filename = name + '.' + fmt.extension
skip_files.append(filename)
# copy/link all topmost files within the variant root
for name in os.listdir(variant_root):
if name in skip_files:
filepath = os.path.join(variant_root, name)
if verbose:
if src_variant.subpath:
msg = ("Did not copy %s - this is part of an "
"overlapping variant's root path.")
else:
msg = "Did not copy package definition file %s"
print_info(msg, filepath)
continue
src_path = os.path.join(variant_root, name)
dest_path = os.path.join(variant_install_path, name)
if os.path.islink(src_path):
copy_func(src_path, dest_path)
else:
maybe_symlink(src_path, dest_path)
# copy permissions of source variant dirs onto dest
src_package = src_variant.parent
src_pkg_repo = src_package.repository
src_pkg_payload_path = src_pkg_repo.get_package_payload_path(
package_name=src_package.name, package_version=src_package.version)
shutil.copystat(src_pkg_payload_path, dest_pkg_payload_path)
subpath = src_variant.subpath
while subpath:
src_path = os.path.join(src_pkg_payload_path, subpath)
dest_path = os.path.join(dest_pkg_payload_path, subpath)
shutil.copystat(src_path, dest_path)
subpath = os.path.dirname(subpath)
def _get_preprocessed(self, data):
"""
Returns:
(DeveloperPackage, new_data) 2-tuple IFF the preprocess function
changed the package; otherwise None.
"""
from rez.serialise import process_python_objects
from rez.utils.data_utils import get_dict_diff_str
from copy import deepcopy
with add_sys_paths(config.package_definition_build_python_paths):
preprocess_func = getattr(self, "preprocess", None)
if preprocess_func:
print_info("Applying preprocess from package.py")
else:
# load globally configured preprocess function
dotted = self.config.package_preprocess_function
if not dotted:
return None
if '.' not in dotted:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. Package "
"preprocessing has not been applied.")
return None
name, funcname = dotted.rsplit('.', 1)
try:
module = __import__(name=name, fromlist=[funcname])
except Exception as e:
print_error("Failed to load preprocessing function '%s': %s"
% (dotted, str(e)))
return None
setattr(module, "InvalidPackageError", InvalidPackageError)
preprocess_func = getattr(module, funcname)
if not preprocess_func or not isfunction(isfunction):
print_error("Function '%s' not found" % dotted)
return None
print_info("Applying preprocess function %s" % dotted)
preprocessed_data = deepcopy(data)
# apply preprocessing
try:
preprocess_func(this=self, data=preprocessed_data)
except InvalidPackageError:
raise
except Exception as e:
print_error("Failed to apply preprocess: %s: %s"
% (e.__class__.__name__, str(e)))
return None
# if preprocess added functions, these may need to be converted to
# SourceCode instances
preprocessed_data = process_python_objects(preprocessed_data)
if preprocessed_data == data:
return None
# recreate package from modified package data
package = create_package(self.name, preprocessed_data,
package_cls=self.__class__)
# print summary of changed package attributes
txt = get_dict_diff_str(
data,
preprocessed_data,
title="Package attributes were changed in preprocessing:"
)
print_info(txt)
return package, preprocessed_data
def _get_preprocessed(self, data):
"""
Returns:
(DeveloperPackage, new_data) 2-tuple IF the preprocess function
changed the package; otherwise None.
"""
from rez.serialise import process_python_objects
from rez.utils.data_utils import get_dict_diff_str
from copy import deepcopy
def _get_package_level():
return getattr(self, "preprocess", None)
def _get_global_level():
# load globally configured preprocess function
package_preprocess_function = self.config.package_preprocess_function
if not package_preprocess_function:
return None
elif isfunction(package_preprocess_function):
preprocess_func = package_preprocess_function
else:
if '.' not in package_preprocess_function:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. Package "
"preprocessing has not been applied.")
return None
elif isinstance(package_preprocess_function, basestring):
if '.' not in package_preprocess_function:
print_error(
"Setting 'package_preprocess_function' must be of "
"form 'module[.module.module...].funcname'. "
"Package preprocessing has not been applied.")
return None
name, funcname = package_preprocess_function.rsplit('.', 1)
try:
module = __import__(name=name, fromlist=[funcname])
except Exception as e:
print_error(
"Failed to load preprocessing function '%s': %s" %
(package_preprocess_function, str(e)))
return None
setattr(module, "InvalidPackageError", InvalidPackageError)
preprocess_func = getattr(module, funcname)
else:
print_error("Invalid package_preprocess_function: %s" %
package_preprocess_function)
return None
if not preprocess_func or not isfunction(preprocess_func):
print_error("Function '%s' not found" %
package_preprocess_function)
return None
return preprocess_func
with add_sys_paths(config.package_definition_build_python_paths):
preprocess_mode = PreprocessMode[
self.config.package_preprocess_mode]
package_preprocess = _get_package_level()
global_preprocess = _get_global_level()
if preprocess_mode == PreprocessMode.after:
preprocessors = [global_preprocess, package_preprocess]
elif preprocess_mode == PreprocessMode.before:
preprocessors = [package_preprocess, global_preprocess]
else:
preprocessors = [package_preprocess or global_preprocess]
preprocessed_data = deepcopy(data)
for preprocessor in preprocessors:
if not preprocessor:
continue
level = "global" if preprocessor == global_preprocess else "local"
print_info("Applying {0} preprocess function".format(level))
# apply preprocessing
try:
preprocessor(this=self, data=preprocessed_data)
except InvalidPackageError:
raise
except Exception as e:
print_error("Failed to apply preprocess: %s: %s" %
(e.__class__.__name__, str(e)))
return None
# if preprocess added functions, these may need to be converted to
# SourceCode instances
preprocessed_data = process_python_objects(preprocessed_data)
if preprocessed_data == data:
return None
# recreate package from modified package data
package = create_package(self.name,
preprocessed_data,
package_cls=self.__class__)
# print summary of changed package attributes
txt = get_dict_diff_str(
data,
preprocessed_data,
title="Package attributes were changed in preprocessing:")
print_info(txt)
return package, preprocessed_data
def copy_package(package, dest_repository, variants=None, shallow=False,
dest_name=None, dest_version=None, overwrite=False, force=False,
follow_symlinks=False, dry_run=False, keep_timestamp=False,
skip_payload=False, overrides=None, verbose=False):
"""Copy a package from one package repository to another.
This copies the package definition and payload. The package can also be
re-named and/or re-versioned using the `dest_name` and `dest_version` args.
The result is a dict describing which package variants were and were not
copied. For example:
{
"copied": [
(`Variant`, `Variant`)
],
"skipped": [
(`Variant`, `Variant`)
]
}
Each 2-tuple in the 'copied' or 'skipped' list contains the source and
destination variant respectively. In the 'skipped' list, the source variant
is the variant that was NOT copied, and the dest variant is the existing
target variant that caused the source not to be copied. Skipped variants
will only be present when `overwrite` is False.
Note:
Whether or not a package can be copied is determined by its 'relocatable'
attribute (see the `default_relocatable` config setting for more details).
An attempt to copy a non-relocatable package will fail. You can override
this behaviour with the `force` argument.
Args:
package (`Package`): Package to copy.
dest_repository (`PackageRepository` or str): The package repository, or
a package repository path, to copy the package into.
variants (list of int): Indexes of variants to build, or all if None.
shallow (bool): If True, symlinks of each variant's root directory are
created, rather than the payload being copied.
dest_name (str): If provided, copy the package to a new package name.
dest_version (str or `Version`): If provided, copy the package to a new
version.
overwrite (bool): Overwrite variants if they already exist in the
destination package. In this case, the existing payload is removed
before the new payload is copied.
force (bool): Copy the package regardless of its relocatable attribute.
Use at your own risk (there is no guarantee the resulting package
will be functional).
follow_symlinks (bool): Follow symlinks when copying package payload,
rather than copying the symlinks themselves.
keep_timestamp (bool): By default, a newly copied package will get a
new timestamp (because that's when it was added to the target repo).
By setting this option to True, the original package's timestamp
is kept intact. Note that this will have no effect if variant(s)
are copied into an existing package.
skip_payload (bool): If True, do not copy the package payload.
overrides (dict): See `PackageRepository.install_variant`.
verbose (bool): Verbose mode.
dry_run (bool): Dry run mode. Dest variants in the result will be None
in this case.
Returns:
Dict: See comments above.
"""
copied = []
skipped = []
def finalize():
return {
"copied": copied,
"skipped": skipped
}
# check that package is relocatable
if not force and not skip_payload and not package.is_relocatable:
raise PackageCopyError(
"Cannot copy non-relocatable package: %s" % package.uri
)
if isinstance(dest_repository, basestring):
repo_path = dest_repository
dest_pkg_repo = package_repository_manager.get_repository(repo_path)
else:
dest_pkg_repo = dest_repository
# cannot copy package over the top of itself
if package.repository == dest_pkg_repo and \
(dest_name is None or dest_name == package.name) and \
(dest_version is None or str(dest_version) == str(package.version)):
raise PackageCopyError(
"Cannot copy package over itself: %s." % package.uri
)
# determine variants to potentially install
src_variants = []
for variant in package.iter_variants():
if variants is None or variant.index in variants:
src_variants.append(variant)
if not src_variants:
return finalize()
# Construct overrides.
#
overrides = (overrides or {}).copy()
if dest_name:
overrides["name"] = dest_name
if dest_version:
overrides["version"] = dest_version
# Find variants that already exist in the dest package, and remove them
# from the copy candidates if overwriting is disabled.
#
new_src_variants = []
for src_variant in src_variants:
existing_variant = dest_pkg_repo.install_variant(
src_variant.resource,
overrides=overrides,
dry_run=True
)
if existing_variant:
if overwrite:
if verbose:
print_info("Source variant %s will overwrite %s",
src_variant.uri, existing_variant.uri)
else:
if verbose:
print_info(
"Skipping source variant %s - already exists in "
"destination package at %s",
src_variant.uri, existing_variant.uri
)
skipped.append((src_variant, existing_variant))
continue
new_src_variants.append(src_variant)
src_variants = new_src_variants
# Install each variant and associated payload.
#
for i, src_variant in enumerate(src_variants):
if verbose:
print_info("Copying source variant %s into repository %s...",
src_variant.uri, str(dest_pkg_repo))
if dry_run:
dest_variant = None
else:
if not skip_payload:
# Perform pre-install steps. For eg, a "building" marker file is
# created in the filesystem pkg repo, so that the package dir
# (which doesn't have variants copied into it yet) is not picked
# up as a valid package.
#
dest_pkg_repo.pre_variant_install(src_variant.resource)
# copy include modules before the first variant install
if i == 0:
_copy_package_include_modules(
src_variant.parent,
dest_pkg_repo,
overrides=overrides
)
# copy the variant's payload
_copy_variant_payload(
src_variant=src_variant,
dest_pkg_repo=dest_pkg_repo,
shallow=shallow,
follow_symlinks=follow_symlinks,
overrides=overrides,
verbose=verbose
)
# construct overrides
overrides_ = overrides.copy()
if not keep_timestamp and "timestamp" not in overrides:
overrides_["timestamp"] = int(time.time())
# install the variant into the package definition
dest_variant = dest_pkg_repo.install_variant(
variant_resource=src_variant.resource,
overrides=overrides_
)
if verbose:
print_info("Copied source variant %s to target variant %s",
src_variant, dest_variant)
copied.append((src_variant, dest_variant))
return finalize()
