def _build_variant_base(self,
variant,
build_type,
install_path=None,
clean=False,
install=False,
**kwargs):
# create build/install paths
install_path = install_path or self.package.config.local_packages_path
variant_install_path = self.get_package_install_path(install_path)
variant_build_path = self.build_path
if variant.subpath:
variant_build_path = os.path.join(variant_build_path,
variant.subpath)
variant_install_path = os.path.join(variant_install_path,
variant.subpath)
# create directories (build, install)
if clean and os.path.exists(variant_build_path):
shutil.rmtree(variant_build_path)
safe_makedirs(variant_build_path)
if install:
# inform package repo that a variant is about to be built/installed
pkg_repo = package_repository_manager.get_repository(install_path)
pkg_repo.pre_variant_install(variant.resource)
if not os.path.exists(variant_install_path):
safe_makedirs(variant_install_path)
# create build environment
context, rxt_filepath = self.create_build_context(
variant=variant,
build_type=build_type,
build_path=variant_build_path)
# run build system
build_system_name = self.build_system.name()
self._print("\nInvoking %s build system...", build_system_name)
build_result = self.build_system.build(
context=context,
variant=variant,
build_path=variant_build_path,
install_path=variant_install_path,
install=install,
build_type=build_type)
if not build_result.get("success"):
raise BuildError("The %s build system failed." % build_system_name)
if install:
# install some files for debugging purposes
extra_files = build_result.get("extra_files", []) + [rxt_filepath]
for file_ in extra_files:
copy_or_replace(file_, variant_install_path)
return build_result
def install(self, path, dry_run=False, overrides=None):
"""Install this variant into another package repository.
If the package already exists, this variant will be correctly merged
into the package. If the variant already exists in this package, the
existing variant is returned.
Args:
path (str): Path to destination package repository.
dry_run (bool): If True, do not actually install the variant. In this
mode, a `Variant` instance is only returned if the equivalent
variant already exists in this repository; otherwise, None is
returned.
overrides (dict): Use this to change or add attributes to the
installed variant.
Returns:
`Variant` object - the (existing or newly created) variant in the
specified repository. If `dry_run` is True, None may be returned.
"""
repo = package_repository_manager.get_repository(path)
resource = repo.install_variant(self.resource,
dry_run=dry_run,
overrides=overrides)
if resource is None:
return None
elif resource is self.resource:
return self
else:
return Variant(resource)
def _find_in_path(path):
repo = package_repository_manager.get_repository(path)
variant_resource = repo.get_variant_from_uri(uri)
if variant_resource is not None:
return Variant(variant_resource)
else:
return None
def test_intersects_resolve(self):
"""Test intersects with resolve object"""
resolved_pkg_data = {
"foo": {
"1": {
"name": "foo",
"version": "1"
}
},
"maya": {
"2020.1": {
"name": "maya",
"version": "2020.1"
}
},
}
mem_path = "memory@%s" % hex(id(resolved_pkg_data))
resolved_repo = package_repository_manager.get_repository(mem_path)
resolved_repo.data = resolved_pkg_data
resolved_packages = [
variant for family in iter_package_families(paths=[mem_path])
for package in family.iter_packages()
for variant in package.iter_variants()
]
resolve = VariantsBinding(resolved_packages)
self.assertTrue(intersects(resolve.foo, "1"))
self.assertFalse(intersects(resolve.foo, "0"))
self.assertTrue(intersects(resolve.maya, "2019+"))
self.assertFalse(intersects(resolve.maya, "<=2019"))
def remove_packages_ignored_since(days,
paths=None,
dry_run=False,
verbose=False):
"""Remove packages ignored for >= specified number of days.
Args:
days (int): Remove packages ignored >= this many days
paths (list of str, optional): Paths to search for packages, defaults
to `config.packages_path`.
dry_run: Dry run mode
verbose (bool): Verbose mode
Returns:
int: Number of packages removed. In dry-run mode, returns the number of
packages that _would_ be removed.
"""
num_removed = 0
for path in (paths or config.packages_path):
repo = package_repository_manager.get_repository(path)
if verbose:
print_info("Searching %s...", repo)
num_removed += repo.remove_ignored_since(days=days,
dry_run=dry_run,
verbose=verbose)
return num_removed
def _find_in_path(path):
repo = package_repository_manager.get_repository(path)
pkg_resource = repo.get_package_from_uri(uri)
if pkg_resource is not None:
return Package(pkg_resource)
else:
return None
def _get_families(name, paths=None):
entries = []
for path in (paths or config.packages_path):
repo = package_repository_manager.get_repository(path)
family_resource = repo.get_package_family(name)
if family_resource:
entries.append((repo, family_resource))
return entries
def list_repos():
from rez.config import config
from rez.package_repository import package_repository_manager
print("No action taken. Run again, and set PATH to one of:")
for path in config.packages_path:
repo = package_repository_manager.get_repository(path)
print(str(repo))
def clear_caches(self, location=None):
"""Clear repository cache for current session to spot new package
:param location: One single package path to clear. Clear cache of all
paths (`packages_path`) if not given.
:type location: str or None
:return: None
"""
paths = [location] if location else self._paths
for path in paths:
repo = package_repository_manager.get_repository(path)
repo.clear_caches()
def get_package_install_path(self, path):
"""Return the installation path for a package (where its payload goes).
Args:
path (str): Package repository path.
"""
from rez.package_repository import package_repository_manager
pkg_repo = package_repository_manager.get_repository(path)
return pkg_repo.get_package_payload_path(
package_name=self.package.name,
package_version=self.package.version)
def get_package_install_path(self, path):
"""Return the installation path for a package (where its payload goes).
Args:
path (str): Package repository path.
"""
from rez.package_repository import package_repository_manager
pkg_repo = package_repository_manager.get_repository(path)
return pkg_repo.get_package_payload_path(
package_name=self.package.name,
package_version=self.package.version
)
def remove_package_family(name, path, force=False):
"""Remove a package family from its repository.
A family can only be deleted if it contains no packages, hidden or
otherwise, unless `force` is True.
Args:
name (str): Name of package family.
path (str): Package repository path containing the package family.
force (bool): If True, delete family even if not empty.
Returns:
bool: True if the package family was removed, False if not found.
"""
repo = package_repository_manager.get_repository(path)
return repo.remove_package_family(name, force=force)
def iter_package_families(paths=None):
"""Iterate over package families, in no particular order.
Note that multiple package families with the same name can be returned.
Unlike packages, families later in the searchpath are not hidden by earlier
families.
Args:
paths (list of str, optional): paths to search for package families,
defaults to `config.packages_path`.
Returns:
`PackageFamily` iterator.
"""
for path in (paths or config.packages_path):
repo = package_repository_manager.get_repository(path)
for resource in repo.iter_package_families():
yield PackageFamily(resource)
def _memcache_key(self, timestamped=False):
"""Makes a key suitable as a memcache entry."""
request = tuple(map(str, self.package_requests))
repo_ids = []
for path in self.package_paths:
repo = package_repository_manager.get_repository(path)
repo_ids.append(repo.uid)
t = [
"resolve", request,
tuple(repo_ids), self.package_filter_hash, self.building,
config.prune_failed_graph
]
if timestamped and self.timestamp:
t.append(self.timestamp)
return str(tuple(t))
def _memcache_key(self, timestamped=False):
"""Makes a key suitable as a memcache entry."""
request = tuple(map(str, self.package_requests))
repo_ids = []
for path in self.package_paths:
repo = package_repository_manager.get_repository(path)
repo_ids.append(repo.uid)
t = ["resolve",
request,
tuple(repo_ids),
self.package_filter_hash,
self.building,
config.prune_failed_graph]
if timestamped and self.timestamp:
t.append(self.timestamp)
return str(tuple(t))
def remove_package(name, version, path):
"""Remove a package from its repository.
Note that you are able to remove a package that is hidden (ie ignored).
This is why a Package instance is not specified (if the package were hidden,
you wouldn't be able to get one).
Args:
name (str): Name of package.
version (Version or str): Version of the package, eg '1.0.0'
path (str): Package repository path containing the package.
Returns:
bool: True if the package was removed, False if package not found.
"""
if isinstance(version, basestring):
version = Version(version)
repo = package_repository_manager.get_repository(path)
return repo.remove_package(name, version)
def list_repos_containing_pkg(pkg_name, pkg_version):
from rez.config import config
from rez.package_repository import package_repository_manager
import sys
# search for package in each searchpath
matching_repos = []
for path in config.packages_path:
repo = package_repository_manager.get_repository(path)
if repo.get_package(pkg_name, pkg_version):
matching_repos.append(repo)
if matching_repos:
print("No action taken. Run again, and set PATH to one of:")
for repo in matching_repos:
print(str(repo))
else:
print("Package not found.", file=sys.stderr)
sys.exit(1)
def get_package_from_repository(name, version, path):
"""Get a package from a repository.
Args:
name (str): Name of the package, eg 'maya'.
version (Version or str): Version of the package, eg '1.0.0'
Returns:
`Package` object, or None if the package was not found.
"""
repo = package_repository_manager.get_repository(path)
if isinstance(version, basestring):
version = Version(version)
package_resource = repo.get_package(name, version)
if package_resource is None:
return None
return Package(package_resource)
def test_package_remove(self):
"""Test package remove."""
pkg_name = "pydad"
pkg_version = Version("2")
# copy packages to a temp repo
repo_path = os.path.join(self.root, "tmp4_packages")
shutil.copytree(self.solver_packages_path, repo_path)
# verify that source pkg exists
src_pkg = get_package_from_repository(pkg_name, pkg_version, repo_path)
self.assertNotEqual(src_pkg, None)
# remove it
was_removed = remove_package(pkg_name, pkg_version, repo_path)
self.assertTrue(was_removed)
# verify that source pkg no longer exists (and isn't simply ignored)
repo = package_repository_manager.get_repository(repo_path)
i = repo.unignore_package(pkg_name, pkg_version)
self.assertEqual(i, -1)
def test_remove_packages_ignored_since(self):
pkg_name = "pydad"
pkg_version = Version("2")
# copy packages to a temp repo
repo_path = os.path.join(self.root, "tmp5_packages")
shutil.copytree(self.solver_packages_path, repo_path)
# verify that source pkg exists
src_pkg = get_package_from_repository(pkg_name, pkg_version, repo_path)
self.assertNotEqual(src_pkg, None)
# ignore it
repo = package_repository_manager.get_repository(repo_path)
i = repo.ignore_package(pkg_name, pkg_version)
self.assertEqual(i, 1)
# remove all ignored packages
num_removed = remove_packages_ignored_since(days=0, paths=[repo_path])
self.assertEqual(num_removed, 1)
# verify that source pkg no longer exists (and isn't simply ignored)
i = repo.unignore_package(pkg_name, pkg_version)
self.assertEqual(i, -1)
def command(opts, parser, extra_arg_groups=None):
from rez.package_repository import package_repository_manager
from rez.vendor.version.requirement import VersionedObject
import sys
obj = VersionedObject(opts.PKG)
if opts.PATH is None:
if opts.allow_missing:
list_repos()
else:
list_repos_containing_pkg(obj.name, obj.version)
sys.exit(0)
repo = package_repository_manager.get_repository(opts.PATH)
if opts.unignore:
i = repo.unignore_package(obj.name, obj.version)
else:
i = repo.ignore_package(obj.name,
obj.version,
allow_missing=opts.allow_missing)
if i == 1:
if opts.unignore:
print("Package is now visible to resolves once more")
else:
print("Package is now ignored and will not be visible to resolves")
elif i == 0:
if opts.unignore:
print("No action taken - package was already visible")
else:
print("No action taken - package was already ignored")
else:
print("Package not found", file=sys.stderr)
sys.exit(1)
def move_package(package,
dest_repository,
keep_timestamp=False,
force=False,
verbose=False):
"""Move a package.
Moving a package means copying the package to a destination repo, and
ignoring (ie hiding - not removing) the source package. The package must
not already exist in the destination repo.
Args:
package (`Package`): Package to move.
dest_repository (`PackageRepository` or str): The package repository, or
a package repository path, to move the package into.
keep_timestamp (bool): By default, a newly copied package will get a
new timestamp (because that's when it was added to the target repo).
By setting this option to True, the original package's timestamp
is kept intact.
force (bool): Move the package regardless of its relocatable attribute.
Use at your own risk (there is no guarantee the resulting package
will be functional).
verbose (bool): Verbose mode.
Returns:
`Package`: The newly created package in the destination repo.
"""
def _info(msg, *nargs):
if verbose:
print_info(msg, *nargs)
# get dest repo
if isinstance(dest_repository, basestring):
repo_path = dest_repository
dest_pkg_repo = package_repository_manager.get_repository(repo_path)
else:
dest_pkg_repo = dest_repository
# check that the package doesn't already exist in the dest repo
pkg = dest_pkg_repo.get_package(package.name, package.version)
if pkg:
raise PackageMoveError("Package already exists at destination: %s" %
pkg.uri)
# move the pkg as atomically as possible:
#
# 1. Hide the dest package (even tho it doesn't exist yet)
# 2. Copy the package
# 3. Unhide the dest package
# 4. Hide the src package
#
# 1.
dest_pkg_repo.ignore_package(package.name,
package.version,
allow_missing=True)
_info("Ignored %s in %s ahead of time", package.qualified_name,
dest_pkg_repo)
try:
# 2.
result = copy_package(package=package,
dest_repository=dest_pkg_repo,
force=force,
keep_timestamp=keep_timestamp,
verbose=verbose)
finally:
# 3.
dest_pkg_repo.unignore_package(package.name, package.version)
_info("Unignored %s in %s", package.qualified_name, dest_pkg_repo)
# 4.
package.repository.ignore_package(package.name, package.version)
_info("Ignored %s", package.uri)
# finish up
a_dest_variant = result["copied"][0][1]
dest_pkg = a_dest_variant.parent
_info("Package %s moved to %s", package.uri, dest_pkg.uri)
return dest_pkg
def clear_caches():
for path in config.packages_path:
repo = package_repository_manager.get_repository(path)
repo.clear_caches()
def is_local(self):
"""Returns True if the package is in the local package repository"""
local_repo = package_repository_manager.get_repository(
self.config.local_packages_path)
return (self.resource._repository.uid == local_repo.uid)
def _build_variant_base(self, variant, build_type, install_path=None,
clean=False, install=False, **kwargs):
# create build/install paths
install_path = install_path or self.package.config.local_packages_path
variant_install_path = self.get_package_install_path(install_path)
variant_build_path = self.build_path
if variant.subpath:
variant_build_path = os.path.join(variant_build_path, variant.subpath)
variant_install_path = os.path.join(variant_install_path, variant.subpath)
# create directories (build, install)
if clean and os.path.exists(variant_build_path):
shutil.rmtree(variant_build_path)
safe_makedirs(variant_build_path)
# find last dir of installation path that exists, and possibly make it
# writable during variant installation
#
last_dir = get_existing_path(variant_install_path,
topmost_path=install_path)
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
with ctxt:
if install:
# inform package repo that a variant is about to be built/installed
pkg_repo = package_repository_manager.get_repository(install_path)
pkg_repo.pre_variant_install(variant.resource)
if not os.path.exists(variant_install_path):
safe_makedirs(variant_install_path)
# Re-evaluate the variant, so that variables such as 'building' and
# 'build_variant_index' are set, and any early-bound package attribs
# are re-evaluated wrt these vars. This is done so that attribs such as
# 'requires' can change depending on whether a build is occurring or not.
#
# Note that this re-evaluated variant is ONLY used here, for the purposes
# of creating the build context. The variant that is actually installed
# is the one evaluated where 'building' is False.
#
re_evaluated_package = variant.parent.get_reevaluated({
"building": True,
"build_variant_index": variant.index or 0,
"build_variant_requires": variant.variant_requires
})
re_evaluated_variant = re_evaluated_package.get_variant(variant.index)
# create build environment
context, rxt_filepath = self.create_build_context(
variant=re_evaluated_variant,
build_type=build_type,
build_path=variant_build_path)
# run build system
build_system_name = self.build_system.name()
self._print("\nInvoking %s build system...", build_system_name)
build_result = self.build_system.build(
context=context,
variant=variant,
build_path=variant_build_path,
install_path=variant_install_path,
install=install,
build_type=build_type)
if not build_result.get("success"):
raise BuildError("The %s build system failed." % build_system_name)
if install:
# install some files for debugging purposes
extra_files = build_result.get("extra_files", [])
if rxt_filepath:
extra_files = extra_files + [rxt_filepath]
for file_ in extra_files:
copy_or_replace(file_, variant_install_path)
# Install include modules. Note that this doesn't need to be done
# multiple times, but for subsequent variants it has no effect.
#
self._install_include_modules(install_path)
return build_result
def copy_package(package, dest_repository, variants=None, shallow=False,
dest_name=None, dest_version=None, overwrite=False, force=False,
follow_symlinks=False, dry_run=False, keep_timestamp=False,
skip_payload=False, overrides=None, verbose=False):
"""Copy a package from one package repository to another.
This copies the package definition and payload. The package can also be
re-named and/or re-versioned using the `dest_name` and `dest_version` args.
The result is a dict describing which package variants were and were not
copied. For example:
{
"copied": [
(`Variant`, `Variant`)
],
"skipped": [
(`Variant`, `Variant`)
]
}
Each 2-tuple in the 'copied' or 'skipped' list contains the source and
destination variant respectively. In the 'skipped' list, the source variant
is the variant that was NOT copied, and the dest variant is the existing
target variant that caused the source not to be copied. Skipped variants
will only be present when `overwrite` is False.
Note:
Whether or not a package can be copied is determined by its 'relocatable'
attribute (see the `default_relocatable` config setting for more details).
An attempt to copy a non-relocatable package will fail. You can override
this behaviour with the `force` argument.
Args:
package (`Package`): Package to copy.
dest_repository (`PackageRepository` or str): The package repository, or
a package repository path, to copy the package into.
variants (list of int): Indexes of variants to build, or all if None.
shallow (bool): If True, symlinks of each variant's root directory are
created, rather than the payload being copied.
dest_name (str): If provided, copy the package to a new package name.
dest_version (str or `Version`): If provided, copy the package to a new
version.
overwrite (bool): Overwrite variants if they already exist in the
destination package. In this case, the existing payload is removed
before the new payload is copied.
force (bool): Copy the package regardless of its relocatable attribute.
Use at your own risk (there is no guarantee the resulting package
will be functional).
follow_symlinks (bool): Follow symlinks when copying package payload,
rather than copying the symlinks themselves.
keep_timestamp (bool): By default, a newly copied package will get a
new timestamp (because that's when it was added to the target repo).
By setting this option to True, the original package's timestamp
is kept intact. Note that this will have no effect if variant(s)
are copied into an existing package.
skip_payload (bool): If True, do not copy the package payload.
overrides (dict): See `PackageRepository.install_variant`.
verbose (bool): Verbose mode.
dry_run (bool): Dry run mode. Dest variants in the result will be None
in this case.
Returns:
Dict: See comments above.
"""
copied = []
skipped = []
def finalize():
return {
"copied": copied,
"skipped": skipped
}
# check that package is relocatable
if not force and not skip_payload and not package.is_relocatable:
raise PackageCopyError(
"Cannot copy non-relocatable package: %s" % package.uri
)
if isinstance(dest_repository, basestring):
repo_path = dest_repository
dest_pkg_repo = package_repository_manager.get_repository(repo_path)
else:
dest_pkg_repo = dest_repository
# cannot copy package over the top of itself
if package.repository == dest_pkg_repo and \
(dest_name is None or dest_name == package.name) and \
(dest_version is None or str(dest_version) == str(package.version)):
raise PackageCopyError(
"Cannot copy package over itself: %s." % package.uri
)
# determine variants to potentially install
src_variants = []
for variant in package.iter_variants():
if variants is None or variant.index in variants:
src_variants.append(variant)
if not src_variants:
return finalize()
# Construct overrides.
#
overrides = (overrides or {}).copy()
if dest_name:
overrides["name"] = dest_name
if dest_version:
overrides["version"] = dest_version
# Find variants that already exist in the dest package, and remove them
# from the copy candidates if overwriting is disabled.
#
new_src_variants = []
for src_variant in src_variants:
existing_variant = dest_pkg_repo.install_variant(
src_variant.resource,
overrides=overrides,
dry_run=True
)
if existing_variant:
if overwrite:
if verbose:
print_info("Source variant %s will overwrite %s",
src_variant.uri, existing_variant.uri)
else:
if verbose:
print_info(
"Skipping source variant %s - already exists in "
"destination package at %s",
src_variant.uri, existing_variant.uri
)
skipped.append((src_variant, existing_variant))
continue
new_src_variants.append(src_variant)
src_variants = new_src_variants
# Install each variant and associated payload.
#
for i, src_variant in enumerate(src_variants):
if verbose:
print_info("Copying source variant %s into repository %s...",
src_variant.uri, str(dest_pkg_repo))
if dry_run:
dest_variant = None
else:
if not skip_payload:
# Perform pre-install steps. For eg, a "building" marker file is
# created in the filesystem pkg repo, so that the package dir
# (which doesn't have variants copied into it yet) is not picked
# up as a valid package.
#
dest_pkg_repo.pre_variant_install(src_variant.resource)
# copy include modules before the first variant install
if i == 0:
_copy_package_include_modules(
src_variant.parent,
dest_pkg_repo,
overrides=overrides
)
# copy the variant's payload
_copy_variant_payload(
src_variant=src_variant,
dest_pkg_repo=dest_pkg_repo,
shallow=shallow,
follow_symlinks=follow_symlinks,
overrides=overrides,
verbose=verbose
)
# construct overrides
overrides_ = overrides.copy()
if not keep_timestamp and "timestamp" not in overrides:
overrides_["timestamp"] = int(time.time())
# install the variant into the package definition
dest_variant = dest_pkg_repo.install_variant(
variant_resource=src_variant.resource,
overrides=overrides_
)
if verbose:
print_info("Copied source variant %s to target variant %s",
src_variant, dest_variant)
copied.append((src_variant, dest_variant))
return finalize()
def command(opts, parser, extra_arg_groups=None):
import os
import sys
from rez.config import config
from rez.package_repository import package_repository_manager
from rez.package_copy import copy_package
from rez.utils.formatting import PackageRequest
from rez.packages_ import iter_packages
if (not opts.dest_path) and not (opts.rename or opts.reversion):
parser.error("--dest-path must be specified unless --rename or "
"--reversion are used.")
# Load the source package.
#
if opts.paths:
paths = opts.paths.split(os.pathsep)
paths = [x for x in paths if x]
elif opts.no_local:
paths = config.nonlocal_packages_path
else:
paths = None
req = PackageRequest(opts.PKG)
it = iter_packages(
name=req.name,
range_=req.range_,
paths=paths
)
src_pkgs = list(it)
if not src_pkgs:
print >> sys.stderr, "No matching packages found."
sys.exit(1)
if len(src_pkgs) > 1:
print >> sys.stderr, "More than one package matches, please choose:"
for pkg in sorted(src_pkgs, key=lambda x: x.version):
print >> sys.stderr, pkg.qualified_name
sys.exit(1)
src_pkg = src_pkgs[0]
# Determine repo and perform checks.
#
# A common mistake may be to specify a dest package path, rather than the
# _repo_ path. This would cause a mess, since a package would be installed
# into a nested location within an existing package.
#
if opts.dest_path:
dest_pkg_repo = package_repository_manager.get_repository(opts.dest_path)
if (not opts.allow_empty) and dest_pkg_repo.is_empty():
print >> sys.stderr, (
"Attempting to copy a package into an EMPTY repository. Are you "
"sure that --dest-path is the correct path? This should not "
"include package name and/or version."
"\n\n"
"If this is a valid new package repository, use the "
"--allow-empty flag to continue."
)
sys.exit(1)
else:
dest_pkg_repo = src_pkg.repository
# Perform the copy.
#
variants = opts.variants or None
result = copy_package(
package=src_pkg,
dest_repository=dest_pkg_repo,
dest_name=opts.rename,
dest_version=opts.reversion,
variants=variants,
overwrite=opts.overwrite,
shallow=opts.shallow,
follow_symlinks=opts.follow_symlinks,
keep_timestamp=opts.keep_timestamp,
force=opts.force,
verbose=opts.verbose,
dry_run=opts.dry_run
)
# Print info about the result.
#
copied = result["copied"]
skipped = result["skipped"]
if opts.dry_run:
# show a good indication of target variant when it doesn't get created
path = dest_pkg_repo.get_package_payload_path(
package_name=opts.rename or src_pkg.name,
package_version=opts.reversion or src_pkg.version
)
dry_run_uri = path + "/?"
verb = "would be"
else:
verb = "were"
# specific output for non-varianted packages
if src_pkg.num_variants == 0:
if copied:
dest_pkg = copied[0][1].parent
print("Copied %s to %s" % (src_pkg.uri, dest_pkg.uri))
else:
assert skipped
dest_pkg = skipped[0][1].parent
print(
"Target package already exists: %s. Use 'overwrite' to replace it."
% dest_pkg.uri
)
# varianted package
else:
if copied:
print("%d variants %s copied:" % (len(copied), verb))
for src_variant, dest_variant in copied:
# None possible if dry_run
if dest_variant is None:
dest_uri = dry_run_uri
else:
dest_uri = dest_variant.uri
print(" %s -> %s" % (src_variant.uri, dest_uri))
if skipped:
print("%d variants %s skipped (target exists):" % (len(skipped), verb))
for src_variant, dest_variant in skipped:
print(" %s !-> %s" % (src_variant.uri, dest_variant.uri))
def __init__(self, path):
assert path.startswith("memory@")
self._repo = prm.get_repository(path) # type: MemoryPackageRepository
self._path = path
def _repository_uids(self):
uids = set()
for path in self.paths:
repo = package_repository_manager.get_repository(path)
uids.add(repo.uid)
return uids
def _build_variant_base(self, variant, build_type, install_path=None,
clean=False, install=False, **kwargs):
# create build/install paths
install_path = install_path or self.package.config.local_packages_path
package_install_path = self.get_package_install_path(install_path)
variant_build_path = self.build_path
if variant.index is None:
variant_install_path = package_install_path
else:
subpath = variant._non_shortlinked_subpath
variant_build_path = os.path.join(variant_build_path, subpath)
variant_install_path = os.path.join(package_install_path, subpath)
# create directories (build, install)
if clean and os.path.exists(variant_build_path):
shutil.rmtree(variant_build_path)
safe_makedirs(variant_build_path)
# find last dir of installation path that exists, and possibly make it
# writable during variant installation
#
last_dir = get_existing_path(variant_install_path,
topmost_path=install_path)
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
with ctxt:
if install:
# inform package repo that a variant is about to be built/installed
pkg_repo = package_repository_manager.get_repository(install_path)
pkg_repo.pre_variant_install(variant.resource)
if not os.path.exists(variant_install_path):
safe_makedirs(variant_install_path)
# if hashed variants are enabled, create the variant shortlink
if variant.parent.hashed_variants:
try:
# create the dir containing all shortlinks
base_shortlinks_path = os.path.join(
package_install_path,
variant.parent.config.variant_shortlinks_dirname
)
safe_makedirs(base_shortlinks_path)
# create the shortlink
rel_variant_path = os.path.relpath(
variant_install_path, base_shortlinks_path)
create_unique_base26_symlink(
base_shortlinks_path, rel_variant_path)
except Exception as e:
# Treat any error as warning - lack of shortlink is not
# a breaking issue, it just means the variant root path
# will be long.
#
print_warning(
"Error creating variant shortlink for %s: %s: %s",
variant_install_path, e.__class__.__name__, e
)
# Re-evaluate the variant, so that variables such as 'building' and
# 'build_variant_index' are set, and any early-bound package attribs
# are re-evaluated wrt these vars. This is done so that attribs such as
# 'requires' can change depending on whether a build is occurring or not.
#
# Note that this re-evaluated variant is ONLY used here, for the purposes
# of creating the build context. The variant that is actually installed
# is the one evaluated where 'building' is False.
#
re_evaluated_package = variant.parent.get_reevaluated({
"building": True,
"build_variant_index": variant.index or 0,
"build_variant_requires": variant.variant_requires
})
re_evaluated_variant = re_evaluated_package.get_variant(variant.index)
# create build environment
context, rxt_filepath = self.create_build_context(
variant=re_evaluated_variant,
build_type=build_type,
build_path=variant_build_path)
# run build system
build_system_name = self.build_system.name()
self._print("\nInvoking %s build system...", build_system_name)
build_result = self.build_system.build(
context=context,
variant=variant,
build_path=variant_build_path,
install_path=variant_install_path,
install=install,
build_type=build_type)
if not build_result.get("success"):
raise BuildError("The %s build system failed." % build_system_name)
if install:
# Install the 'variant.json' file, which identifies which variant
# this is. This is important for hashed variants, where it is not
# obvious which variant is in which root path. The file is there
# for debugging purposes only.
#
if variant.index is not None:
data = {
"index": variant.index,
"data": variant.parent.data["variants"][variant.index]
}
filepath = os.path.join(variant_install_path, "variant.json")
with open(filepath, 'w') as f:
json.dump(data, f, indent=2)
# install some files for debugging purposes (incl build.rxt)
extra_files = build_result.get("extra_files", [])
if rxt_filepath:
extra_files = extra_files + [rxt_filepath]
for file_ in extra_files:
copy_or_replace(file_, variant_install_path)
# Install include modules. Note that this doesn't need to be done
# multiple times, but for subsequent variants it has no effect.
#
self._install_include_modules(install_path)
return build_result
def command(opts, parser, extra_arg_groups=None):
import os
import sys
from rez.config import config
from rez.package_repository import package_repository_manager
from rez.package_copy import copy_package
from rez.utils.formatting import PackageRequest
from rez.packages_ import iter_packages
if (not opts.dest_path) and not (opts.rename or opts.reversion):
parser.error("--dest-path must be specified unless --rename or "
"--reversion are used.")
# Load the source package.
#
if opts.paths:
paths = opts.paths.split(os.pathsep)
paths = [x for x in paths if x]
elif opts.no_local:
paths = config.nonlocal_packages_path
else:
paths = None
req = PackageRequest(opts.PKG)
it = iter_packages(name=req.name, range_=req.range_, paths=paths)
src_pkgs = list(it)
if not src_pkgs:
print >> sys.stderr, "No matching packages found."
sys.exit(1)
if len(src_pkgs) > 1:
print >> sys.stderr, "More than one package matches, please choose:"
for pkg in sorted(src_pkgs, key=lambda x: x.version):
print >> sys.stderr, pkg.qualified_name
sys.exit(1)
src_pkg = src_pkgs[0]
# Determine repo and perform checks.
#
# A common mistake may be to specify a dest package path, rather than the
# _repo_ path. This would cause a mess, since a package would be installed
# into a nested location within an existing package.
#
if opts.dest_path:
dest_pkg_repo = package_repository_manager.get_repository(
opts.dest_path)
if (not opts.allow_empty) and dest_pkg_repo.is_empty():
print >> sys.stderr, (
"Attempting to copy a package into an EMPTY repository. Are you "
"sure that --dest-path is the correct path? This should not "
"include package name and/or version."
"\n\n"
"If this is a valid new package repository, use the "
"--allow-empty flag to continue.")
sys.exit(1)
else:
dest_pkg_repo = src_pkg.repository
# Perform the copy.
#
variants = opts.variants or None
result = copy_package(package=src_pkg,
dest_repository=dest_pkg_repo,
dest_name=opts.rename,
dest_version=opts.reversion,
variants=variants,
overwrite=opts.overwrite,
shallow=opts.shallow,
keep_timestamp=opts.keep_timestamp,
force=opts.force,
verbose=opts.verbose,
dry_run=opts.dry_run)
# Print info about the result.
#
copied = result["copied"]
skipped = result["skipped"]
if opts.dry_run:
# show a good indication of target variant when it doesn't get created
path = dest_pkg_repo.get_package_payload_path(
package_name=opts.rename or src_pkg.name,
package_version=opts.reversion or src_pkg.version)
dry_run_uri = path + "/?"
verb = "would be"
else:
verb = "were"
# specific output for non-varianted packages
if src_pkg.num_variants == 0:
if copied:
dest_pkg = copied[0][1].parent
print("Copied %s to %s" % (src_pkg.uri, dest_pkg.uri))
else:
assert skipped
dest_pkg = skipped[0][1].parent
print(
"Target package already exists: %s. Use 'overwrite' to replace it."
% dest_pkg.uri)
# varianted package
else:
if copied:
print("%d variants %s copied:" % (len(copied), verb))
for src_variant, dest_variant in copied:
# None possible if dry_run
if dest_variant is None:
dest_uri = dry_run_uri
else:
dest_uri = dest_variant.uri
print(" %s -> %s" % (src_variant.uri, dest_uri))
if skipped:
print("%d variants %s skipped (target exists):" %
(len(skipped), verb))
for src_variant, dest_variant in skipped:
print(" %s !-> %s" % (src_variant.uri, dest_variant.uri))
def copy_package(package,
dest_repository,
variants=None,
shallow=False,
dest_name=None,
dest_version=None,
overwrite=False,
force=False,
follow_symlinks=False,
dry_run=False,
keep_timestamp=False,
skip_payload=False,
overrides=None,
verbose=False):
"""Copy a package from one package repository to another.
This copies the package definition and payload. The package can also be
re-named and/or re-versioned using the `dest_name` and `dest_version` args.
The result is a dict describing which package variants were and were not
copied. For example:
{
"copied": [
(`Variant`, `Variant`)
],
"skipped": [
(`Variant`, `Variant`)
]
}
Each 2-tuple in the 'copied' or 'skipped' list contains the source and
destination variant respectively. In the 'skipped' list, the source variant
is the variant that was NOT copied, and the dest variant is the existing
target variant that caused the source not to be copied. Skipped variants
will only be present when `overwrite` is False.
Note:
Whether or not a package can be copied is determined by its 'relocatable'
attribute (see the `default_relocatable` config setting for more details).
An attempt to copy a non-relocatable package will fail. You can override
this behaviour with the `force` argument.
Args:
package (`Package`): Package to copy.
dest_repository (`PackageRepository` or str): The package repository, or
a package repository path, to copy the package into.
variants (list of int): Indexes of variants to build, or all if None.
shallow (bool): If True, symlinks of each variant's root directory are
created, rather than the payload being copied.
dest_name (str): If provided, copy the package to a new package name.
dest_version (str or `Version`): If provided, copy the package to a new
version.
overwrite (bool): Overwrite variants if they already exist in the
destination package. In this case, the existing payload is removed
before the new payload is copied.
force (bool): Copy the package regardless of its relocatable attribute.
Use at your own risk (there is no guarantee the resulting package
will be functional).
follow_symlinks (bool): Follow symlinks when copying package payload,
rather than copying the symlinks themselves.
keep_timestamp (bool): By default, a newly copied package will get a
new timestamp (because that's when it was added to the target repo).
By setting this option to True, the original package's timestamp
is kept intact. Note that this will have no effect if variant(s)
are copied into an existing package.
skip_payload (bool): If True, do not copy the package payload.
overrides (dict): See `PackageRepository.install_variant`.
verbose (bool): Verbose mode.
dry_run (bool): Dry run mode. Dest variants in the result will be None
in this case.
Returns:
Dict: See comments above.
"""
copied = []
skipped = []
def finalize():
return {"copied": copied, "skipped": skipped}
# check that package is relocatable
if not force and not skip_payload and not package.is_relocatable:
raise PackageCopyError("Cannot copy non-relocatable package: %s" %
package.uri)
if isinstance(dest_repository, basestring):
repo_path = dest_repository
dest_pkg_repo = package_repository_manager.get_repository(repo_path)
else:
dest_pkg_repo = dest_repository
# cannot copy package over the top of itself
if package.repository == dest_pkg_repo and \
(dest_name is None or dest_name == package.name) and \
(dest_version is None or str(dest_version) == str(package.version)):
raise PackageCopyError("Cannot copy package over itself: %s." %
package.uri)
# determine variants to potentially install
src_variants = []
for variant in package.iter_variants():
if variants is None or variant.index in variants:
src_variants.append(variant)
if not src_variants:
return finalize()
# Construct overrides.
#
overrides = (overrides or {}).copy()
if dest_name:
overrides["name"] = dest_name
if dest_version:
overrides["version"] = dest_version
# Find variants that already exist in the dest package, and remove them
# from the copy candidates if overwriting is disabled.
#
new_src_variants = []
for src_variant in src_variants:
existing_variant = dest_pkg_repo.install_variant(src_variant.resource,
overrides=overrides,
dry_run=True)
if existing_variant:
if overwrite:
if verbose:
print_info("Source variant %s will overwrite %s",
src_variant.uri, existing_variant.uri)
else:
if verbose:
print_info(
"Skipping source variant %s - already exists in "
"destination package at %s", src_variant.uri,
existing_variant.uri)
skipped.append((src_variant, existing_variant))
continue
new_src_variants.append(src_variant)
src_variants = new_src_variants
# Install each variant and associated payload.
#
for i, src_variant in enumerate(src_variants):
if verbose:
print_info("Copying source variant %s into repository %s...",
src_variant.uri, str(dest_pkg_repo))
if dry_run:
dest_variant = None
else:
if not skip_payload:
# Perform pre-install steps. For eg, a "building" marker file is
# created in the filesystem pkg repo, so that the package dir
# (which doesn't have variants copied into it yet) is not picked
# up as a valid package.
#
dest_pkg_repo.pre_variant_install(src_variant.resource)
# copy include modules before the first variant install
if i == 0:
_copy_package_include_modules(src_variant.parent,
dest_pkg_repo,
overrides=overrides)
# copy the variant's payload
_copy_variant_payload(src_variant=src_variant,
dest_pkg_repo=dest_pkg_repo,
shallow=shallow,
follow_symlinks=follow_symlinks,
overrides=overrides,
verbose=verbose)
# construct overrides
overrides_ = overrides.copy()
if not keep_timestamp and "timestamp" not in overrides:
overrides_["timestamp"] = int(time.time())
# install the variant into the package definition
dest_variant = dest_pkg_repo.install_variant(
variant_resource=src_variant.resource, overrides=overrides_)
if verbose:
print_info("Copied source variant %s to target variant %s",
src_variant, dest_variant)
copied.append((src_variant, dest_variant))
return finalize()
def _build_variant_base(self,
variant,
build_type,
install_path=None,
clean=False,
install=False,
**kwargs):
# create build/install paths
install_path = install_path or self.package.config.local_packages_path
package_install_path = self.get_package_install_path(install_path)
variant_build_path = self.build_path
if variant.index is None:
variant_install_path = package_install_path
else:
subpath = variant._non_shortlinked_subpath
variant_build_path = os.path.join(variant_build_path, subpath)
variant_install_path = os.path.join(package_install_path, subpath)
# create directories (build, install)
if clean and os.path.exists(variant_build_path):
self._rmtree(variant_build_path)
safe_makedirs(variant_build_path)
# find last dir of installation path that exists, and possibly make it
# writable during variant installation
#
last_dir = get_existing_path(variant_install_path,
topmost_path=install_path)
if last_dir:
ctxt = make_path_writable(last_dir)
else:
ctxt = with_noop()
with ctxt:
if install:
# inform package repo that a variant is about to be built/installed
pkg_repo = package_repository_manager.get_repository(
install_path)
pkg_repo.pre_variant_install(variant.resource)
if not os.path.exists(variant_install_path):
safe_makedirs(variant_install_path)
# if hashed variants are enabled, create the variant shortlink
if variant.parent.hashed_variants:
try:
# create the dir containing all shortlinks
base_shortlinks_path = os.path.join(
package_install_path,
variant.parent.config.variant_shortlinks_dirname)
safe_makedirs(base_shortlinks_path)
# create the shortlink
rel_variant_path = os.path.relpath(
variant_install_path, base_shortlinks_path)
create_unique_base26_symlink(base_shortlinks_path,
rel_variant_path)
except Exception as e:
# Treat any error as warning - lack of shortlink is not
# a breaking issue, it just means the variant root path
# will be long.
#
print_warning(
"Error creating variant shortlink for %s: %s: %s",
variant_install_path, e.__class__.__name__, e)
# Re-evaluate the variant, so that variables such as 'building' and
# 'build_variant_index' are set, and any early-bound package attribs
# are re-evaluated wrt these vars. This is done so that attribs such as
# 'requires' can change depending on whether a build is occurring or not.
#
# Note that this re-evaluated variant is ONLY used here, for the purposes
# of creating the build context. The variant that is actually installed
# is the one evaluated where 'building' is False.
#
re_evaluated_package = variant.parent.get_reevaluated({
"building":
True,
"build_variant_index":
variant.index or 0,
"build_variant_requires":
variant.variant_requires
})
re_evaluated_variant = re_evaluated_package.get_variant(
variant.index)
# create build environment (also creates build.rxt file)
context, rxt_filepath = self.create_build_context(
variant=re_evaluated_variant,
build_type=build_type,
build_path=variant_build_path)
# list of extra files (build.rxt etc) that are installed if an
# installation is taking place
#
extra_install_files = [rxt_filepath]
# create variant.json file. This identifies which variant this is.
# This is important for hashed variants, where it is not obvious
# which variant is in which root path. The file is there for
# debugging purposes only.
#
if variant.index is not None:
data = {
"index": variant.index,
"data": variant.parent.data["variants"][variant.index]
}
filepath = os.path.join(variant_build_path, "variant.json")
extra_install_files.append(filepath)
with open(filepath, 'w') as f:
json.dump(data, f, indent=2)
# run build system
build_system_name = self.build_system.name()
self._print("\nInvoking %s build system...", build_system_name)
build_result = self.build_system.build(
context=context,
variant=variant,
build_path=variant_build_path,
install_path=variant_install_path,
install=install,
build_type=build_type)
if not build_result.get("success"):
# delete the possibly partially installed variant payload
if install:
self._rmtree(variant_install_path)
raise BuildError("The %s build system failed." %
build_system_name)
if install:
# add some installation details to build result
build_result.update({
"package_install_path":
package_install_path,
"variant_install_path":
variant_install_path
})
# the build system can also specify extra files that need to
# be installed
filepaths = build_result.get("extra_files")
if filepaths:
extra_install_files.extend(filepaths)
# install extra files
for file_ in extra_install_files:
copy_or_replace(file_, variant_install_path)
# Install include modules. Note that this doesn't need to be done
# multiple times, but for subsequent variants it has no effect.
#
self._install_include_modules(install_path)
return build_result
def cancel_variant_install():
pkg_repo = package_repository_manager.get_repository(release_path)
pkg_repo.on_variant_install_cancelled(variant.resource)
