def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path variant_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.subpath: variant_build_path = os.path.join(variant_build_path, variant.subpath) variant_install_path = os.path.join(variant_install_path, variant.subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # create build environment context, rxt_filepath = self.create_build_context( variant=variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # install some files for debugging purposes extra_files = build_result.get("extra_files", []) + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) return build_result
def install(self, path, dry_run=False, overrides=None): """Install this variant into another package repository. If the package already exists, this variant will be correctly merged into the package. If the variant already exists in this package, the existing variant is returned. Args: path (str): Path to destination package repository. dry_run (bool): If True, do not actually install the variant. In this mode, a `Variant` instance is only returned if the equivalent variant already exists in this repository; otherwise, None is returned. overrides (dict): Use this to change or add attributes to the installed variant. Returns: `Variant` object - the (existing or newly created) variant in the specified repository. If `dry_run` is True, None may be returned. """ repo = package_repository_manager.get_repository(path) resource = repo.install_variant(self.resource, dry_run=dry_run, overrides=overrides) if resource is None: return None elif resource is self.resource: return self else: return Variant(resource)
def _find_in_path(path): repo = package_repository_manager.get_repository(path) variant_resource = repo.get_variant_from_uri(uri) if variant_resource is not None: return Variant(variant_resource) else: return None
def test_intersects_resolve(self): """Test intersects with resolve object""" resolved_pkg_data = { "foo": { "1": { "name": "foo", "version": "1" } }, "maya": { "2020.1": { "name": "maya", "version": "2020.1" } }, } mem_path = "memory@%s" % hex(id(resolved_pkg_data)) resolved_repo = package_repository_manager.get_repository(mem_path) resolved_repo.data = resolved_pkg_data resolved_packages = [ variant for family in iter_package_families(paths=[mem_path]) for package in family.iter_packages() for variant in package.iter_variants() ] resolve = VariantsBinding(resolved_packages) self.assertTrue(intersects(resolve.foo, "1")) self.assertFalse(intersects(resolve.foo, "0")) self.assertTrue(intersects(resolve.maya, "2019+")) self.assertFalse(intersects(resolve.maya, "<=2019"))
def remove_packages_ignored_since(days, paths=None, dry_run=False, verbose=False): """Remove packages ignored for >= specified number of days. Args: days (int): Remove packages ignored >= this many days paths (list of str, optional): Paths to search for packages, defaults to `config.packages_path`. dry_run: Dry run mode verbose (bool): Verbose mode Returns: int: Number of packages removed. In dry-run mode, returns the number of packages that _would_ be removed. """ num_removed = 0 for path in (paths or config.packages_path): repo = package_repository_manager.get_repository(path) if verbose: print_info("Searching %s...", repo) num_removed += repo.remove_ignored_since(days=days, dry_run=dry_run, verbose=verbose) return num_removed
def _find_in_path(path): repo = package_repository_manager.get_repository(path) pkg_resource = repo.get_package_from_uri(uri) if pkg_resource is not None: return Package(pkg_resource) else: return None
def _get_families(name, paths=None): entries = [] for path in (paths or config.packages_path): repo = package_repository_manager.get_repository(path) family_resource = repo.get_package_family(name) if family_resource: entries.append((repo, family_resource)) return entries
def list_repos(): from rez.config import config from rez.package_repository import package_repository_manager print("No action taken. Run again, and set PATH to one of:") for path in config.packages_path: repo = package_repository_manager.get_repository(path) print(str(repo))
def clear_caches(self, location=None): """Clear repository cache for current session to spot new package :param location: One single package path to clear. Clear cache of all paths (`packages_path`) if not given. :type location: str or None :return: None """ paths = [location] if location else self._paths for path in paths: repo = package_repository_manager.get_repository(path) repo.clear_caches()
def get_package_install_path(self, path): """Return the installation path for a package (where its payload goes). Args: path (str): Package repository path. """ from rez.package_repository import package_repository_manager pkg_repo = package_repository_manager.get_repository(path) return pkg_repo.get_package_payload_path( package_name=self.package.name, package_version=self.package.version)
def get_package_install_path(self, path): """Return the installation path for a package (where its payload goes). Args: path (str): Package repository path. """ from rez.package_repository import package_repository_manager pkg_repo = package_repository_manager.get_repository(path) return pkg_repo.get_package_payload_path( package_name=self.package.name, package_version=self.package.version )
def remove_package_family(name, path, force=False): """Remove a package family from its repository. A family can only be deleted if it contains no packages, hidden or otherwise, unless `force` is True. Args: name (str): Name of package family. path (str): Package repository path containing the package family. force (bool): If True, delete family even if not empty. Returns: bool: True if the package family was removed, False if not found. """ repo = package_repository_manager.get_repository(path) return repo.remove_package_family(name, force=force)
def iter_package_families(paths=None): """Iterate over package families, in no particular order. Note that multiple package families with the same name can be returned. Unlike packages, families later in the searchpath are not hidden by earlier families. Args: paths (list of str, optional): paths to search for package families, defaults to `config.packages_path`. Returns: `PackageFamily` iterator. """ for path in (paths or config.packages_path): repo = package_repository_manager.get_repository(path) for resource in repo.iter_package_families(): yield PackageFamily(resource)
def _memcache_key(self, timestamped=False): """Makes a key suitable as a memcache entry.""" request = tuple(map(str, self.package_requests)) repo_ids = [] for path in self.package_paths: repo = package_repository_manager.get_repository(path) repo_ids.append(repo.uid) t = [ "resolve", request, tuple(repo_ids), self.package_filter_hash, self.building, config.prune_failed_graph ] if timestamped and self.timestamp: t.append(self.timestamp) return str(tuple(t))
def _memcache_key(self, timestamped=False): """Makes a key suitable as a memcache entry.""" request = tuple(map(str, self.package_requests)) repo_ids = [] for path in self.package_paths: repo = package_repository_manager.get_repository(path) repo_ids.append(repo.uid) t = ["resolve", request, tuple(repo_ids), self.package_filter_hash, self.building, config.prune_failed_graph] if timestamped and self.timestamp: t.append(self.timestamp) return str(tuple(t))
def remove_package(name, version, path): """Remove a package from its repository. Note that you are able to remove a package that is hidden (ie ignored). This is why a Package instance is not specified (if the package were hidden, you wouldn't be able to get one). Args: name (str): Name of package. version (Version or str): Version of the package, eg '1.0.0' path (str): Package repository path containing the package. Returns: bool: True if the package was removed, False if package not found. """ if isinstance(version, basestring): version = Version(version) repo = package_repository_manager.get_repository(path) return repo.remove_package(name, version)
def list_repos_containing_pkg(pkg_name, pkg_version): from rez.config import config from rez.package_repository import package_repository_manager import sys # search for package in each searchpath matching_repos = [] for path in config.packages_path: repo = package_repository_manager.get_repository(path) if repo.get_package(pkg_name, pkg_version): matching_repos.append(repo) if matching_repos: print("No action taken. Run again, and set PATH to one of:") for repo in matching_repos: print(str(repo)) else: print("Package not found.", file=sys.stderr) sys.exit(1)
def get_package_from_repository(name, version, path): """Get a package from a repository. Args: name (str): Name of the package, eg 'maya'. version (Version or str): Version of the package, eg '1.0.0' Returns: `Package` object, or None if the package was not found. """ repo = package_repository_manager.get_repository(path) if isinstance(version, basestring): version = Version(version) package_resource = repo.get_package(name, version) if package_resource is None: return None return Package(package_resource)
def test_package_remove(self): """Test package remove.""" pkg_name = "pydad" pkg_version = Version("2") # copy packages to a temp repo repo_path = os.path.join(self.root, "tmp4_packages") shutil.copytree(self.solver_packages_path, repo_path) # verify that source pkg exists src_pkg = get_package_from_repository(pkg_name, pkg_version, repo_path) self.assertNotEqual(src_pkg, None) # remove it was_removed = remove_package(pkg_name, pkg_version, repo_path) self.assertTrue(was_removed) # verify that source pkg no longer exists (and isn't simply ignored) repo = package_repository_manager.get_repository(repo_path) i = repo.unignore_package(pkg_name, pkg_version) self.assertEqual(i, -1)
def test_remove_packages_ignored_since(self): pkg_name = "pydad" pkg_version = Version("2") # copy packages to a temp repo repo_path = os.path.join(self.root, "tmp5_packages") shutil.copytree(self.solver_packages_path, repo_path) # verify that source pkg exists src_pkg = get_package_from_repository(pkg_name, pkg_version, repo_path) self.assertNotEqual(src_pkg, None) # ignore it repo = package_repository_manager.get_repository(repo_path) i = repo.ignore_package(pkg_name, pkg_version) self.assertEqual(i, 1) # remove all ignored packages num_removed = remove_packages_ignored_since(days=0, paths=[repo_path]) self.assertEqual(num_removed, 1) # verify that source pkg no longer exists (and isn't simply ignored) i = repo.unignore_package(pkg_name, pkg_version) self.assertEqual(i, -1)
def command(opts, parser, extra_arg_groups=None): from rez.package_repository import package_repository_manager from rez.vendor.version.requirement import VersionedObject import sys obj = VersionedObject(opts.PKG) if opts.PATH is None: if opts.allow_missing: list_repos() else: list_repos_containing_pkg(obj.name, obj.version) sys.exit(0) repo = package_repository_manager.get_repository(opts.PATH) if opts.unignore: i = repo.unignore_package(obj.name, obj.version) else: i = repo.ignore_package(obj.name, obj.version, allow_missing=opts.allow_missing) if i == 1: if opts.unignore: print("Package is now visible to resolves once more") else: print("Package is now ignored and will not be visible to resolves") elif i == 0: if opts.unignore: print("No action taken - package was already visible") else: print("No action taken - package was already ignored") else: print("Package not found", file=sys.stderr) sys.exit(1)
def move_package(package, dest_repository, keep_timestamp=False, force=False, verbose=False): """Move a package. Moving a package means copying the package to a destination repo, and ignoring (ie hiding - not removing) the source package. The package must not already exist in the destination repo. Args: package (`Package`): Package to move. dest_repository (`PackageRepository` or str): The package repository, or a package repository path, to move the package into. keep_timestamp (bool): By default, a newly copied package will get a new timestamp (because that's when it was added to the target repo). By setting this option to True, the original package's timestamp is kept intact. force (bool): Move the package regardless of its relocatable attribute. Use at your own risk (there is no guarantee the resulting package will be functional). verbose (bool): Verbose mode. Returns: `Package`: The newly created package in the destination repo. """ def _info(msg, *nargs): if verbose: print_info(msg, *nargs) # get dest repo if isinstance(dest_repository, basestring): repo_path = dest_repository dest_pkg_repo = package_repository_manager.get_repository(repo_path) else: dest_pkg_repo = dest_repository # check that the package doesn't already exist in the dest repo pkg = dest_pkg_repo.get_package(package.name, package.version) if pkg: raise PackageMoveError("Package already exists at destination: %s" % pkg.uri) # move the pkg as atomically as possible: # # 1. Hide the dest package (even tho it doesn't exist yet) # 2. Copy the package # 3. Unhide the dest package # 4. Hide the src package # # 1. dest_pkg_repo.ignore_package(package.name, package.version, allow_missing=True) _info("Ignored %s in %s ahead of time", package.qualified_name, dest_pkg_repo) try: # 2. result = copy_package(package=package, dest_repository=dest_pkg_repo, force=force, keep_timestamp=keep_timestamp, verbose=verbose) finally: # 3. dest_pkg_repo.unignore_package(package.name, package.version) _info("Unignored %s in %s", package.qualified_name, dest_pkg_repo) # 4. package.repository.ignore_package(package.name, package.version) _info("Ignored %s", package.uri) # finish up a_dest_variant = result["copied"][0][1] dest_pkg = a_dest_variant.parent _info("Package %s moved to %s", package.uri, dest_pkg.uri) return dest_pkg
def clear_caches(): for path in config.packages_path: repo = package_repository_manager.get_repository(path) repo.clear_caches()
def is_local(self): """Returns True if the package is in the local package repository""" local_repo = package_repository_manager.get_repository( self.config.local_packages_path) return (self.resource._repository.uid == local_repo.uid)
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path variant_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.subpath: variant_build_path = os.path.join(variant_build_path, variant.subpath) variant_install_path = os.path.join(variant_install_path, variant.subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant(variant.index) # create build environment context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # install some files for debugging purposes extra_files = build_result.get("extra_files", []) if rxt_filepath: extra_files = extra_files + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def copy_package(package, dest_repository, variants=None, shallow=False, dest_name=None, dest_version=None, overwrite=False, force=False, follow_symlinks=False, dry_run=False, keep_timestamp=False, skip_payload=False, overrides=None, verbose=False): """Copy a package from one package repository to another. This copies the package definition and payload. The package can also be re-named and/or re-versioned using the `dest_name` and `dest_version` args. The result is a dict describing which package variants were and were not copied. For example: { "copied": [ (`Variant`, `Variant`) ], "skipped": [ (`Variant`, `Variant`) ] } Each 2-tuple in the 'copied' or 'skipped' list contains the source and destination variant respectively. In the 'skipped' list, the source variant is the variant that was NOT copied, and the dest variant is the existing target variant that caused the source not to be copied. Skipped variants will only be present when `overwrite` is False. Note: Whether or not a package can be copied is determined by its 'relocatable' attribute (see the `default_relocatable` config setting for more details). An attempt to copy a non-relocatable package will fail. You can override this behaviour with the `force` argument. Args: package (`Package`): Package to copy. dest_repository (`PackageRepository` or str): The package repository, or a package repository path, to copy the package into. variants (list of int): Indexes of variants to build, or all if None. shallow (bool): If True, symlinks of each variant's root directory are created, rather than the payload being copied. dest_name (str): If provided, copy the package to a new package name. dest_version (str or `Version`): If provided, copy the package to a new version. overwrite (bool): Overwrite variants if they already exist in the destination package. In this case, the existing payload is removed before the new payload is copied. force (bool): Copy the package regardless of its relocatable attribute. Use at your own risk (there is no guarantee the resulting package will be functional). follow_symlinks (bool): Follow symlinks when copying package payload, rather than copying the symlinks themselves. keep_timestamp (bool): By default, a newly copied package will get a new timestamp (because that's when it was added to the target repo). By setting this option to True, the original package's timestamp is kept intact. Note that this will have no effect if variant(s) are copied into an existing package. skip_payload (bool): If True, do not copy the package payload. overrides (dict): See `PackageRepository.install_variant`. verbose (bool): Verbose mode. dry_run (bool): Dry run mode. Dest variants in the result will be None in this case. Returns: Dict: See comments above. """ copied = [] skipped = [] def finalize(): return { "copied": copied, "skipped": skipped } # check that package is relocatable if not force and not skip_payload and not package.is_relocatable: raise PackageCopyError( "Cannot copy non-relocatable package: %s" % package.uri ) if isinstance(dest_repository, basestring): repo_path = dest_repository dest_pkg_repo = package_repository_manager.get_repository(repo_path) else: dest_pkg_repo = dest_repository # cannot copy package over the top of itself if package.repository == dest_pkg_repo and \ (dest_name is None or dest_name == package.name) and \ (dest_version is None or str(dest_version) == str(package.version)): raise PackageCopyError( "Cannot copy package over itself: %s." % package.uri ) # determine variants to potentially install src_variants = [] for variant in package.iter_variants(): if variants is None or variant.index in variants: src_variants.append(variant) if not src_variants: return finalize() # Construct overrides. # overrides = (overrides or {}).copy() if dest_name: overrides["name"] = dest_name if dest_version: overrides["version"] = dest_version # Find variants that already exist in the dest package, and remove them # from the copy candidates if overwriting is disabled. # new_src_variants = [] for src_variant in src_variants: existing_variant = dest_pkg_repo.install_variant( src_variant.resource, overrides=overrides, dry_run=True ) if existing_variant: if overwrite: if verbose: print_info("Source variant %s will overwrite %s", src_variant.uri, existing_variant.uri) else: if verbose: print_info( "Skipping source variant %s - already exists in " "destination package at %s", src_variant.uri, existing_variant.uri ) skipped.append((src_variant, existing_variant)) continue new_src_variants.append(src_variant) src_variants = new_src_variants # Install each variant and associated payload. # for i, src_variant in enumerate(src_variants): if verbose: print_info("Copying source variant %s into repository %s...", src_variant.uri, str(dest_pkg_repo)) if dry_run: dest_variant = None else: if not skip_payload: # Perform pre-install steps. For eg, a "building" marker file is # created in the filesystem pkg repo, so that the package dir # (which doesn't have variants copied into it yet) is not picked # up as a valid package. # dest_pkg_repo.pre_variant_install(src_variant.resource) # copy include modules before the first variant install if i == 0: _copy_package_include_modules( src_variant.parent, dest_pkg_repo, overrides=overrides ) # copy the variant's payload _copy_variant_payload( src_variant=src_variant, dest_pkg_repo=dest_pkg_repo, shallow=shallow, follow_symlinks=follow_symlinks, overrides=overrides, verbose=verbose ) # construct overrides overrides_ = overrides.copy() if not keep_timestamp and "timestamp" not in overrides: overrides_["timestamp"] = int(time.time()) # install the variant into the package definition dest_variant = dest_pkg_repo.install_variant( variant_resource=src_variant.resource, overrides=overrides_ ) if verbose: print_info("Copied source variant %s to target variant %s", src_variant, dest_variant) copied.append((src_variant, dest_variant)) return finalize()
def command(opts, parser, extra_arg_groups=None): import os import sys from rez.config import config from rez.package_repository import package_repository_manager from rez.package_copy import copy_package from rez.utils.formatting import PackageRequest from rez.packages_ import iter_packages if (not opts.dest_path) and not (opts.rename or opts.reversion): parser.error("--dest-path must be specified unless --rename or " "--reversion are used.") # Load the source package. # if opts.paths: paths = opts.paths.split(os.pathsep) paths = [x for x in paths if x] elif opts.no_local: paths = config.nonlocal_packages_path else: paths = None req = PackageRequest(opts.PKG) it = iter_packages( name=req.name, range_=req.range_, paths=paths ) src_pkgs = list(it) if not src_pkgs: print >> sys.stderr, "No matching packages found." sys.exit(1) if len(src_pkgs) > 1: print >> sys.stderr, "More than one package matches, please choose:" for pkg in sorted(src_pkgs, key=lambda x: x.version): print >> sys.stderr, pkg.qualified_name sys.exit(1) src_pkg = src_pkgs[0] # Determine repo and perform checks. # # A common mistake may be to specify a dest package path, rather than the # _repo_ path. This would cause a mess, since a package would be installed # into a nested location within an existing package. # if opts.dest_path: dest_pkg_repo = package_repository_manager.get_repository(opts.dest_path) if (not opts.allow_empty) and dest_pkg_repo.is_empty(): print >> sys.stderr, ( "Attempting to copy a package into an EMPTY repository. Are you " "sure that --dest-path is the correct path? This should not " "include package name and/or version." "\n\n" "If this is a valid new package repository, use the " "--allow-empty flag to continue." ) sys.exit(1) else: dest_pkg_repo = src_pkg.repository # Perform the copy. # variants = opts.variants or None result = copy_package( package=src_pkg, dest_repository=dest_pkg_repo, dest_name=opts.rename, dest_version=opts.reversion, variants=variants, overwrite=opts.overwrite, shallow=opts.shallow, follow_symlinks=opts.follow_symlinks, keep_timestamp=opts.keep_timestamp, force=opts.force, verbose=opts.verbose, dry_run=opts.dry_run ) # Print info about the result. # copied = result["copied"] skipped = result["skipped"] if opts.dry_run: # show a good indication of target variant when it doesn't get created path = dest_pkg_repo.get_package_payload_path( package_name=opts.rename or src_pkg.name, package_version=opts.reversion or src_pkg.version ) dry_run_uri = path + "/?" verb = "would be" else: verb = "were" # specific output for non-varianted packages if src_pkg.num_variants == 0: if copied: dest_pkg = copied[0][1].parent print("Copied %s to %s" % (src_pkg.uri, dest_pkg.uri)) else: assert skipped dest_pkg = skipped[0][1].parent print( "Target package already exists: %s. Use 'overwrite' to replace it." % dest_pkg.uri ) # varianted package else: if copied: print("%d variants %s copied:" % (len(copied), verb)) for src_variant, dest_variant in copied: # None possible if dry_run if dest_variant is None: dest_uri = dry_run_uri else: dest_uri = dest_variant.uri print(" %s -> %s" % (src_variant.uri, dest_uri)) if skipped: print("%d variants %s skipped (target exists):" % (len(skipped), verb)) for src_variant, dest_variant in skipped: print(" %s !-> %s" % (src_variant.uri, dest_variant.uri))
def __init__(self, path): assert path.startswith("memory@") self._repo = prm.get_repository(path) # type: MemoryPackageRepository self._path = path
def _repository_uids(self): uids = set() for path in self.paths: repo = package_repository_manager.get_repository(path) uids.add(repo.uid) return uids
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path package_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.index is None: variant_install_path = package_install_path else: subpath = variant._non_shortlinked_subpath variant_build_path = os.path.join(variant_build_path, subpath) variant_install_path = os.path.join(package_install_path, subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): shutil.rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository(install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # if hashed variants are enabled, create the variant shortlink if variant.parent.hashed_variants: try: # create the dir containing all shortlinks base_shortlinks_path = os.path.join( package_install_path, variant.parent.config.variant_shortlinks_dirname ) safe_makedirs(base_shortlinks_path) # create the shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink( base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e ) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant(variant.index) # create build environment context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): raise BuildError("The %s build system failed." % build_system_name) if install: # Install the 'variant.json' file, which identifies which variant # this is. This is important for hashed variants, where it is not # obvious which variant is in which root path. The file is there # for debugging purposes only. # if variant.index is not None: data = { "index": variant.index, "data": variant.parent.data["variants"][variant.index] } filepath = os.path.join(variant_install_path, "variant.json") with open(filepath, 'w') as f: json.dump(data, f, indent=2) # install some files for debugging purposes (incl build.rxt) extra_files = build_result.get("extra_files", []) if rxt_filepath: extra_files = extra_files + [rxt_filepath] for file_ in extra_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def command(opts, parser, extra_arg_groups=None): import os import sys from rez.config import config from rez.package_repository import package_repository_manager from rez.package_copy import copy_package from rez.utils.formatting import PackageRequest from rez.packages_ import iter_packages if (not opts.dest_path) and not (opts.rename or opts.reversion): parser.error("--dest-path must be specified unless --rename or " "--reversion are used.") # Load the source package. # if opts.paths: paths = opts.paths.split(os.pathsep) paths = [x for x in paths if x] elif opts.no_local: paths = config.nonlocal_packages_path else: paths = None req = PackageRequest(opts.PKG) it = iter_packages(name=req.name, range_=req.range_, paths=paths) src_pkgs = list(it) if not src_pkgs: print >> sys.stderr, "No matching packages found." sys.exit(1) if len(src_pkgs) > 1: print >> sys.stderr, "More than one package matches, please choose:" for pkg in sorted(src_pkgs, key=lambda x: x.version): print >> sys.stderr, pkg.qualified_name sys.exit(1) src_pkg = src_pkgs[0] # Determine repo and perform checks. # # A common mistake may be to specify a dest package path, rather than the # _repo_ path. This would cause a mess, since a package would be installed # into a nested location within an existing package. # if opts.dest_path: dest_pkg_repo = package_repository_manager.get_repository( opts.dest_path) if (not opts.allow_empty) and dest_pkg_repo.is_empty(): print >> sys.stderr, ( "Attempting to copy a package into an EMPTY repository. Are you " "sure that --dest-path is the correct path? This should not " "include package name and/or version." "\n\n" "If this is a valid new package repository, use the " "--allow-empty flag to continue.") sys.exit(1) else: dest_pkg_repo = src_pkg.repository # Perform the copy. # variants = opts.variants or None result = copy_package(package=src_pkg, dest_repository=dest_pkg_repo, dest_name=opts.rename, dest_version=opts.reversion, variants=variants, overwrite=opts.overwrite, shallow=opts.shallow, keep_timestamp=opts.keep_timestamp, force=opts.force, verbose=opts.verbose, dry_run=opts.dry_run) # Print info about the result. # copied = result["copied"] skipped = result["skipped"] if opts.dry_run: # show a good indication of target variant when it doesn't get created path = dest_pkg_repo.get_package_payload_path( package_name=opts.rename or src_pkg.name, package_version=opts.reversion or src_pkg.version) dry_run_uri = path + "/?" verb = "would be" else: verb = "were" # specific output for non-varianted packages if src_pkg.num_variants == 0: if copied: dest_pkg = copied[0][1].parent print("Copied %s to %s" % (src_pkg.uri, dest_pkg.uri)) else: assert skipped dest_pkg = skipped[0][1].parent print( "Target package already exists: %s. Use 'overwrite' to replace it." % dest_pkg.uri) # varianted package else: if copied: print("%d variants %s copied:" % (len(copied), verb)) for src_variant, dest_variant in copied: # None possible if dry_run if dest_variant is None: dest_uri = dry_run_uri else: dest_uri = dest_variant.uri print(" %s -> %s" % (src_variant.uri, dest_uri)) if skipped: print("%d variants %s skipped (target exists):" % (len(skipped), verb)) for src_variant, dest_variant in skipped: print(" %s !-> %s" % (src_variant.uri, dest_variant.uri))
def copy_package(package, dest_repository, variants=None, shallow=False, dest_name=None, dest_version=None, overwrite=False, force=False, follow_symlinks=False, dry_run=False, keep_timestamp=False, skip_payload=False, overrides=None, verbose=False): """Copy a package from one package repository to another. This copies the package definition and payload. The package can also be re-named and/or re-versioned using the `dest_name` and `dest_version` args. The result is a dict describing which package variants were and were not copied. For example: { "copied": [ (`Variant`, `Variant`) ], "skipped": [ (`Variant`, `Variant`) ] } Each 2-tuple in the 'copied' or 'skipped' list contains the source and destination variant respectively. In the 'skipped' list, the source variant is the variant that was NOT copied, and the dest variant is the existing target variant that caused the source not to be copied. Skipped variants will only be present when `overwrite` is False. Note: Whether or not a package can be copied is determined by its 'relocatable' attribute (see the `default_relocatable` config setting for more details). An attempt to copy a non-relocatable package will fail. You can override this behaviour with the `force` argument. Args: package (`Package`): Package to copy. dest_repository (`PackageRepository` or str): The package repository, or a package repository path, to copy the package into. variants (list of int): Indexes of variants to build, or all if None. shallow (bool): If True, symlinks of each variant's root directory are created, rather than the payload being copied. dest_name (str): If provided, copy the package to a new package name. dest_version (str or `Version`): If provided, copy the package to a new version. overwrite (bool): Overwrite variants if they already exist in the destination package. In this case, the existing payload is removed before the new payload is copied. force (bool): Copy the package regardless of its relocatable attribute. Use at your own risk (there is no guarantee the resulting package will be functional). follow_symlinks (bool): Follow symlinks when copying package payload, rather than copying the symlinks themselves. keep_timestamp (bool): By default, a newly copied package will get a new timestamp (because that's when it was added to the target repo). By setting this option to True, the original package's timestamp is kept intact. Note that this will have no effect if variant(s) are copied into an existing package. skip_payload (bool): If True, do not copy the package payload. overrides (dict): See `PackageRepository.install_variant`. verbose (bool): Verbose mode. dry_run (bool): Dry run mode. Dest variants in the result will be None in this case. Returns: Dict: See comments above. """ copied = [] skipped = [] def finalize(): return {"copied": copied, "skipped": skipped} # check that package is relocatable if not force and not skip_payload and not package.is_relocatable: raise PackageCopyError("Cannot copy non-relocatable package: %s" % package.uri) if isinstance(dest_repository, basestring): repo_path = dest_repository dest_pkg_repo = package_repository_manager.get_repository(repo_path) else: dest_pkg_repo = dest_repository # cannot copy package over the top of itself if package.repository == dest_pkg_repo and \ (dest_name is None or dest_name == package.name) and \ (dest_version is None or str(dest_version) == str(package.version)): raise PackageCopyError("Cannot copy package over itself: %s." % package.uri) # determine variants to potentially install src_variants = [] for variant in package.iter_variants(): if variants is None or variant.index in variants: src_variants.append(variant) if not src_variants: return finalize() # Construct overrides. # overrides = (overrides or {}).copy() if dest_name: overrides["name"] = dest_name if dest_version: overrides["version"] = dest_version # Find variants that already exist in the dest package, and remove them # from the copy candidates if overwriting is disabled. # new_src_variants = [] for src_variant in src_variants: existing_variant = dest_pkg_repo.install_variant(src_variant.resource, overrides=overrides, dry_run=True) if existing_variant: if overwrite: if verbose: print_info("Source variant %s will overwrite %s", src_variant.uri, existing_variant.uri) else: if verbose: print_info( "Skipping source variant %s - already exists in " "destination package at %s", src_variant.uri, existing_variant.uri) skipped.append((src_variant, existing_variant)) continue new_src_variants.append(src_variant) src_variants = new_src_variants # Install each variant and associated payload. # for i, src_variant in enumerate(src_variants): if verbose: print_info("Copying source variant %s into repository %s...", src_variant.uri, str(dest_pkg_repo)) if dry_run: dest_variant = None else: if not skip_payload: # Perform pre-install steps. For eg, a "building" marker file is # created in the filesystem pkg repo, so that the package dir # (which doesn't have variants copied into it yet) is not picked # up as a valid package. # dest_pkg_repo.pre_variant_install(src_variant.resource) # copy include modules before the first variant install if i == 0: _copy_package_include_modules(src_variant.parent, dest_pkg_repo, overrides=overrides) # copy the variant's payload _copy_variant_payload(src_variant=src_variant, dest_pkg_repo=dest_pkg_repo, shallow=shallow, follow_symlinks=follow_symlinks, overrides=overrides, verbose=verbose) # construct overrides overrides_ = overrides.copy() if not keep_timestamp and "timestamp" not in overrides: overrides_["timestamp"] = int(time.time()) # install the variant into the package definition dest_variant = dest_pkg_repo.install_variant( variant_resource=src_variant.resource, overrides=overrides_) if verbose: print_info("Copied source variant %s to target variant %s", src_variant, dest_variant) copied.append((src_variant, dest_variant)) return finalize()
def _build_variant_base(self, variant, build_type, install_path=None, clean=False, install=False, **kwargs): # create build/install paths install_path = install_path or self.package.config.local_packages_path package_install_path = self.get_package_install_path(install_path) variant_build_path = self.build_path if variant.index is None: variant_install_path = package_install_path else: subpath = variant._non_shortlinked_subpath variant_build_path = os.path.join(variant_build_path, subpath) variant_install_path = os.path.join(package_install_path, subpath) # create directories (build, install) if clean and os.path.exists(variant_build_path): self._rmtree(variant_build_path) safe_makedirs(variant_build_path) # find last dir of installation path that exists, and possibly make it # writable during variant installation # last_dir = get_existing_path(variant_install_path, topmost_path=install_path) if last_dir: ctxt = make_path_writable(last_dir) else: ctxt = with_noop() with ctxt: if install: # inform package repo that a variant is about to be built/installed pkg_repo = package_repository_manager.get_repository( install_path) pkg_repo.pre_variant_install(variant.resource) if not os.path.exists(variant_install_path): safe_makedirs(variant_install_path) # if hashed variants are enabled, create the variant shortlink if variant.parent.hashed_variants: try: # create the dir containing all shortlinks base_shortlinks_path = os.path.join( package_install_path, variant.parent.config.variant_shortlinks_dirname) safe_makedirs(base_shortlinks_path) # create the shortlink rel_variant_path = os.path.relpath( variant_install_path, base_shortlinks_path) create_unique_base26_symlink(base_shortlinks_path, rel_variant_path) except Exception as e: # Treat any error as warning - lack of shortlink is not # a breaking issue, it just means the variant root path # will be long. # print_warning( "Error creating variant shortlink for %s: %s: %s", variant_install_path, e.__class__.__name__, e) # Re-evaluate the variant, so that variables such as 'building' and # 'build_variant_index' are set, and any early-bound package attribs # are re-evaluated wrt these vars. This is done so that attribs such as # 'requires' can change depending on whether a build is occurring or not. # # Note that this re-evaluated variant is ONLY used here, for the purposes # of creating the build context. The variant that is actually installed # is the one evaluated where 'building' is False. # re_evaluated_package = variant.parent.get_reevaluated({ "building": True, "build_variant_index": variant.index or 0, "build_variant_requires": variant.variant_requires }) re_evaluated_variant = re_evaluated_package.get_variant( variant.index) # create build environment (also creates build.rxt file) context, rxt_filepath = self.create_build_context( variant=re_evaluated_variant, build_type=build_type, build_path=variant_build_path) # list of extra files (build.rxt etc) that are installed if an # installation is taking place # extra_install_files = [rxt_filepath] # create variant.json file. This identifies which variant this is. # This is important for hashed variants, where it is not obvious # which variant is in which root path. The file is there for # debugging purposes only. # if variant.index is not None: data = { "index": variant.index, "data": variant.parent.data["variants"][variant.index] } filepath = os.path.join(variant_build_path, "variant.json") extra_install_files.append(filepath) with open(filepath, 'w') as f: json.dump(data, f, indent=2) # run build system build_system_name = self.build_system.name() self._print("\nInvoking %s build system...", build_system_name) build_result = self.build_system.build( context=context, variant=variant, build_path=variant_build_path, install_path=variant_install_path, install=install, build_type=build_type) if not build_result.get("success"): # delete the possibly partially installed variant payload if install: self._rmtree(variant_install_path) raise BuildError("The %s build system failed." % build_system_name) if install: # add some installation details to build result build_result.update({ "package_install_path": package_install_path, "variant_install_path": variant_install_path }) # the build system can also specify extra files that need to # be installed filepaths = build_result.get("extra_files") if filepaths: extra_install_files.extend(filepaths) # install extra files for file_ in extra_install_files: copy_or_replace(file_, variant_install_path) # Install include modules. Note that this doesn't need to be done # multiple times, but for subsequent variants it has no effect. # self._install_include_modules(install_path) return build_result
def cancel_variant_install(): pkg_repo = package_repository_manager.get_repository(release_path) pkg_repo.on_variant_install_cancelled(variant.resource)