def test_2(self): """package iteration.""" all_packages = set() all_fams = iter_package_families() for fam in all_fams: packages = _to_qnames(iter_packages(fam.name)) all_packages.update(packages) self.assertEqual(all_packages, ALL_PACKAGES) res = _to_qnames(iter_packages('nada')) self.assertEqual(res, set(['nada'])) res = _to_qnames(iter_packages('python')) self.assertEqual(res, set(['python-2.5.2', 'python-2.6.0', 'python-2.6.8', 'python-2.7.0'])) res = _to_qnames(iter_packages('pydad', "<3")) self.assertEqual(res, set(['pydad-1', 'pydad-2'])) for fam_name in ALL_FAMILIES: for package in iter_packages(fam_name): family = package.parent self.assertEqual(family.name, fam_name) it = family.iter_packages() self.assertTrue(package in it)
def test_2(self): """package iteration.""" all_packages = set() all_fams = iter_package_families() for fam in all_fams: packages = _to_qnames(iter_packages(fam.name)) all_packages.update(packages) self.assertEqual(all_packages, ALL_PACKAGES) res = _to_qnames(iter_packages('nada')) self.assertEqual(res, set(['nada'])) res = _to_qnames(iter_packages('python')) self.assertEqual( res, set([ 'python-2.5.2', 'python-2.6.0', 'python-2.6.8', 'python-2.7.0' ])) res = _to_qnames(iter_packages('pydad', "<3")) self.assertEqual(res, set(['pydad-1', 'pydad-2'])) for fam_name in ALL_FAMILIES: for package in iter_packages(fam_name): family = package.parent self.assertEqual(family.name, fam_name) it = family.iter_packages() self.assertTrue(package in it)
def _test_reorder(self, orderer, package_name, expected_order): """Ensure ordered order package version as expected.""" it = iter_packages(package_name) descending = sorted(it, key=lambda x: x.version, reverse=True) ordered = orderer.reorder(descending) or descending result = [str(x.version) for x in ordered] self.assertEqual(expected_order, result)
def print_package_versions(self): """Print a list of versions of the package this tool comes from, and indicate which version this tool is from.""" variants = self.context.get_tool_variants(self.tool_name) if variants: if len(variants) > 1: self._print_conflicting(variants) return 1 else: from rez.packages_ import iter_packages variant = iter(variants).next() it = iter_packages(name=variant.name) rows = [] colors = [] for pkg in sorted(it, key=lambda x: x.version, reverse=True): if pkg.version == variant.version: name = "* %s" % pkg.qualified_name col = heading else: name = " %s" % pkg.qualified_name col = local if pkg.is_local else None label = "(local)" if pkg.is_local else "" rows.append((name, pkg.path, label)) colors.append(col) _pr = Printer() for col, line in zip(colors, columnise(rows)): _pr(line, col) return 0
def main(): for package in iter_packages("arnold"): variant = package.get_variant(0) root = get_package_root(variant) if root is None: continue changelog = get_changelog(root) if changelog is None: continue body = get_body(changelog) if body is None: continue try: body = post_process(body) except Exception as e: print "Failed to create", type(e) print e version_root = os.path.join(hugo_arnold_root, str(package.version)) post_path = os.path.join( version_root, "arnold_changelog_{version}.md".format(version=package.version)) title = "Arnold Changelog v{version}".format(version=package.version) write_document( post_path, header.format(title=title, date=datetime.utcnow().replace( tzinfo=simple_utc()).isoformat()), body)
def find_site_python(module_name, paths=None): """Find the rez native python package that contains the given module. This function is used by python 'native' rez installers to find the native rez python package that represents the python installation that this module is installed into. Note: This function is dependent on the behavior found in the python '_native' package found in the 'rez-recipes' repository. Specifically, it expects to find a python package with a '_site_paths' list attribute listing the site directories associated with the python installation. Args: module_name (str): Target python module. paths (list of str, optional): paths to search for packages, defaults to `config.packages_path`. Returns: `Package`: Native python package containing the named module. """ from rez.packages_ import iter_packages import subprocess import ast import os py_cmd = 'import {x}; print({x}.__path__)'.format(x=module_name) p = Popen(["python", "-c", py_cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) out, err = p.communicate() if p.returncode: raise InvalidPackageError( "Failed to find installed python module '%s':\n%s" % (module_name, err)) module_paths = ast.literal_eval(out.strip()) def issubdir(path, parent_path): return path.startswith(parent_path + os.sep) for package in iter_packages("python", paths=paths): if not hasattr(package, "_site_paths"): continue contained = True for module_path in module_paths: if not any(issubdir(module_path, x) for x in package._site_paths): contained = False if contained: return package raise InvalidPackageError( "Failed to find python installation containing the module '%s'. Has " "python been installed as a rez package?" % module_name)
def pre_release(self): # test that the release path exists release_path = self.package.config.release_packages_path if not os.path.exists(release_path): raise ReleaseError("Release path does not exist: %r" % release_path) # test that the repo is in a state to release assert self.vcs self._print("Checking state of repository...") self.vcs.validate_repostate() it = iter_packages(self.package.name, paths=[release_path]) packages = sorted(it, key=lambda x: x.version, reverse=True) # check UUID. This stops unrelated packages that happen to have the same # name, being released as though they are the same package if self.package.uuid and packages: latest_package = packages[0] if latest_package.uuid and latest_package.uuid != self.package.uuid: raise ReleaseError( "Cannot release - the packages are not the same (UUID mismatch)" ) # test that a newer package version hasn't already been released if self.ensure_latest: for package in packages: if package.version > self.package.version: raise ReleaseError( "Cannot release - a newer package version already " "exists (%s)" % package.uri) else: break
def _update_status(self): def _ok(): self._set_style() self.setToolTip("") def _err(msg, color="red"): self._set_style("QLineEdit { border : 2px solid %s;}" % color) self.setToolTip(msg) txt = str(self.text()) if not txt: _ok() return try: req = Requirement(str(txt)) except Exception as e: _err(str(e)) return _ok() if not req.conflict: try: it = iter_packages(name=req.name, range_=req.range, paths=self._paths) pkg = sorted(it, key=lambda x: x.version)[-1] except Exception: _err("cannot find package: %r" % txt, "orange") return if pkg.description: self.setToolTip(pkg.description)
def test_1(self): """Basic release.""" # release should fail because release path does not exist self._setup_release() builder = self._create_builder() with self.assertRaises(ReleaseError): builder.release() # release should work this time os.mkdir(self.install_root) builder.release() # check a file to see the release made it filepath = os.path.join(self.install_root, "foo", "1.0", "data", "data.txt") self.assertTrue(os.path.exists(filepath)) # failed release (same version released again) builder = self._create_builder() num_variants = builder.release() self.assertEqual(num_variants, 0) # update package version and release again self.package_data["version"] = "1.1" self._write_package() builder = self._create_builder() builder.release() # change version to earlier and do failed release attempt self.package_data["version"] = "1.0.1" self._write_package() builder = self._create_builder() with self.assertRaises(ReleaseError): builder.release() # release again, this time allow not latest builder = self._create_builder(ensure_latest=False) builder.release() # change uuid and do failed release attempt self.package_data["version"] = "1.2" self.package_data["uuid"] += "_CHANGED" self._write_package() builder = self._create_builder() with self.assertRaises(ReleaseError): builder.release() # check the vcs contains the tags we expect expected_value = set(["foo-1.0", "foo-1.0.1", "foo-1.1"]) with open(self.stubfile) as f: stub_data = yaml.load(f.read(), Loader=yaml.FullLoader) tags = set(stub_data.get("tags", {}).keys()) self.assertEqual(tags, expected_value) # check the package install path contains the packages we expect it = iter_packages("foo", paths=[self.install_root]) qnames = set(x.qualified_name for x in it) self.assertEqual(qnames, expected_value)
def find_site_python(module_name, paths=None): """Find the rez native python package that contains the given module. This function is used by python 'native' rez installers to find the native rez python package that represents the python installation that this module is installed into. Note: This function is dependent on the behavior found in the python '_native' package found in the 'rez-recipes' repository. Specifically, it expects to find a python package with a '_site_paths' list attribute listing the site directories associated with the python installation. Args: module_name (str): Target python module. paths (list of str, optional): paths to search for packages, defaults to `config.packages_path`. Returns: `Package`: Native python package containing the named module. """ from rez.packages_ import iter_packages import subprocess import ast import os py_cmd = 'import {x}; print {x}.__path__'.format(x=module_name) p = subprocess.Popen(["python", "-c", py_cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if p.returncode: raise InvalidPackageError( "Failed to find installed python module '%s':\n%s" % (module_name, err)) module_paths = ast.literal_eval(out.strip()) def issubdir(path, parent_path): return path.startswith(parent_path + os.sep) for package in iter_packages("python", paths=paths): if not hasattr(package, "_site_paths"): continue contained = True for module_path in module_paths: if not any(issubdir(module_path, x) for x in package._site_paths): contained = False if contained: return package raise InvalidPackageError( "Failed to find python installation containing the module '%s'. Has " "python been installed as a rez package?" % module_name)
def test_1(self): """Basic release.""" # release should fail because release path does not exist self._setup_release() builder = self._create_builder() with self.assertRaises(ReleaseError): builder.release() # release should work this time os.mkdir(self.install_root) builder.release() # check a file to see the release made it filepath = os.path.join(self.install_root, "foo", "1.0", "data", "data.txt") self.assertTrue(os.path.exists(filepath)) # failed release (same version released again) builder = self._create_builder() num_variants = builder.release() self.assertEqual(num_variants, 0) # update package version and release again self.package_data["version"] = "1.1" self._write_package() builder = self._create_builder() builder.release() # change version to earlier and do failed release attempt self.package_data["version"] = "1.0.1" self._write_package() builder = self._create_builder() with self.assertRaises(ReleaseError): builder.release() # release again, this time allow not latest builder = self._create_builder(ensure_latest=False) builder.release() # change uuid and do failed release attempt self.package_data["version"] = "1.2" self.package_data["uuid"] += "_CHANGED" self._write_package() builder = self._create_builder() with self.assertRaises(ReleaseError): builder.release() # check the vcs contains the tags we expect expected_value = set(["foo-1.0", "foo-1.0.1", "foo-1.1"]) with open(self.stubfile) as f: stub_data = yaml.load(f.read()) tags = set(stub_data.get("tags", {}).keys()) self.assertEqual(tags, expected_value) # check the package install path contains the packages we expect it = iter_packages("foo", paths=[self.install_root]) qnames = set(x.qualified_name for x in it) self.assertEqual(qnames, expected_value)
def get_previous_release(self): release_path = self.package.config.release_packages_path it = iter_packages(self.package.name, paths=[release_path]) packages = sorted(it, key=lambda x: x.version, reverse=True) for package in packages: if package.version < self.package.version: return package return None
def __init__(self, package_name, version_range=None, paths=None, verbose=False): """Create a PackageHelp object. Args: package_name (str): Package to search. version_range (`VersionRange`): Versions to search. """ self.package = None self._verbose = verbose self._sections = [] # find latest package with a help entry package = None it = iter_packages(package_name, range_=version_range) packages = sorted(it, key=lambda x: x.version, reverse=True) for package_ in packages: if self._verbose: print "searching for help in %s..." % package_.uri if package_.help: package = package_ break if package: help_ = package.help if isinstance(help_, basestring): sections = [["Help", help_]] elif isinstance(help_, list): sections = help_ if self._verbose: print "found %d help entries in %s." % (len(sections), package.uri) # create string formatter for help entries if package.num_variants == 0: base = package.base root = base else: variant = package.get_variant(0) base = variant.base root = variant.root formatter = scoped_formatter( base=base, root=root, config=config, version=VersionBinding(package.version), system=system) # format sections for section in sections: uri = section[1] uri = convert_old_command_expansions(uri) uri = uri.replace("$BROWSER", "").strip() uri = formatter.format(uri) section[1] = uri self.package = package self._sections = sections
def __init__(self, package_name, version_range=None, paths=None, verbose=False): """Create a PackageHelp object. Args: package_name (str): Package to search. version_range (`VersionRange`): Versions to search. """ self.package = None self._verbose = verbose self._sections = [] # find latest package with a help entry package = None it = iter_packages(package_name, range_=version_range) packages = sorted(it, key=lambda x: x.version, reverse=True) for package_ in packages: if self._verbose: print("searching for help in %s..." % package_.uri) if package_.help: package = package_ break if package: help_ = package.help if isinstance(help_, six.string_types): sections = [["Help", help_]] elif isinstance(help_, list): sections = help_ if self._verbose: print("found %d help entries in %s." % (len(sections), package.uri)) # create string formatter for help entries if package.num_variants == 0: base = package.base root = base else: variant = package.get_variant(0) base = variant.base root = variant.root formatter = scoped_formatter( base=base, root=root, config=config, version=VersionBinding(package.version), system=system) # format sections for section in sections: uri = section[1] uri = convert_old_command_expansions(uri) uri = uri.replace("$BROWSER", "").strip() uri = formatter.format(uri) section[1] = uri self.package = package self._sections = sections
def _test(orderer, package_name, expected_order): it = iter_packages(package_name) descending = sorted(it, key=lambda x: x.version, reverse=True) pod = to_pod(orderer) orderer2 = from_pod(pod) for orderer_ in (orderer, orderer2): ordered = orderer_.reorder(descending) result = [str(x.version) for x in ordered] self.assertEqual(result, expected_order)
def _print_package_info(self, value, buf=sys.stdout, b=False): word = "is also" if b else "is" _pr = Printer(buf) request_str = os.path.basename(value) if request_str != value: return False def _print_package(package): if isinstance(package, Package): name = package.qualified_name else: name = package.qualified_package_name # Variant _pr("Package: %s" % name) path_str = "URI: %s" % package.uri if package.is_local: path_str += " (local)" _pr(path_str) try: req = PackageRequest(request_str) except: return False if req.conflict: return False package_name = req.name version_range = req.range # check for the package in the active context if self.context: variant = self.context.get_resolved_package(package_name) if variant and variant.version in version_range: _pr("'%s' %s a package in the active context:" % (package_name, word)) _print_package(variant) if self.context.load_path: _pr("Context: %s" % self.context.load_path) return True # find the package it = iter_packages(package_name, version_range) packages = sorted(it, key=lambda x: x.version) if packages: txt = "'%s' %s a package. The latest version" % (package_name, word) if not version_range.is_any(): txt += " in the range '%s'" % str(version_range) txt += " is:" _pr(txt) _print_package(packages[-1]) return True return False
def pre_release(self): release_settings = self.package.config.plugins.release_vcs # test that the release path exists release_path = self.package.config.release_packages_path if not os.path.exists(release_path): raise ReleaseError("Release path does not exist: %r" % release_path) # test that the repo is in a state to release if self.vcs: self._print("Checking state of repository...") with self.repo_operation(): self.vcs.validate_repostate() # check if the repo is already tagged at the current version if release_settings.check_tag and not self.ignore_existing_tag: tag_name = self.get_current_tag_name() tag_exists = False with self.repo_operation(): tag_exists = self.vcs.tag_exists(tag_name) if tag_exists: raise ReleaseError( "Cannot release - the current package version '%s' is " "already tagged in the repository. Use --ignore-existing-tag " "to force the release" % self.package.version) it = iter_packages(self.package.name, paths=[release_path]) packages = sorted(it, key=lambda x: x.version, reverse=True) # check UUID. This stops unrelated packages that happen to have the same # name, being released as though they are the same package if self.package.uuid and packages: latest_package = packages[0] if latest_package.uuid and latest_package.uuid != self.package.uuid: raise ReleaseError( "Cannot release - the packages are not the same (UUID mismatch)" ) # test that a newer package version hasn't already been released if self.ensure_latest: for package in packages: if package.version > self.package.version: raise ReleaseError( "Cannot release - a newer package version already " "exists (%s)" % package.uri) else: break
def iter_packages(self, name, range_=None, paths=None): """Same as iter_packages in packages.py, but also applies this filter. Args: name (str): Name of the package, eg 'maya'. range_ (VersionRange or str): If provided, limits the versions returned to those in `range_`. paths (list of str, optional): paths to search for packages, defaults to `config.packages_path`. Returns: `Package` iterator. """ for package in iter_packages(name, range_, paths): if not self.excludes(package): yield package
def pre_release(self): release_settings = self.package.config.plugins.release_vcs # test that the release path exists release_path = self.package.config.release_packages_path if not os.path.exists(release_path): raise ReleaseError("Release path does not exist: %r" % release_path) # test that the repo is in a state to release if self.vcs: self._print("Checking state of repository...") with self.repo_operation(): self.vcs.validate_repostate() # check if the repo is already tagged at the current version if release_settings.check_tag and not self.ignore_existing_tag: tag_name = self.get_current_tag_name() tag_exists = False with self.repo_operation(): tag_exists = self.vcs.tag_exists(tag_name) if tag_exists: raise ReleaseError( "Cannot release - the current package version '%s' is " "already tagged in the repository. Use --ignore-existing-tag " "to force the release" % self.package.version) it = iter_packages(self.package.name, paths=[release_path]) packages = sorted(it, key=lambda x: x.version, reverse=True) # check UUID. This stops unrelated packages that happen to have the same # name, being released as though they are the same package if self.package.uuid and packages: latest_package = packages[0] if latest_package.uuid and latest_package.uuid != self.package.uuid: raise ReleaseError( "Cannot release - the packages are not the same (UUID mismatch)") # test that a newer package version hasn't already been released if self.ensure_latest: for package in packages: if package.version > self.package.version: raise ReleaseError( "Cannot release - a newer package version already " "exists (%s)" % package.uri) else: break
def run(self): it = iter_packages(name=self.package_name, paths=self.package_paths, range_=self.range_) packages = sorted(it, key=lambda x: x.version, reverse=True) num_packages = len(packages) self.progress.emit(0, num_packages) for i, package in enumerate(packages): if self.stopped: return if self.callback and not self.callback(package): break for attr in self.package_attributes: getattr(package, attr) # cause load and/or data validation self.progress.emit(i + 1, num_packages) if not self.stopped: self.finished.emit(packages)
def command(opts, parser, extra_arg_groups=None): from rez.utils.formatting import PackageRequest from rez.serialise import FileFormat from rez.packages_ import iter_packages from rez.status import status import sys req = PackageRequest(opts.PKG) if opts.current: context = status.context if context is None: print("not in a resolved environment context.", file=sys.stderr) sys.exit(1) variant = context.get_resolved_package(req.name) if variant is None: print("Package %r is not in the current context" % req.name, file=sys.stderr) sys.exit(1) package = variant.parent else: it = iter_packages(req.name, req.range) packages = sorted(it, key=lambda x: x.version) if not packages: print("no matches found") sys.exit(1) package = packages[-1] if not opts.brief: print("URI:") print(package.uri) print() print("CONTENTS:") if opts.format == "py": format_ = FileFormat.py else: format_ = FileFormat.yaml package.print_info(format_=format_, include_release=opts.all)
def command(opts, parser, extra_arg_groups=None): from rez.utils.formatting import PackageRequest from rez.serialise import FileFormat from rez.packages_ import iter_packages from rez.status import status import sys req = PackageRequest(opts.PKG) if opts.current: context = status.context if context is None: print >> sys.stderr, "not in a resolved environment context." sys.exit(1) variant = context.get_resolved_package(req.name) if variant is None: print >> sys.stderr, "Package %r is not in the current context" % req.name sys.exit(1) package = variant.parent else: it = iter_packages(req.name, req.range) packages = sorted(it, key=lambda x: x.version) if not packages: print "no matches found" sys.exit(1) package = packages[-1] if not opts.brief: print "URI:" print package.uri print print "CONTENTS:" if opts.format == "py": format_ = FileFormat.py else: format_ = FileFormat.yaml package.print_info(format_=format_, include_release=opts.all)
def diff_packages(pkg1, pkg2=None): """Invoke a diff editor to show the difference between the source of two packages. Args: pkg1 (`Package`): Package to diff. pkg2 (`Package`): Package to diff against. If None, the next most recent package version is used. """ if pkg2 is None: it = iter_packages(pkg1.name) pkgs = [x for x in it if x.version < pkg1.version] if not pkgs: raise RezError("No package to diff with - %s is the earliest " "package version" % pkg1.qualified_name) pkgs = sorted(pkgs, key=lambda x: x.version) pkg2 = pkgs[-1] def _check_pkg(pkg): if not (pkg.vcs and pkg.revision): raise RezError("Cannot diff package %s: it is a legacy format " "package that does not contain enough information" % pkg.qualified_name) _check_pkg(pkg1) _check_pkg(pkg2) path = mkdtemp(prefix="rez-pkg-diff") paths = [] for pkg in (pkg1, pkg2): print("Exporting %s..." % pkg.qualified_name) path_ = os.path.join(path, pkg.qualified_name) vcs_cls_1 = plugin_manager.get_plugin_class("release_vcs", pkg1.vcs) vcs_cls_1.export(revision=pkg.revision, path=path_) paths.append(path_) difftool = config.difftool print("Opening diff viewer %s..." % difftool) with Popen([difftool] + paths) as p: p.wait()
def diff_packages(pkg1, pkg2=None): """Invoke a diff editor to show the difference between the source of two packages. Args: pkg1 (`Package`): Package to diff. pkg2 (`Package`): Package to diff against. If None, the next most recent package version is used. """ if pkg2 is None: it = iter_packages(pkg1.name) pkgs = [x for x in it if x.version < pkg1.version] if not pkgs: raise RezError("No package to diff with - %s is the earliest " "package version" % pkg1.qualified_name) pkgs = sorted(pkgs, key=lambda x: x.version) pkg2 = pkgs[-1] def _check_pkg(pkg): if not (pkg.vcs and pkg.revision): raise RezError("Cannot diff package %s: it is a legacy format " "package that does not contain enough information" % pkg.qualified_name) _check_pkg(pkg1) _check_pkg(pkg2) path = mkdtemp(prefix="rez-pkg-diff") paths = [] for pkg in (pkg1, pkg2): print "Exporting %s..." % pkg.qualified_name path_ = os.path.join(path, pkg.qualified_name) vcs_cls_1 = plugin_manager.get_plugin_class("release_vcs", pkg1.vcs) vcs_cls_1.export(revision=pkg.revision, path=path_) paths.append(path_) difftool = config.difftool print "Opening diff viewer %s..." % difftool proc = Popen([difftool] + paths) proc.wait()
def test_1(self): """Basic release.""" # start fresh system.clear_caches() if os.path.exists(self.install_root): shutil.rmtree(self.install_root) if os.path.exists(self.src_root): shutil.rmtree(self.src_root) shutil.copytree(self.src_path, self.src_root) working_dir = self.src_root packagefile = os.path.join(working_dir, "package.yaml") with open(packagefile) as f: package_data = yaml.load(f.read()) def _write_package(): with open(packagefile, 'w') as f: dump_package_data(package_data, f, format_=FileFormat.yaml) # create the build system buildsys = create_build_system(working_dir, verbose=True) self.assertEqual(buildsys.name(), "bez") # create the vcs with self.assertRaises(ReleaseVCSError): vcs = create_release_vcs(working_dir) stubfile = os.path.join(working_dir, ".stub") with open(stubfile, 'w'): pass vcs = create_release_vcs(working_dir) self.assertEqual(vcs.name(), "stub") def _create_builder(ensure_latest=True): return create_build_process(process_type="local", working_dir=working_dir, build_system=buildsys, vcs=vcs, ensure_latest=ensure_latest, ignore_existing_tag=True, verbose=True) # release should fail because release path does not exist builder = _create_builder() with self.assertRaises(ReleaseError): builder.release() # release should work this time os.mkdir(self.install_root) builder.release() # check a file to see the release made it filepath = os.path.join(self.install_root, "foo", "1.0", "data", "data.txt") self.assertTrue(os.path.exists(filepath)) # failed release (same version released again) builder = _create_builder() num_variants = builder.release() self.assertEqual(num_variants, 0) # update package version and release again package_data["version"] = "1.1" _write_package() builder = _create_builder() builder.release() # change version to earlier and do failed release attempt package_data["version"] = "1.0.1" _write_package() builder = _create_builder() with self.assertRaises(ReleaseError): builder.release() # release again, this time allow not latest builder = _create_builder(ensure_latest=False) builder.release() # change uuid and do failed release attempt package_data["version"] = "1.2" package_data["uuid"] += "_CHANGED" _write_package() builder = _create_builder() with self.assertRaises(ReleaseError): builder.release() # check the vcs contains the tags we expect expected_value = set(["foo-1.0", "foo-1.0.1", "foo-1.1"]) with open(stubfile) as f: stub_data = yaml.load(f.read()) tags = set(stub_data.get("tags", {}).keys()) self.assertEqual(tags, expected_value) # check the package install path contains the packages we expect it = iter_packages("foo", paths=[self.install_root]) qnames = set(x.qualified_name for x in it) self.assertEqual(qnames, expected_value)
def test_2_variant_add(self): """Test variant installation on release """ orig_src_path = self.src_path self.src_path = os.path.join(self.src_path, "variants") try: self._setup_release() finally: # due to per_available_shell, this will run multiple times, don't # want to add src_path/variants/variants self.src_path = orig_src_path # copy the spangle package onto the packages path os.mkdir(self.install_root) shutil.copytree(os.path.join(self.src_root, 'spangle'), os.path.join(self.install_root, 'spangle')) # release the bar package, which has spangle-1.0 and 1.1 variants builder = self._create_builder() builder.release() # check that the released package has two variants, and the "old" # description... rel_packages = list(iter_packages("bar", paths=[self.install_root])) self.assertEqual(len(rel_packages), 1) rel_package = rel_packages[0] self.assertVariantsEqual(rel_package.variants, [['spangle-1.0'], ['spangle-1.1']]) self.assertEqual(rel_package.description, 'a package with two variants') # now, change the package so it has a single spangle-2.0 variant... self.package_data['variants'] = [['spangle-2.0']] new_desc = 'added spangle-2.0 variant' self.package_data['description'] = new_desc self._write_package() # ...then try to re-release builder = self._create_builder() builder.release() # check that the released package now three variants, and the "new" # description... rel_packages = list(iter_packages("bar", paths=[self.install_root])) self.assertEqual(len(rel_packages), 1) rel_package = rel_packages[0] self.assertVariantsEqual(rel_package.variants, [['spangle-1.0'], ['spangle-1.1'], ['spangle-2.0']]) self.assertEqual(rel_package.description, new_desc) # finally, release a package that contains a variant already released, # but with a different index... self.package_data['variants'] = [['spangle-1.1']] third_desc = 'releasing with already existing variant' self.package_data['description'] = third_desc self._write_package() builder = self._create_builder() builder.release() # make sure that the variant indices stayed the same... rel_packages = list(iter_packages("bar", paths=[self.install_root])) self.assertEqual(len(rel_packages), 1) rel_package = rel_packages[0] self.assertVariantsEqual(rel_package.variants, [['spangle-1.0'], ['spangle-1.1'], ['spangle-2.0']]) # ...but that the description was updated self.assertEqual(rel_package.description, third_desc)
def set_package_name(self, package_name): package_paths = self.context_model.packages_path self.packages = {} self.clear() rows = [] busy_cursor = QtGui.QCursor(QtCore.Qt.WaitCursor) QtGui.QApplication.setOverrideCursor(busy_cursor) try: packages = list(iter_packages(name=str(package_name), paths=package_paths)) except RezError: packages = [] if not packages: self.setEnabled(False) self.package_name = None QtGui.QApplication.restoreOverrideCursor() return for i, package in enumerate(sorted(packages, key=lambda x: x.version, reverse=True)): version_str = str(package.version) + ' ' path_str = package.uri + " " release_str = get_timestamp_str(package.timestamp) \ if package.timestamp else '-' enabled = self.callback(package) if self.callback else True rows.append((enabled, version_str, path_str, release_str)) self.packages[i] = package QtGui.QApplication.restoreOverrideCursor() self.setRowCount(len(rows)) first_selectable_row = -1 for i, row in enumerate(rows): enabled, version_str = row[:2] row = row[2:] item = QtGui.QTableWidgetItem(version_str) item.setTextAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter) self.setVerticalHeaderItem(i, item) for j in range(len(row)): item = QtGui.QTableWidgetItem(row[j]) if enabled: if first_selectable_row == -1: first_selectable_row = i else: item.setFlags(QtCore.Qt.NoItemFlags) self.setItem(i, j, item) self.setHorizontalHeaderLabels(["path", "released"]) self.resizeRowsToContents() self.resizeColumnsToContents() vh = self.verticalHeader() vh.setVisible(True) hh = self.horizontalHeader() hh.setStretchLastSection(True) hh.setVisible(True) self.package_name = package_name self.setEnabled(True) if first_selectable_row != -1: self.selectRow(first_selectable_row)
def command(opts, parser, extra_arg_groups=None): from rez.config import config from rez.packages_ import iter_package_families, iter_packages from rez.utils.yaml import dump_yaml from rez.utils.memcached import Client from rez.utils.formatting import columnise, readable_time_duration, \ readable_memory_size import sys memcache_client = Client(servers=config.memcached_uri, debug=config.debug_memcache) if not memcache_client: print("memcaching is not enabled.", file=sys.stderr) sys.exit(1) if opts.poll: poll(memcache_client, opts.interval) return if opts.flush: memcache_client.flush(hard=True) print("memcached servers are flushed.") return if opts.warm: seen = set() paths = config.nonlocal_packages_path for family in iter_package_families(paths=paths): if family.name in seen: continue for package in iter_packages(family.name, paths=paths): if opts.verbose: print("warming: %s" % package.qualified_name) # forces package definition load, which puts in memcache _ = package.data # noqa seen.add(family.name) print("memcached servers are warmed.") return if opts.reset_stats: memcache_client.reset_stats() print("memcached servers are stat reset.") return def _fail(): print("memcached servers are not responding.", file=sys.stderr) sys.exit(1) stats = memcache_client.get_stats() if opts.stats: if stats: txt = dump_yaml(stats) print(txt) else: _fail() return # print stats summary if not stats: _fail() rows = [["CACHE SERVER", "UPTIME", "HITS", "MISSES", "HIT RATIO", "MEMORY", "USED"], ["------------", "------", "----", "------", "---------", "------", "----"]] for server_id, stats_dict in stats: server_uri = server_id.split()[0] uptime = int(stats_dict.get("uptime", 0)) hits = int(stats_dict.get("get_hits", 0)) misses = int(stats_dict.get("get_misses", 0)) memory = int(stats_dict.get("limit_maxbytes", 0)) used = int(stats_dict.get("bytes", 0)) hit_ratio = float(hits) / max(hits + misses, 1) hit_percent = int(hit_ratio * 100.0) used_ratio = float(used) / max(memory, 1) used_percent = int(used_ratio * 100.0) row = (server_uri, readable_time_duration(uptime), str(hits), str(misses), "%d%%" % hit_percent, readable_memory_size(memory), "%s (%d%%)" % (readable_memory_size(used), used_percent)) rows.append(row) print('\n'.join(columnise(rows)))
def command(opts, parser, extra_arg_groups=None): import os import sys from rez.config import config from rez.package_repository import package_repository_manager from rez.package_copy import copy_package from rez.utils.formatting import PackageRequest from rez.packages_ import iter_packages if (not opts.dest_path) and not (opts.rename or opts.reversion): parser.error("--dest-path must be specified unless --rename or " "--reversion are used.") # Load the source package. # if opts.paths: paths = opts.paths.split(os.pathsep) paths = [x for x in paths if x] elif opts.no_local: paths = config.nonlocal_packages_path else: paths = None req = PackageRequest(opts.PKG) it = iter_packages( name=req.name, range_=req.range_, paths=paths ) src_pkgs = list(it) if not src_pkgs: print >> sys.stderr, "No matching packages found." sys.exit(1) if len(src_pkgs) > 1: print >> sys.stderr, "More than one package matches, please choose:" for pkg in sorted(src_pkgs, key=lambda x: x.version): print >> sys.stderr, pkg.qualified_name sys.exit(1) src_pkg = src_pkgs[0] # Determine repo and perform checks. # # A common mistake may be to specify a dest package path, rather than the # _repo_ path. This would cause a mess, since a package would be installed # into a nested location within an existing package. # if opts.dest_path: dest_pkg_repo = package_repository_manager.get_repository(opts.dest_path) if (not opts.allow_empty) and dest_pkg_repo.is_empty(): print >> sys.stderr, ( "Attempting to copy a package into an EMPTY repository. Are you " "sure that --dest-path is the correct path? This should not " "include package name and/or version." "\n\n" "If this is a valid new package repository, use the " "--allow-empty flag to continue." ) sys.exit(1) else: dest_pkg_repo = src_pkg.repository # Perform the copy. # variants = opts.variants or None result = copy_package( package=src_pkg, dest_repository=dest_pkg_repo, dest_name=opts.rename, dest_version=opts.reversion, variants=variants, overwrite=opts.overwrite, shallow=opts.shallow, follow_symlinks=opts.follow_symlinks, keep_timestamp=opts.keep_timestamp, force=opts.force, verbose=opts.verbose, dry_run=opts.dry_run ) # Print info about the result. # copied = result["copied"] skipped = result["skipped"] if opts.dry_run: # show a good indication of target variant when it doesn't get created path = dest_pkg_repo.get_package_payload_path( package_name=opts.rename or src_pkg.name, package_version=opts.reversion or src_pkg.version ) dry_run_uri = path + "/?" verb = "would be" else: verb = "were" # specific output for non-varianted packages if src_pkg.num_variants == 0: if copied: dest_pkg = copied[0][1].parent print("Copied %s to %s" % (src_pkg.uri, dest_pkg.uri)) else: assert skipped dest_pkg = skipped[0][1].parent print( "Target package already exists: %s. Use 'overwrite' to replace it." % dest_pkg.uri ) # varianted package else: if copied: print("%d variants %s copied:" % (len(copied), verb)) for src_variant, dest_variant in copied: # None possible if dry_run if dest_variant is None: dest_uri = dry_run_uri else: dest_uri = dest_variant.uri print(" %s -> %s" % (src_variant.uri, dest_uri)) if skipped: print("%d variants %s skipped (target exists):" % (len(skipped), verb)) for src_variant, dest_variant in skipped: print(" %s !-> %s" % (src_variant.uri, dest_variant.uri))
def __init__(self, context_model, variant_left=None, variant_right=None, parent=None): super(CompareCell, self).__init__(parent) self.context_model = context_model self.left_variant = variant_left self.right_variant = variant_right self.color = None self.side = None self.mode = None self.comparable = False package_paths = self.context_model.packages_path widget = None if self.left_variant and self.right_variant: self.side = "both" equal_versions = ( self.left_variant.version == self.right_variant.version) right_variant_visible = (self.right_variant.wrapped.location in package_paths) self.comparable = right_variant_visible and not equal_versions if self.comparable: # determine how far apart the variant versions are versions = sorted( [self.left_variant.version, self.right_variant.version]) range_ = VersionRange.as_span(*versions) it = iter_packages(name=self.left_variant.name, paths=package_paths, range_=range_) diff_num = sum(1 for x in it) - 1 unit = "version" if diff_num == 1 else "versions" icon_suffixes = {1: "_1", 2: "_2", 3: "_3"} icon_suffix = icon_suffixes.get(diff_num, "") if self.left_variant == self.right_variant: self.mode = "equal_to" self._set_color(0.7, 0.7, 0.7) widget = IconButton("equal_to", "packages are equal") elif self.left_variant.version == self.right_variant.version: # same version, but package is different. This can happen when # a local package is released which then hides a central package # of the same version self.mode = "equalish" self._set_color(1, 1, 0) widget = IconButton( "equalish", "packages versions are equal, but package is different") elif self.left_variant.version > self.right_variant.version: self.mode = "greater_than" self._set_color(0, 1, 0) if self.comparable: desc = "package is %d %s ahead" % (diff_num, unit) widget = IconButton("greater_than" + icon_suffix, desc) else: widget = IconButton("greater_than", "package is newer") else: self.mode = "less_than" self._set_color(1, 0, 0) if self.comparable: desc = "package is %d %s behind" % (diff_num, unit) widget = IconButton("less_than" + icon_suffix, desc) else: widget = IconButton("less_than", "package is older") elif self.right_variant: self.side = "right" self.mode = "missing" self._set_color(1, 0, 0) widget = IconButton("missing", "package is missing") elif self.left_variant: self.side = "left" self.mode = "new" self._set_color(0, 1, 0) widget = IconButton("new", "package is new") if widget: create_pane([None, widget, None], True, compact=True, parent_widget=self) widget.clicked.connect(self._clicked)
def __init__(self, context_model, variant_left=None, variant_right=None, parent=None): super(CompareCell, self).__init__(parent) self.context_model = context_model self.left_variant = variant_left self.right_variant = variant_right self.color = None self.side = None self.mode = None self.comparable = False package_paths = self.context_model.packages_path widget = None if self.left_variant and self.right_variant: self.side = "both" equal_versions = (self.left_variant.version == self.right_variant.version) right_variant_visible = (self.right_variant.wrapped.location in package_paths) self.comparable = right_variant_visible and not equal_versions if self.comparable: # determine how far apart the variant versions are versions = sorted([self.left_variant.version, self.right_variant.version]) range_ = VersionRange.as_span(*versions) it = iter_packages(name=self.left_variant.name, paths=package_paths, range_=range_) diff_num = sum(1 for x in it) - 1 unit = "version" if diff_num == 1 else "versions" icon_suffixes = {1: "_1", 2: "_2", 3: "_3"} icon_suffix = icon_suffixes.get(diff_num, "") if self.left_variant == self.right_variant: self.mode = "equal_to" self._set_color(0.7, 0.7, 0.7) widget = IconButton("equal_to", "packages are equal") elif self.left_variant.version == self.right_variant.version: # same version, but package is different. This can happen when # a local package is released which then hides a central package # of the same version self.mode = "equalish" self._set_color(1, 1, 0) widget = IconButton( "equalish", "packages versions are equal, but package is different") elif self.left_variant.version > self.right_variant.version: self.mode = "greater_than" self._set_color(0, 1, 0) if self.comparable: desc = "package is %d %s ahead" % (diff_num, unit) widget = IconButton("greater_than" + icon_suffix, desc) else: widget = IconButton("greater_than", "package is newer") else: self.mode = "less_than" self._set_color(1, 0, 0) if self.comparable: desc = "package is %d %s behind" % (diff_num, unit) widget = IconButton("less_than" + icon_suffix, desc) else: widget = IconButton("less_than", "package is older") elif self.right_variant: self.side = "right" self.mode = "missing" self._set_color(1, 0, 0) widget = IconButton("missing", "package is missing") elif self.left_variant: self.side = "left" self.mode = "new" self._set_color(0, 1, 0) widget = IconButton("new", "package is new") if widget: create_pane([None, widget, None], True, compact=True, parent_widget=self) widget.clicked.connect(self._clicked)
def get_reverse_dependency_tree(package_name, depth=None, paths=None, build_requires=False, private_build_requires=False): """Find packages that depend on the given package. This is a reverse dependency lookup. A tree is constructed, showing what packages depend on the given package, with an optional depth limit. A resolve does not occur. Only the latest version of each package is used, and requirements from all variants of that package are used. Args: package_name (str): Name of the package depended on. depth (int): Tree depth limit, unlimited if None. paths (list of str): paths to search for packages, defaults to `config.packages_path`. build_requires (bool): If True, includes packages' build_requires. private_build_requires (bool): If True, include `package_name`'s private_build_requires. Returns: A 2-tuple: - (list of list of str): Lists of package names, where each list is a single depth in the tree. The first list is always [`package_name`]. - `pygraph.digraph` object, where nodes are package names, and `package_name` is always the leaf node. """ pkgs_list = [[package_name]] g = digraph() g.add_node(package_name) # build reverse lookup it = iter_package_families(paths) package_names = set(x.name for x in it) if package_name not in package_names: raise PackageFamilyNotFoundError("No such package family %r" % package_name) if depth == 0: return pkgs_list, g bar = ProgressBar("Searching", len(package_names)) lookup = defaultdict(set) for i, package_name_ in enumerate(package_names): bar.next() packages = list(iter_packages(name=package_name_, paths=paths)) if not packages: continue pkg = max(packages, key=lambda x: x.version) requires = set(pkg.requires or []) for req_list in (pkg.variants or []): requires.update(req_list) for req in requires: if not req.conflict: lookup[req.name].add(package_name_) # perform traversal bar.finish() n = 0 consumed = set([package_name]) working_set = set([package_name]) node_color = "#F6F6F6" node_fontsize = 10 node_attrs = [("fillcolor", node_color), ("style", "filled"), ("fontsize", node_fontsize)] while working_set and (depth is None or n < depth): working_set_ = set() for child in working_set: parents = lookup[child] - consumed working_set_.update(parents) consumed.update(parents) for parent in parents: g.add_node(parent, attrs=node_attrs) g.add_edge((parent, child)) if working_set_: pkgs_list.append(sorted(list(working_set_))) working_set = working_set_ n += 1 return pkgs_list, g
def search(self, resources_request=None): """Search for resources. Args: resources_request (str): Resource to search, glob-style patterns are supported. If None, returns all matching resource types. Returns: 2-tuple: - str: resource type (family, package, variant); - List of `ResourceSearchResult`: Matching resources. Will be in alphabetical order if families, and version ascending for packages or variants. """ # Find matching package families name_pattern, version_range = self._parse_request(resources_request) family_names = set( x.name for x in iter_package_families(paths=self.package_paths) if fnmatch.fnmatch(x.name, name_pattern)) family_names = sorted(family_names) # determine what type of resource we're searching for if self.resource_type: resource_type = self.resource_type elif version_range or len(family_names) == 1: resource_type = "package" else: resource_type = "family" if not family_names: return resource_type, [] # return list of family names (validation is n/a in this case) if resource_type == "family": results = [ResourceSearchResult(x, "family") for x in family_names] return "family", results results = [] # iterate over packages/variants for name in family_names: it = iter_packages(name, version_range, paths=self.package_paths) packages = sorted(it, key=lambda x: x.version) if self.latest and packages: packages = [packages[-1]] for package in packages: # validate and check time (accessing timestamp may cause # validation fail) try: if package.timestamp: if self.after_time and package.timestamp < self.after_time: continue if self.before_time and package.timestamp >= self.before_time: continue if self.validate: package.validate_data() except ResourceContentError as e: if resource_type == "package": result = ResourceSearchResult(package, "package", str(e)) results.append(result) continue if resource_type == "package": result = ResourceSearchResult(package, "package") results.append(result) continue # iterate variants try: for variant in package.iter_variants(): if self.validate: try: variant.validate_data() except ResourceContentError as e: result = ResourceSearchResult( variant, "variant", str(e)) results.append(result) continue result = ResourceSearchResult(variant, "variant") results.append(result) except ResourceContentError: # this may happen if 'variants' in package is malformed continue return resource_type, results
def _set_variant(self, variant, preloaded_packages=None): self.clear() hh = self.horizontalHeader() self.setHorizontalHeaderLabels(["path", "released"]) hh.setResizeMode(0, QtGui.QHeaderView.Interactive) hh.setStretchLastSection(True) hh.setVisible(True) package_paths = self.context_model.packages_path package_filter = PackageFilterList.from_pod(self.context_model.package_filter) if variant and variant.wrapped.location in package_paths: self.version_index = -1 self.reference_version_index = -1 reference_version = None range_ = None if self.reference_variant and self.reference_variant.name == variant.name: reference_version = self.reference_variant.version versions = sorted([reference_version, variant.version]) range_ = VersionRange.as_span(*versions) timestamp = self.context().timestamp if preloaded_packages is not None: if range_ is None: packages = preloaded_packages else: packages = [x for x in preloaded_packages if x.version in range_] else: it = iter_packages(name=variant.name, paths=package_paths, range_=range_) packages = sorted(it, key=lambda x: x.version, reverse=True) self.setRowCount(len(packages)) brush = self.palette().brush(QtGui.QPalette.Active, QtGui.QPalette.Base) for row, package in enumerate(packages): version_str = str(package.version) + ' ' item = QtGui.QTableWidgetItem(version_str) item.setTextAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter) self.setVerticalHeaderItem(row, item) if package.version == variant.version: self.version_index = row update_font(item, bold=True) if reference_version is not None \ and package.version == reference_version: self.reference_version_index = row update_font(item, bold=True, italic=True) def _item(): item_ = QtGui.QTableWidgetItem() item_.setBackground(brush) # get rid of mouse-hover coloring return item_ if package.timestamp: release_str = get_timestamp_str(package.timestamp) in_future = (package.timestamp > timestamp) else: release_str = '-' in_future = False item = _item() txt = package.uri + " " icons = [] if in_future: icon = get_icon_widget( "clock_warning", "package did not exist at time of resolve") icons.append(icon) rule = package_filter.excludes(package) if rule: icon = get_icon_widget( "excluded", "package was excluded by rule %s" % str(rule)) icons.append(icon) if icons: label = QtGui.QLabel(txt) pane = create_pane(icons + [label, None], True, compact=True) self.setCellWidget(row, 0, pane) else: item.setText(txt) self.setItem(row, 0, item) item = _item() item.setText(release_str) self.setItem(row, 1, item) vh = self.verticalHeader() vh.setVisible(True) self.resizeColumnsToContents() hh.setStretchLastSection(True) self.update() self.allow_selection = True self.selectRow(self.version_index) self.allow_selection = False self.variant = variant
def test_2_variant_add(self): """Test variant installation on release """ self.src_path = os.path.join(self.src_path, "variants") self._setup_release() # copy the spangle package onto the packages path os.mkdir(self.install_root) shutil.copytree(os.path.join(self.src_root, 'spangle'), os.path.join(self.install_root, 'spangle')) # release the bar package, which has spangle-1.0 and 1.1 variants builder = self._create_builder() builder.release() # check that the released package has two variants, and the "old" # description... rel_packages = list(iter_packages("bar", paths=[self.install_root])) self.assertEqual(len(rel_packages), 1) rel_package = rel_packages[0] self.assertVariantsEqual(rel_package.variants, [['spangle-1.0'], ['spangle-1.1']]) self.assertEqual(rel_package.description, 'a package with two variants') # now, change the package so it has a single spangle-2.0 variant... self.package_data['variants'] = [['spangle-2.0']] new_desc = 'added spangle-2.0 variant' self.package_data['description'] = new_desc self._write_package() # ...then try to re-release builder = self._create_builder() builder.release() # check that the released package now three variants, and the "new" # description... rel_packages = list(iter_packages("bar", paths=[self.install_root])) self.assertEqual(len(rel_packages), 1) rel_package = rel_packages[0] self.assertVariantsEqual(rel_package.variants, [['spangle-1.0'], ['spangle-1.1'], ['spangle-2.0']]) self.assertEqual(rel_package.description, new_desc) # finally, release a package that contains a variant already released, # but with a different index... self.package_data['variants'] = [['spangle-1.1']] third_desc = 'releasing with already existing variant' self.package_data['description'] = third_desc self._write_package() builder = self._create_builder() builder.release() # make sure that the variant indices stayed the same... rel_packages = list(iter_packages("bar", paths=[self.install_root])) self.assertEqual(len(rel_packages), 1) rel_package = rel_packages[0] self.assertVariantsEqual(rel_package.variants, [['spangle-1.0'], ['spangle-1.1'], ['spangle-2.0']]) # ...but that the description was updated self.assertEqual(rel_package.description, third_desc)
def set_package_name(self, package_name): package_paths = self.context_model.packages_path self.packages = {} self.clear() rows = [] busy_cursor = QtGui.QCursor(QtCore.Qt.WaitCursor) QtGui.QApplication.setOverrideCursor(busy_cursor) try: packages = list( iter_packages(name=str(package_name), paths=package_paths)) except RezError: packages = [] if not packages: self.setEnabled(False) self.package_name = None QtGui.QApplication.restoreOverrideCursor() return for i, package in enumerate( sorted(packages, key=lambda x: x.version, reverse=True)): version_str = str(package.version) + ' ' path_str = package.uri + " " release_str = get_timestamp_str(package.timestamp) \ if package.timestamp else '-' enabled = self.callback(package) if self.callback else True rows.append((enabled, version_str, path_str, release_str)) self.packages[i] = package QtGui.QApplication.restoreOverrideCursor() self.setRowCount(len(rows)) first_selectable_row = -1 for i, row in enumerate(rows): enabled, version_str = row[:2] row = row[2:] item = QtGui.QTableWidgetItem(version_str) item.setTextAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter) self.setVerticalHeaderItem(i, item) for j in range(len(row)): item = QtGui.QTableWidgetItem(row[j]) if enabled: if first_selectable_row == -1: first_selectable_row = i else: item.setFlags(QtCore.Qt.NoItemFlags) self.setItem(i, j, item) self.setHorizontalHeaderLabels(["path", "released"]) self.resizeRowsToContents() self.resizeColumnsToContents() vh = self.verticalHeader() vh.setVisible(True) hh = self.horizontalHeader() hh.setStretchLastSection(True) hh.setVisible(True) self.package_name = package_name self.setEnabled(True) if first_selectable_row != -1: self.selectRow(first_selectable_row)
def get_reverse_dependency_tree(package_name, depth=None, paths=None): """Find packages that depend on the given package. This is a reverse dependency lookup. A tree is constructed, showing what packages depend on the given package, with an optional depth limit. A resolve does not occur. Only the latest version of each package is used, and requirements from all variants of that package are used. Args: package_name (str): Name of the package depended on. depth (int): Tree depth limit, unlimited if None. paths (list of str): paths to search for packages, defaults to `config.packages_path`. Returns: A 2-tuple: - (list of list of str): Lists of package names, where each list is a single depth in the tree. The first list is always [`package_name`]. - `pygraph.digraph` object, where nodes are package names, and `package_name` is always the leaf node. """ pkgs_list = [[package_name]] g = digraph() g.add_node(package_name) # build reverse lookup it = iter_package_families(paths) package_names = set(x.name for x in it) if package_name not in package_names: raise PackageFamilyNotFoundError("No such package family %r" % package_name) if depth == 0: return pkgs_list, g bar = ProgressBar("Searching", len(package_names)) lookup = defaultdict(set) for i, package_name_ in enumerate(package_names): bar.next() it = iter_packages(name=package_name_, paths=paths) packages = list(it) if not packages: continue pkg = max(packages, key=lambda x: x.version) requires = set(pkg.requires or []) for req_list in pkg.variants or []: requires.update(req_list) for req in requires: if not req.conflict: lookup[req.name].add(package_name_) # perform traversal bar.finish() n = 0 consumed = set([package_name]) working_set = set([package_name]) node_color = "#F6F6F6" node_fontsize = 10 node_attrs = [("fillcolor", node_color), ("style", "filled"), ("fontsize", node_fontsize)] while working_set and (depth is None or n < depth): working_set_ = set() for child in working_set: parents = lookup[child] - consumed working_set_.update(parents) consumed.update(parents) for parent in parents: g.add_node(parent, attrs=node_attrs) g.add_edge((parent, child)) if working_set_: pkgs_list.append(sorted(list(working_set_))) working_set = working_set_ n += 1 return pkgs_list, g
def command(opts, parser, extra_arg_groups=None): import os import sys from rez.config import config from rez.package_repository import package_repository_manager from rez.package_copy import copy_package from rez.utils.formatting import PackageRequest from rez.packages_ import iter_packages if (not opts.dest_path) and not (opts.rename or opts.reversion): parser.error("--dest-path must be specified unless --rename or " "--reversion are used.") # Load the source package. # if opts.paths: paths = opts.paths.split(os.pathsep) paths = [x for x in paths if x] elif opts.no_local: paths = config.nonlocal_packages_path else: paths = None req = PackageRequest(opts.PKG) it = iter_packages(name=req.name, range_=req.range_, paths=paths) src_pkgs = list(it) if not src_pkgs: print >> sys.stderr, "No matching packages found." sys.exit(1) if len(src_pkgs) > 1: print >> sys.stderr, "More than one package matches, please choose:" for pkg in sorted(src_pkgs, key=lambda x: x.version): print >> sys.stderr, pkg.qualified_name sys.exit(1) src_pkg = src_pkgs[0] # Determine repo and perform checks. # # A common mistake may be to specify a dest package path, rather than the # _repo_ path. This would cause a mess, since a package would be installed # into a nested location within an existing package. # if opts.dest_path: dest_pkg_repo = package_repository_manager.get_repository( opts.dest_path) if (not opts.allow_empty) and dest_pkg_repo.is_empty(): print >> sys.stderr, ( "Attempting to copy a package into an EMPTY repository. Are you " "sure that --dest-path is the correct path? This should not " "include package name and/or version." "\n\n" "If this is a valid new package repository, use the " "--allow-empty flag to continue.") sys.exit(1) else: dest_pkg_repo = src_pkg.repository # Perform the copy. # variants = opts.variants or None result = copy_package(package=src_pkg, dest_repository=dest_pkg_repo, dest_name=opts.rename, dest_version=opts.reversion, variants=variants, overwrite=opts.overwrite, shallow=opts.shallow, keep_timestamp=opts.keep_timestamp, force=opts.force, verbose=opts.verbose, dry_run=opts.dry_run) # Print info about the result. # copied = result["copied"] skipped = result["skipped"] if opts.dry_run: # show a good indication of target variant when it doesn't get created path = dest_pkg_repo.get_package_payload_path( package_name=opts.rename or src_pkg.name, package_version=opts.reversion or src_pkg.version) dry_run_uri = path + "/?" verb = "would be" else: verb = "were" # specific output for non-varianted packages if src_pkg.num_variants == 0: if copied: dest_pkg = copied[0][1].parent print("Copied %s to %s" % (src_pkg.uri, dest_pkg.uri)) else: assert skipped dest_pkg = skipped[0][1].parent print( "Target package already exists: %s. Use 'overwrite' to replace it." % dest_pkg.uri) # varianted package else: if copied: print("%d variants %s copied:" % (len(copied), verb)) for src_variant, dest_variant in copied: # None possible if dry_run if dest_variant is None: dest_uri = dry_run_uri else: dest_uri = dest_variant.uri print(" %s -> %s" % (src_variant.uri, dest_uri)) if skipped: print("%d variants %s skipped (target exists):" % (len(skipped), verb)) for src_variant, dest_variant in skipped: print(" %s !-> %s" % (src_variant.uri, dest_variant.uri))
def command(opts, parser, extra_arg_groups=None): from rez.config import config from rez.packages_ import iter_package_families, iter_packages from rez.utils.yaml import dump_yaml from rez.utils.memcached import Client from rez.utils.formatting import columnise, readable_time_duration, \ readable_memory_size import sys memcache_client = Client(servers=config.memcached_uri, debug=config.debug_memcache) if not memcache_client: print >> sys.stderr, "memcaching is not enabled." sys.exit(1) if opts.poll: poll(memcache_client, opts.interval) return if opts.flush: memcache_client.flush(hard=True) print "memcached servers are flushed." return if opts.warm: seen = set() paths = config.nonlocal_packages_path for family in iter_package_families(paths=paths): if family.name in seen: continue for package in iter_packages(family.name, paths=paths): if opts.verbose: print("warming: %s" % package.qualified_name) # forces package definition load, which puts in memcache _ = package.data # noqa seen.add(family.name) print "memcached servers are warmed." return if opts.reset_stats: memcache_client.reset_stats() print "memcached servers are stat reset." return def _fail(): print >> sys.stderr, "memcached servers are not responding." sys.exit(1) stats = memcache_client.get_stats() if opts.stats: if stats: txt = dump_yaml(stats) print txt else: _fail() return # print stats summary if not stats: _fail() rows = [["CACHE SERVER", "UPTIME", "HITS", "MISSES", "HIT RATIO", "MEMORY", "USED"], ["------------", "------", "----", "------", "---------", "------", "----"]] for server_id, stats_dict in stats: server_uri = server_id.split()[0] uptime = int(stats_dict.get("uptime", 0)) hits = int(stats_dict.get("get_hits", 0)) misses = int(stats_dict.get("get_misses", 0)) memory = int(stats_dict.get("limit_maxbytes", 0)) used = int(stats_dict.get("bytes", 0)) hit_ratio = float(hits) / max(hits + misses, 1) hit_percent = int(hit_ratio * 100.0) used_ratio = float(used) / max(memory, 1) used_percent = int(used_ratio * 100.0) row = (server_uri, readable_time_duration(uptime), str(hits), str(misses), "%d%%" % hit_percent, readable_memory_size(memory), "%s (%d%%)" % (readable_memory_size(used), used_percent)) rows.append(row) print '\n'.join(columnise(rows))
def command(opts, parser, extra_arg_groups=None): from rez.config import config from rez.exceptions import RezError from rez.utils.formatting import get_epoch_time_from_str, expand_abbreviations from rez.utils.logging_ import print_error from rez.packages_ import iter_package_families, iter_packages from rez.vendor.version.requirement import Requirement import os.path import fnmatch import sys error_class = None if opts.debug else RezError before_time = 0 after_time = 0 if opts.before: before_time = get_epoch_time_from_str(opts.before) if opts.after: after_time = get_epoch_time_from_str(opts.after) if after_time and before_time and (after_time >= before_time): parser.error("non-overlapping --before and --after") if opts.paths is None: pkg_paths = config.nonlocal_packages_path if opts.no_local else None else: pkg_paths = (opts.paths or "").split(os.pathsep) pkg_paths = [os.path.expanduser(x) for x in pkg_paths if x] name_pattern = opts.PKG or '*' version_range = None if opts.PKG: try: req = Requirement(opts.PKG) name_pattern = req.name if not req.range.is_any(): version_range = req.range except: pass type_ = opts.type if opts.errors or (type_ == "auto" and version_range): type_ = "package" # turn some of the nastier rez-1 warnings into errors config.override("error_package_name_mismatch", True) config.override("error_version_mismatch", True) config.override("error_nonstring_version", True) if opts.no_warnings: config.override("warn_none", True) # families found = False family_names = [] families = iter_package_families(paths=pkg_paths) if opts.sort: families = sorted(families, key=lambda x: x.name) for family in families: if family.name not in family_names and \ fnmatch.fnmatch(family.name, name_pattern): family_names.append(family.name) if type_ == "auto": type_ = "package" if family.name == name_pattern else "family" if type_ == "family": print family.name found = True def _handle(e): print_error(str(e)) def _print_resource(r): if opts.validate: try: r.validate_data() except error_class as e: _handle(e) return if opts.format: txt = expand_abbreviations(opts.format, fields) lines = txt.split("\\n") for line in lines: try: line_ = r.format(line) except error_class as e: _handle(e) break if opts.no_newlines: line_ = line_.replace('\n', "\\n") print line_ else: print r.qualified_name # packages/variants if type_ in ("package", "variant"): for name in family_names: packages = iter_packages(name, version_range, paths=pkg_paths) if opts.sort or opts.latest: packages = sorted(packages, key=lambda x: x.version) if opts.latest and packages: packages = [packages[-1]] for package in packages: if ((before_time or after_time) and package.timestamp and (before_time and package.timestamp >= before_time or after_time and package.timestamp <= after_time)): continue if opts.errors: try: package.validate_data() except error_class as e: _handle(e) found = True elif type_ == "package": _print_resource(package) found = True elif type_ == "variant": try: package.validate_data() except error_class as e: _handle(e) continue try: for variant in package.iter_variants(): _print_resource(variant) found = True except error_class as e: _handle(e) continue if not found: if opts.errors: print "no erroneous packages found" else: print "no matches found" sys.exit(1)
def _installed_packages(self, name): return list(iter_packages(name, paths=[self.temprepo]))
def search(self, resources_request=None): """Search for resources. Args: resources_request (str): Resource to search, glob-style patterns are supported. If None, returns all matching resource types. Returns: 2-tuple: - str: resource type (family, package, variant); - List of `ResourceSearchResult`: Matching resources. Will be in alphabetical order if families, and version ascending for packages or variants. """ # Find matching package families name_pattern, version_range = self._parse_request(resources_request) family_names = set( x.name for x in iter_package_families(paths=self.package_paths) if fnmatch.fnmatch(x.name, name_pattern) ) family_names = sorted(family_names) # determine what type of resource we're searching for if self.resource_type: resource_type = self.resource_type elif version_range or len(family_names) == 1: resource_type = "package" else: resource_type = "family" if not family_names: return resource_type, [] # return list of family names (validation is n/a in this case) if resource_type == "family": results = [ResourceSearchResult(x, "family") for x in family_names] return "family", results results = [] # iterate over packages/variants for name in family_names: it = iter_packages(name, version_range, paths=self.package_paths) packages = sorted(it, key=lambda x: x.version) if self.latest and packages: packages = [packages[-1]] for package in packages: # validate and check time (accessing timestamp may cause # validation fail) try: if package.timestamp: if self.after_time and package.timestamp < self.after_time: continue if self.before_time and package.timestamp >= self.before_time: continue if self.validate: package.validate_data() except ResourceContentError as e: if resource_type == "package": result = ResourceSearchResult(package, "package", str(e)) results.append(result) continue if resource_type == "package": result = ResourceSearchResult(package, "package") results.append(result) continue # iterate variants try: for variant in package.iter_variants(): if self.validate: try: variant.validate_data() except ResourceContentError as e: result = ResourceSearchResult( variant, "variant", str(e)) results.append(result) continue result = ResourceSearchResult(variant, "variant") results.append(result) except ResourceContentError: # this may happen if 'variants' in package is malformed continue return resource_type, results
def _set_variant(self, variant, preloaded_packages=None): self.clear() hh = self.horizontalHeader() self.setHorizontalHeaderLabels(["path", "released"]) hh.setResizeMode(0, QtGui.QHeaderView.Interactive) hh.setStretchLastSection(True) hh.setVisible(True) package_paths = self.context_model.packages_path package_filter = PackageFilterList.from_pod( self.context_model.package_filter) if variant and variant.wrapped.location in package_paths: self.version_index = -1 self.reference_version_index = -1 reference_version = None range_ = None if self.reference_variant and self.reference_variant.name == variant.name: reference_version = self.reference_variant.version versions = sorted([reference_version, variant.version]) range_ = VersionRange.as_span(*versions) timestamp = self.context().timestamp if preloaded_packages is not None: if range_ is None: packages = preloaded_packages else: packages = [ x for x in preloaded_packages if x.version in range_ ] else: it = iter_packages(name=variant.name, paths=package_paths, range_=range_) packages = sorted(it, key=lambda x: x.version, reverse=True) self.setRowCount(len(packages)) brush = self.palette().brush(QtGui.QPalette.Active, QtGui.QPalette.Base) for row, package in enumerate(packages): version_str = str(package.version) + ' ' item = QtGui.QTableWidgetItem(version_str) item.setTextAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter) self.setVerticalHeaderItem(row, item) if package.version == variant.version: self.version_index = row update_font(item, bold=True) if reference_version is not None \ and package.version == reference_version: self.reference_version_index = row update_font(item, bold=True, italic=True) def _item(): item_ = QtGui.QTableWidgetItem() item_.setBackground( brush) # get rid of mouse-hover coloring return item_ if package.timestamp: release_str = get_timestamp_str(package.timestamp) in_future = (package.timestamp > timestamp) else: release_str = '-' in_future = False item = _item() txt = package.uri + " " icons = [] if in_future: icon = get_icon_widget( "clock_warning", "package did not exist at time of resolve") icons.append(icon) rule = package_filter.excludes(package) if rule: icon = get_icon_widget( "excluded", "package was excluded by rule %s" % str(rule)) icons.append(icon) if icons: label = QtGui.QLabel(txt) pane = create_pane(icons + [label, None], True, compact=True) self.setCellWidget(row, 0, pane) else: item.setText(txt) self.setItem(row, 0, item) item = _item() item.setText(release_str) self.setItem(row, 1, item) vh = self.verticalHeader() vh.setVisible(True) self.resizeColumnsToContents() hh.setStretchLastSection(True) self.update() self.allow_selection = True self.selectRow(self.version_index) self.allow_selection = False self.variant = variant