def test_pkg_iteration(self):
"""package iteration."""
all_packages = set()
all_fams = iter_package_families()
for fam in all_fams:
packages = _to_qnames(iter_packages(fam.name))
all_packages.update(packages)
self.assertEqual(all_packages, ALL_PACKAGES)
res = _to_qnames(iter_packages('nada'))
self.assertEqual(res, set(['nada']))
res = _to_qnames(iter_packages('python'))
self.assertEqual(
res,
set([
'python-2.5.2', 'python-2.6.0', 'python-2.6.8', 'python-2.7.0'
]))
res = _to_qnames(iter_packages('pydad', "<3"))
self.assertEqual(res, set(['pydad-1', 'pydad-2']))
for fam_name in ALL_FAMILIES:
for package in iter_packages(fam_name):
family = package.parent
self.assertEqual(family.name, fam_name)
it = family.iter_packages()
self.assertTrue(package in it)
def _test_reorder(self, orderer, package_name, expected_order):
"""Ensure ordered order package version as expected."""
it = iter_packages(package_name)
descending = sorted(it, key=lambda x: x.version, reverse=True)
ordered = orderer.reorder(descending) or descending
result = [str(x.version) for x in ordered]
self.assertEqual(expected_order, result)
def find_site_python(module_name, paths=None):
"""Find the rez native python package that contains the given module.
This function is used by python 'native' rez installers to find the native
rez python package that represents the python installation that this module
is installed into.
Note:
This function is dependent on the behavior found in the python '_native'
package found in the 'rez-recipes' repository. Specifically, it expects
to find a python package with a '_site_paths' list attribute listing
the site directories associated with the python installation.
Args:
module_name (str): Target python module.
paths (list of str, optional): paths to search for packages,
defaults to `config.packages_path`.
Returns:
`Package`: Native python package containing the named module.
"""
from rez.packages import iter_packages
import subprocess
import ast
import os
py_cmd = 'import {x}; print({x}.__path__)'.format(x=module_name)
p = Popen(["python", "-c", py_cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
out, err = p.communicate()
if p.returncode:
raise InvalidPackageError(
"Failed to find installed python module '%s':\n%s" %
(module_name, err))
module_paths = ast.literal_eval(out.strip())
def issubdir(path, parent_path):
return path.startswith(parent_path + os.sep)
for package in iter_packages("python", paths=paths):
if not hasattr(package, "_site_paths"):
continue
contained = True
for module_path in module_paths:
if not any(issubdir(module_path, x) for x in package._site_paths):
contained = False
if contained:
return package
raise InvalidPackageError(
"Failed to find python installation containing the module '%s'. Has "
"python been installed as a rez package?" % module_name)
def _update_status(self):
def _ok():
self._set_style()
self.setToolTip("")
def _err(msg, color="red"):
self._set_style("QLineEdit { border : 2px solid %s;}" % color)
self.setToolTip(msg)
txt = str(self.text())
if not txt:
_ok()
return
try:
req = Requirement(str(txt))
except Exception as e:
_err(str(e))
return
_ok()
if not req.conflict:
try:
it = iter_packages(name=req.name,
range_=req.range,
paths=self._paths)
pkg = sorted(it, key=lambda x: x.version)[-1]
except Exception:
_err("cannot find package: %r" % txt, "orange")
return
if pkg.description:
self.setToolTip(pkg.description)
def python_config_jam():
# Lookup all installed python packages
from rez import packages, resolved_context
versions = list()
info_cmd = ("python %s get-python-info" % sys.argv[0])
for python_pkg in packages.iter_packages("python"):
full_ver = str(python_pkg.version)
python_ver = ".".join(full_ver.split(".")[:2]) # exclude patch ver
versions.append(python_ver)
versions.sort()
# Only use python-27 for now (for USD)
# because cmake files only link to the version which built latest.
versions = ["2.7"]
for python_ver in versions:
python_name = "python-%s" % python_ver
rc = resolved_context.ResolvedContext([python_name], caching=False)
popen = rc.execute_shell(command=info_cmd)
popen.wait()
return versions
def print_package_versions(self):
"""Print a list of versions of the package this tool comes from, and
indicate which version this tool is from."""
variants = self.context.get_tool_variants(self.tool_name)
if variants:
if len(variants) > 1:
self._print_conflicting(variants)
return 1
else:
from rez.packages import iter_packages
variant = next(iter(variants))
it = iter_packages(name=variant.name)
rows = []
colors = []
for pkg in sorted(it, key=lambda x: x.version, reverse=True):
if pkg.version == variant.version:
name = "* %s" % pkg.qualified_name
col = heading
else:
name = " %s" % pkg.qualified_name
col = local if pkg.is_local else None
label = "(local)" if pkg.is_local else ""
rows.append((name, pkg.path, label))
colors.append(col)
_pr = Printer()
for col, line in zip(colors, columnise(rows)):
_pr(line, col)
return 0
def filter_versions(fltr_):
matching_versions = set()
for pkg in iter_packages(pkg_family):
if not fltr_.excludes(pkg):
matching_versions.add(str(pkg.version))
self.assertEqual(matching_versions, set(expected_result))
def test_1(self):
"""Basic release."""
# release should fail because release path does not exist
self._setup_release()
builder = self._create_builder()
with self.assertRaises(ReleaseError):
builder.release()
# release should work this time
os.mkdir(self.install_root)
builder.release()
# check a file to see the release made it
filepath = os.path.join(self.install_root, "foo", "1.0", "data",
"data.txt")
self.assertTrue(os.path.exists(filepath))
# failed release (same version released again)
builder = self._create_builder()
num_variants = builder.release()
self.assertEqual(num_variants, 0)
# update package version and release again
self.package_data["version"] = "1.1"
self._write_package()
builder = self._create_builder()
builder.release()
# change version to earlier and do failed release attempt
self.package_data["version"] = "1.0.1"
self._write_package()
builder = self._create_builder()
with self.assertRaises(ReleaseError):
builder.release()
# release again, this time allow not latest
builder = self._create_builder(ensure_latest=False)
builder.release()
# change uuid and do failed release attempt
self.package_data["version"] = "1.2"
self.package_data["uuid"] += "_CHANGED"
self._write_package()
builder = self._create_builder()
with self.assertRaises(ReleaseError):
builder.release()
# check the vcs contains the tags we expect
expected_value = set(["foo-1.0", "foo-1.0.1", "foo-1.1"])
with open(self.stubfile) as f:
stub_data = yaml.load(f.read(), Loader=yaml.FullLoader)
tags = set(stub_data.get("tags", {}).keys())
self.assertEqual(tags, expected_value)
# check the package install path contains the packages we expect
it = iter_packages("foo", paths=[self.install_root])
qnames = set(x.qualified_name for x in it)
self.assertEqual(qnames, expected_value)
def get_previous_release(self):
release_path = self.package.config.release_packages_path
it = iter_packages(self.package.name, paths=[release_path])
packages = sorted(it, key=lambda x: x.version, reverse=True)
for package in packages:
if package.version < self.package.version:
return package
return None
def _test(orderer, package_name, expected_order):
it = iter_packages(package_name)
descending = sorted(it, key=lambda x: x.version, reverse=True)
pod = to_pod(orderer)
orderer2 = from_pod(pod)
for orderer_ in (orderer, orderer2):
ordered = orderer_.reorder(descending)
result = [str(x.version) for x in ordered]
self.assertEqual(result, expected_order)
def _print_package_info(self, value, buf=sys.stdout, b=False):
word = "is also" if b else "is"
_pr = Printer(buf)
request_str = os.path.basename(value)
if request_str != value:
return False
def _print_package(package):
if isinstance(package, Package):
name = package.qualified_name
else:
name = package.qualified_package_name # Variant
_pr("Package: %s" % name)
path_str = "URI: %s" % package.uri
if package.is_local:
path_str += " (local)"
_pr(path_str)
try:
req = PackageRequest(request_str)
except:
return False
if req.conflict:
return False
package_name = req.name
version_range = req.range
# check for the package in the active context
if self.context:
variant = self.context.get_resolved_package(package_name)
if variant and variant.version in version_range:
_pr("'%s' %s a package in the active context:" %
(package_name, word))
_print_package(variant)
if self.context.load_path:
_pr("Context: %s" % self.context.load_path)
return True
# find the package
it = iter_packages(package_name, version_range)
packages = sorted(it, key=lambda x: x.version)
if packages:
txt = "'%s' %s a package. The latest version" % (package_name,
word)
if not version_range.is_any():
txt += " in the range '%s'" % str(version_range)
txt += " is:"
_pr(txt)
_print_package(packages[-1])
return True
return False
def pre_release(self):
release_settings = self.package.config.plugins.release_vcs
# test that the release path exists
release_path = self.package.config.release_packages_path
if not os.path.exists(release_path):
raise ReleaseError("Release path does not exist: %r" %
release_path)
# test that the repo is in a state to release
if self.vcs:
self._print("Checking state of repository...")
with self.repo_operation():
self.vcs.validate_repostate()
# check if the repo is already tagged at the current version
if release_settings.check_tag and not self.ignore_existing_tag:
tag_name = self.get_current_tag_name()
tag_exists = False
with self.repo_operation():
tag_exists = self.vcs.tag_exists(tag_name)
if tag_exists:
raise ReleaseError(
"Cannot release - the current package version '%s' is "
"already tagged in the repository. Use --ignore-existing-tag "
"to force the release" % self.package.version)
it = iter_packages(self.package.name, paths=[release_path])
packages = sorted(it, key=lambda x: x.version, reverse=True)
# check UUID. This stops unrelated packages that happen to have the same
# name, being released as though they are the same package
if self.package.uuid and packages:
latest_package = packages[0]
if latest_package.uuid and latest_package.uuid != self.package.uuid:
raise ReleaseError(
"Cannot release - the packages are not the same (UUID mismatch)"
)
# test that a newer package version hasn't already been released
if self.ensure_latest:
for package in packages:
if package.version > self.package.version:
raise ReleaseError(
"Cannot release - a newer package version already "
"exists (%s)" % package.uri)
else:
break
def iter_packages(self, name, range_=None, paths=None):
"""Same as iter_packages in packages.py, but also applies this filter.
Args:
name (str): Name of the package, eg 'maya'.
range_ (VersionRange or str): If provided, limits the versions returned
to those in `range_`.
paths (list of str, optional): paths to search for packages, defaults
to `config.packages_path`.
Returns:
`Package` iterator.
"""
for package in iter_packages(name, range_, paths):
if not self.excludes(package):
yield package
def command(opts, parser, extra_arg_groups=None):
from rez.utils.formatting import PackageRequest
from rez.serialise import FileFormat
from rez.packages import iter_packages
from rez.status import status
import sys
req = PackageRequest(opts.PKG)
if opts.current:
context = status.context
if context is None:
print("not in a resolved environment context.", file=sys.stderr)
sys.exit(1)
variant = context.get_resolved_package(req.name)
if variant is None:
print("Package %r is not in the current context" % req.name,
file=sys.stderr)
sys.exit(1)
package = variant.parent
else:
it = iter_packages(req.name, req.range)
packages = sorted(it, key=lambda x: x.version)
if not packages:
print("no matches found")
sys.exit(1)
package = packages[-1]
if not opts.brief:
print("URI:")
print(package.uri)
print()
print("CONTENTS:")
if opts.format == "py":
format_ = FileFormat.py
else:
format_ = FileFormat.yaml
package.print_info(format_=format_, include_release=opts.all)
def run(self):
it = iter_packages(name=self.package_name,
paths=self.package_paths,
range_=self.range_)
packages = sorted(it, key=lambda x: x.version, reverse=True)
num_packages = len(packages)
self.progress.emit(0, num_packages)
for i, package in enumerate(packages):
if self.stopped:
return
if self.callback and not self.callback(package):
break
for attr in self.package_attributes:
getattr(package, attr) # cause load and/or data validation
self.progress.emit(i + 1, num_packages)
if not self.stopped:
self.finished.emit(packages)
def diff_packages(pkg1, pkg2=None):
"""Invoke a diff editor to show the difference between the source of two
packages.
Args:
pkg1 (`Package`): Package to diff.
pkg2 (`Package`): Package to diff against. If None, the next most recent
package version is used.
"""
if pkg2 is None:
it = iter_packages(pkg1.name)
pkgs = [x for x in it if x.version < pkg1.version]
if not pkgs:
raise RezError("No package to diff with - %s is the earliest "
"package version" % pkg1.qualified_name)
pkgs = sorted(pkgs, key=lambda x: x.version)
pkg2 = pkgs[-1]
def _check_pkg(pkg):
if not (pkg.vcs and pkg.revision):
raise RezError("Cannot diff package %s: it is a legacy format "
"package that does not contain enough information"
% pkg.qualified_name)
_check_pkg(pkg1)
_check_pkg(pkg2)
path = mkdtemp(prefix="rez-pkg-diff")
paths = []
for pkg in (pkg1, pkg2):
print("Exporting %s..." % pkg.qualified_name)
path_ = os.path.join(path, pkg.qualified_name)
vcs_cls_1 = plugin_manager.get_plugin_class("release_vcs", pkg1.vcs)
vcs_cls_1.export(revision=pkg.revision, path=path_)
paths.append(path_)
difftool = config.difftool
print("Opening diff viewer %s..." % difftool)
with Popen([difftool] + paths) as p:
p.wait()
def iter_versions(self, name, location=None):
"""Iter package versions
:param name: Package name
:param location: One single package path to look for. Loop over all
paths (`packages_path`) if not given.
:type name: str
:type location: str or None
:return: An iterator that yields `PkgVersion` objects
:rtype: collections.Iterator[PkgVersion]
"""
paths = [location] if location else self._paths
_cache = dict()
for p in iter_packages(name, paths=paths):
_l = p.resource.location
if _l in _cache:
norm_location = _cache[_l]
else:
norm_location = util.normpath(_l)
_cache[_l] = norm_location
is_nonlocal = norm_location in self._non_local
yield PkgVersion(
name=name,
version=p.version,
qualified=p.qualified_name,
requires=[str(r) for r in p.requires or []],
variants=[[str(r) for r in var] for var in p.variants or []],
tools=p.tools or [],
uri=p.uri,
timestamp=p.timestamp,
location=norm_location,
is_nonlocal=is_nonlocal,
)
def __init__(self,
context_model,
variant_left=None,
variant_right=None,
parent=None):
super(CompareCell, self).__init__(parent)
self.context_model = context_model
self.left_variant = variant_left
self.right_variant = variant_right
self.color = None
self.side = None
self.mode = None
self.comparable = False
package_paths = self.context_model.packages_path
widget = None
if self.left_variant and self.right_variant:
self.side = "both"
equal_versions = (
self.left_variant.version == self.right_variant.version)
right_variant_visible = (self.right_variant.wrapped.location
in package_paths)
self.comparable = right_variant_visible and not equal_versions
if self.comparable:
# determine how far apart the variant versions are
versions = sorted(
[self.left_variant.version, self.right_variant.version])
range_ = VersionRange.as_span(*versions)
it = iter_packages(name=self.left_variant.name,
paths=package_paths,
range_=range_)
diff_num = sum(1 for x in it) - 1
unit = "version" if diff_num == 1 else "versions"
icon_suffixes = {1: "_1", 2: "_2", 3: "_3"}
icon_suffix = icon_suffixes.get(diff_num, "")
if self.left_variant == self.right_variant:
self.mode = "equal_to"
self._set_color(0.7, 0.7, 0.7)
widget = IconButton("equal_to", "packages are equal")
elif self.left_variant.version == self.right_variant.version:
# same version, but package is different. This can happen when
# a local package is released which then hides a central package
# of the same version
self.mode = "equalish"
self._set_color(1, 1, 0)
widget = IconButton(
"equalish",
"packages versions are equal, but package is different")
elif self.left_variant.version > self.right_variant.version:
self.mode = "greater_than"
self._set_color(0, 1, 0)
if self.comparable:
desc = "package is %d %s ahead" % (diff_num, unit)
widget = IconButton("greater_than" + icon_suffix, desc)
else:
widget = IconButton("greater_than", "package is newer")
else:
self.mode = "less_than"
self._set_color(1, 0, 0)
if self.comparable:
desc = "package is %d %s behind" % (diff_num, unit)
widget = IconButton("less_than" + icon_suffix, desc)
else:
widget = IconButton("less_than", "package is older")
elif self.right_variant:
self.side = "right"
self.mode = "missing"
self._set_color(1, 0, 0)
widget = IconButton("missing", "package is missing")
elif self.left_variant:
self.side = "left"
self.mode = "new"
self._set_color(0, 1, 0)
widget = IconButton("new", "package is new")
if widget:
create_pane([None, widget, None],
True,
compact=True,
parent_widget=self)
widget.clicked.connect(self._clicked)
def test_2_variant_add(self):
"""Test variant installation on release
"""
orig_src_path = self.src_path
self.src_path = os.path.join(self.src_path, "variants")
try:
self._setup_release()
finally:
# due to per_available_shell, this will run multiple times, don't
# want to add src_path/variants/variants
self.src_path = orig_src_path
# copy the spangle package onto the packages path
os.mkdir(self.install_root)
shutil.copytree(os.path.join(self.src_root, 'spangle'),
os.path.join(self.install_root, 'spangle'))
# release the bar package, which has spangle-1.0 and 1.1 variants
builder = self._create_builder()
builder.release()
# check that the released package has two variants, and the "old"
# description...
rel_packages = list(iter_packages("bar", paths=[self.install_root]))
self.assertEqual(len(rel_packages), 1)
rel_package = rel_packages[0]
self.assertVariantsEqual(rel_package.variants,
[['spangle-1.0'], ['spangle-1.1']])
self.assertEqual(rel_package.description,
'a package with two variants')
# now, change the package so it has a single spangle-2.0 variant...
self.package_data['variants'] = [['spangle-2.0']]
new_desc = 'added spangle-2.0 variant'
self.package_data['description'] = new_desc
self._write_package()
# ...then try to re-release
builder = self._create_builder()
builder.release()
# check that the released package now three variants, and the "new"
# description...
rel_packages = list(iter_packages("bar", paths=[self.install_root]))
self.assertEqual(len(rel_packages), 1)
rel_package = rel_packages[0]
self.assertVariantsEqual(
rel_package.variants,
[['spangle-1.0'], ['spangle-1.1'], ['spangle-2.0']])
self.assertEqual(rel_package.description, new_desc)
# finally, release a package that contains a variant already released,
# but with a different index...
self.package_data['variants'] = [['spangle-1.1']]
third_desc = 'releasing with already existing variant'
self.package_data['description'] = third_desc
self._write_package()
builder = self._create_builder()
builder.release()
# make sure that the variant indices stayed the same...
rel_packages = list(iter_packages("bar", paths=[self.install_root]))
self.assertEqual(len(rel_packages), 1)
rel_package = rel_packages[0]
self.assertVariantsEqual(
rel_package.variants,
[['spangle-1.0'], ['spangle-1.1'], ['spangle-2.0']])
# ...but that the description was updated
self.assertEqual(rel_package.description, third_desc)
def command(opts, parser, extra_arg_groups=None):
import os
import sys
from rez.config import config
from rez.package_repository import package_repository_manager
from rez.package_copy import copy_package
from rez.utils.formatting import PackageRequest
from rez.packages import iter_packages, get_variant_from_uri
if opts.variant_uri:
if opts.PKG:
parser.error("Supply PKG or --variant-uri, not both.")
elif not opts.PKG:
parser.error("Expected PKG.")
if (not opts.dest_path) and not (opts.rename or opts.reversion):
parser.error("--dest-path must be specified unless --rename or "
"--reversion are used.")
# Load the source package.
#
if opts.variant_uri:
variant = get_variant_from_uri(opts.variant_uri)
if variant is None:
print("Unknown variant: %s" % opts.variant_uri, file=sys.stderr)
sys.exit(1)
src_pkg = variant.parent
variant_indexes = [variant.index]
else:
if opts.paths:
paths = opts.paths.split(os.pathsep)
paths = [x for x in paths if x]
elif opts.no_local:
paths = config.nonlocal_packages_path
else:
paths = None
req = PackageRequest(opts.PKG)
it = iter_packages(name=req.name, range_=req.range_, paths=paths)
src_pkgs = list(it)
if not src_pkgs:
print("No matching packages found.", file=sys.stderr)
sys.exit(1)
if len(src_pkgs) > 1:
print("More than one package matches, please choose:",
file=sys.stderr)
for pkg in sorted(src_pkgs, key=lambda x: x.version):
print(pkg.qualified_name, file=sys.stderr)
sys.exit(1)
src_pkg = src_pkgs[0]
variant_indexes = opts.variants or None
# Determine repo and perform checks.
#
# A common mistake may be to specify a dest package path, rather than the
# _repo_ path. This would cause a mess, since a package would be installed
# into a nested location within an existing package.
#
if opts.dest_path:
dest_pkg_repo = package_repository_manager.get_repository(
opts.dest_path)
if (not opts.allow_empty) and dest_pkg_repo.is_empty():
print((
"Attempting to copy a package into an EMPTY repository. Are you "
"sure that --dest-path is the correct path? This should not "
"include package name and/or version."
"\n\n"
"If this is a valid new package repository, use the "
"--allow-empty flag to continue."),
file=sys.stderr)
sys.exit(1)
else:
dest_pkg_repo = src_pkg.repository
# Perform the copy.
#
result = copy_package(package=src_pkg,
dest_repository=dest_pkg_repo,
dest_name=opts.rename,
dest_version=opts.reversion,
variants=variant_indexes,
overwrite=opts.overwrite,
shallow=opts.shallow,
follow_symlinks=opts.follow_symlinks,
keep_timestamp=opts.keep_timestamp,
force=opts.force,
verbose=opts.verbose,
dry_run=opts.dry_run)
# Print info about the result.
#
copied = result["copied"]
skipped = result["skipped"]
if opts.dry_run:
# show a good indication of target variant when it doesn't get created
path = dest_pkg_repo.get_package_payload_path(
package_name=opts.rename or src_pkg.name,
package_version=opts.reversion or src_pkg.version)
dry_run_uri = path + "/?"
verb = "would be"
else:
verb = "were"
# specific output for non-varianted packages
if src_pkg.num_variants == 0:
if copied:
dest_pkg = copied[0][1].parent
print("Copied %s to %s" % (src_pkg.uri, dest_pkg.uri))
else:
assert skipped
dest_pkg = skipped[0][1].parent
print(
"Target package already exists: %s. Use 'overwrite' to replace it."
% dest_pkg.uri)
# varianted package
else:
if copied:
print("%d variants %s copied:" % (len(copied), verb))
for src_variant, dest_variant in copied:
# None possible if dry_run
if dest_variant is None:
dest_uri = dry_run_uri
else:
dest_uri = dest_variant.uri
print(" %s -> %s" % (src_variant.uri, dest_uri))
if skipped:
print("%d variants %s skipped (target exists):" %
(len(skipped), verb))
for src_variant, dest_variant in skipped:
print(" %s !-> %s" % (src_variant.uri, dest_variant.uri))
def get_reverse_dependency_tree(package_name,
depth=None,
paths=None,
build_requires=False,
private_build_requires=False):
"""Find packages that depend on the given package.
This is a reverse dependency lookup. A tree is constructed, showing what
packages depend on the given package, with an optional depth limit. A
resolve does not occur. Only the latest version of each package is used,
and requirements from all variants of that package are used.
Args:
package_name (str): Name of the package depended on.
depth (int): Tree depth limit, unlimited if None.
paths (list of str): paths to search for packages, defaults to
`config.packages_path`.
build_requires (bool): If True, includes packages' build_requires.
private_build_requires (bool): If True, include `package_name`'s
private_build_requires.
Returns:
A 2-tuple:
- (list of list of str): Lists of package names, where each list is a
single depth in the tree. The first list is always [`package_name`].
- `pygraph.digraph` object, where nodes are package names, and
`package_name` is always the leaf node.
"""
pkgs_list = [[package_name]]
g = digraph()
g.add_node(package_name)
# build reverse lookup
it = iter_package_families(paths)
package_names = set(x.name for x in it)
if package_name not in package_names:
raise PackageFamilyNotFoundError("No such package family %r" %
package_name)
if depth == 0:
return pkgs_list, g
bar = ProgressBar("Searching", len(package_names))
lookup = defaultdict(set)
for i, package_name_ in enumerate(package_names):
it = iter_packages(name=package_name_, paths=paths)
packages = list(it)
if not packages:
continue
pkg = max(packages, key=lambda x: x.version)
requires = []
for variant in pkg.iter_variants():
pbr = (private_build_requires and pkg.name == package_name)
requires += variant.get_requires(build_requires=build_requires,
private_build_requires=pbr)
for req in requires:
if not req.conflict:
lookup[req.name].add(package_name_)
bar.next()
bar.finish()
# perform traversal
n = 0
consumed = set([package_name])
working_set = set([package_name])
node_color = "#F6F6F6"
node_fontsize = 10
node_attrs = [("fillcolor", node_color), ("style", "filled"),
("fontsize", node_fontsize)]
while working_set and (depth is None or n < depth):
working_set_ = set()
for child in working_set:
parents = lookup[child] - consumed
working_set_.update(parents)
consumed.update(parents)
for parent in parents:
g.add_node(parent, attrs=node_attrs)
g.add_edge((parent, child))
if working_set_:
pkgs_list.append(sorted(list(working_set_)))
working_set = working_set_
n += 1
return pkgs_list, g
def search(self, resources_request=None):
"""Search for resources.
Args:
resources_request (str): Resource to search, glob-style patterns
are supported. If None, returns all matching resource types.
Returns:
2-tuple:
- str: resource type (family, package, variant);
- List of `ResourceSearchResult`: Matching resources. Will be in
alphabetical order if families, and version ascending for
packages or variants.
"""
# Find matching package families
name_pattern, version_range = self._parse_request(resources_request)
family_names = set(
x.name for x in iter_package_families(paths=self.package_paths)
if fnmatch.fnmatch(x.name, name_pattern))
family_names = sorted(family_names)
# determine what type of resource we're searching for
if self.resource_type:
resource_type = self.resource_type
elif version_range or len(family_names) == 1:
resource_type = "package"
else:
resource_type = "family"
if not family_names:
return resource_type, []
# return list of family names (validation is n/a in this case)
if resource_type == "family":
results = [ResourceSearchResult(x, "family") for x in family_names]
return "family", results
results = []
# iterate over packages/variants
for name in family_names:
it = iter_packages(name, version_range, paths=self.package_paths)
packages = sorted(it, key=lambda x: x.version)
if self.latest and packages:
packages = [packages[-1]]
for package in packages:
# validate and check time (accessing timestamp may cause
# validation fail)
try:
if package.timestamp:
if self.after_time and package.timestamp < self.after_time:
continue
if self.before_time and package.timestamp >= self.before_time:
continue
if self.validate:
package.validate_data()
except ResourceContentError as e:
if resource_type == "package":
result = ResourceSearchResult(package, "package",
str(e))
results.append(result)
continue
if resource_type == "package":
result = ResourceSearchResult(package, "package")
results.append(result)
continue
# iterate variants
try:
for variant in package.iter_variants():
if self.validate:
try:
variant.validate_data()
except ResourceContentError as e:
result = ResourceSearchResult(
variant, "variant", str(e))
results.append(result)
continue
result = ResourceSearchResult(variant, "variant")
results.append(result)
except ResourceContentError:
# this may happen if 'variants' in package is malformed
continue
return resource_type, results
def __init__(self,
package_name,
version_range=None,
paths=None,
verbose=False):
"""Create a PackageHelp object.
Args:
package_name (str): Package to search.
version_range (`VersionRange`): Versions to search.
"""
self.package = None
self._verbose = verbose
self._sections = []
# find latest package with a help entry
package = None
it = iter_packages(package_name, range_=version_range)
packages = sorted(it, key=lambda x: x.version, reverse=True)
for package_ in packages:
if self._verbose:
print("searching for help in %s..." % package_.uri)
if package_.help:
package = package_
break
if package:
help_ = package.help
if isinstance(help_, basestring):
sections = [["Help", help_]]
elif isinstance(help_, list):
sections = help_
if self._verbose:
print("found %d help entries in %s." %
(len(sections), package.uri))
# create string formatter for help entries
if package.num_variants == 0:
base = package.base
root = base
else:
variant = package.get_variant(0)
base = variant.base
root = variant.root
formatter = scoped_formatter(base=base,
root=root,
config=config,
version=VersionBinding(
package.version),
system=system)
# format sections
for section in sections:
uri = section[1]
uri = convert_old_command_expansions(uri)
uri = uri.replace("$BROWSER", "").strip()
uri = formatter.format(uri)
section[1] = uri
self.package = package
self._sections = sections
def command(opts, parser, extra_arg_groups=None):
from rez.config import config
from rez.packages import iter_package_families, iter_packages
from rez.utils.yaml import dump_yaml
from rez.utils.memcached import Client
from rez.utils.formatting import columnise, readable_time_duration, \
readable_memory_size
import sys
memcache_client = Client(servers=config.memcached_uri,
debug=config.debug_memcache)
if not memcache_client:
print("memcaching is not enabled.", file=sys.stderr)
sys.exit(1)
if opts.poll:
poll(memcache_client, opts.interval)
return
if opts.flush:
memcache_client.flush(hard=True)
print("memcached servers are flushed.")
return
if opts.warm:
seen = set()
paths = config.nonlocal_packages_path
for family in iter_package_families(paths=paths):
if family.name in seen:
continue
for package in iter_packages(family.name, paths=paths):
if opts.verbose:
print("warming: %s" % package.qualified_name)
# forces package definition load, which puts in memcache
_ = package.data # noqa
seen.add(family.name)
print("memcached servers are warmed.")
return
if opts.reset_stats:
memcache_client.reset_stats()
print("memcached servers are stat reset.")
return
def _fail():
print("memcached servers are not responding.", file=sys.stderr)
sys.exit(1)
stats = memcache_client.get_stats()
if opts.stats:
if stats:
txt = dump_yaml(stats)
print(txt)
else:
_fail()
return
# print stats summary
if not stats:
_fail()
rows = [[
"CACHE SERVER", "UPTIME", "HITS", "MISSES", "HIT RATIO", "MEMORY",
"USED"
],
[
"------------", "------", "----", "------", "---------",
"------", "----"
]]
for server_id, stats_dict in stats:
server_uri = server_id.split()[0]
uptime = int(stats_dict.get("uptime", 0))
hits = int(stats_dict.get("get_hits", 0))
misses = int(stats_dict.get("get_misses", 0))
memory = int(stats_dict.get("limit_maxbytes", 0))
used = int(stats_dict.get("bytes", 0))
hit_ratio = float(hits) / max(hits + misses, 1)
hit_percent = int(hit_ratio * 100.0)
used_ratio = float(used) / max(memory, 1)
used_percent = int(used_ratio * 100.0)
row = (server_uri, readable_time_duration(uptime), str(hits),
str(misses), "%d%%" % hit_percent, readable_memory_size(memory),
"%s (%d%%)" % (readable_memory_size(used), used_percent))
rows.append(row)
print('\n'.join(columnise(rows)))
def _set_variant(self, variant, preloaded_packages=None):
self.clear()
hh = self.horizontalHeader()
self.setHorizontalHeaderLabels(["path", "released"])
QtCompat.QHeaderView.setSectionResizeMode(
hh, 0, QtWidgets.QHeaderView.Interactive)
hh.setStretchLastSection(True)
hh.setVisible(True)
package_paths = self.context_model.packages_path
package_filter = PackageFilterList.from_pod(
self.context_model.package_filter)
if variant and variant.wrapped.location in package_paths:
self.version_index = -1
self.reference_version_index = -1
reference_version = None
range_ = None
if self.reference_variant and self.reference_variant.name == variant.name:
reference_version = self.reference_variant.version
versions = sorted([reference_version, variant.version])
range_ = VersionRange.as_span(*versions)
timestamp = self.context().timestamp
if preloaded_packages is not None:
if range_ is None:
packages = preloaded_packages
else:
packages = [
x for x in preloaded_packages if x.version in range_
]
else:
it = iter_packages(name=variant.name,
paths=package_paths,
range_=range_)
packages = sorted(it, key=lambda x: x.version, reverse=True)
self.setRowCount(len(packages))
brush = self.palette().brush(QtGui.QPalette.Active,
QtGui.QPalette.Base)
for row, package in enumerate(packages):
version_str = str(package.version) + ' '
item = QtWidgets.QTableWidgetItem(version_str)
item.setTextAlignment(QtCore.Qt.AlignRight
| QtCore.Qt.AlignVCenter)
self.setVerticalHeaderItem(row, item)
if package.version == variant.version:
self.version_index = row
update_font(item, bold=True)
if reference_version is not None \
and package.version == reference_version:
self.reference_version_index = row
update_font(item, bold=True, italic=True)
def _item():
item_ = QtWidgets.QTableWidgetItem()
item_.setBackground(
brush) # get rid of mouse-hover coloring
return item_
if package.timestamp:
release_str = get_timestamp_str(package.timestamp)
in_future = (package.timestamp > timestamp)
else:
release_str = '-'
in_future = False
item = _item()
txt = package.uri + " "
icons = []
if in_future:
icon = get_icon_widget(
"clock_warning",
"package did not exist at time of resolve")
icons.append(icon)
rule = package_filter.excludes(package)
if rule:
icon = get_icon_widget(
"excluded",
"package was excluded by rule %s" % str(rule))
icons.append(icon)
if icons:
label = QtWidgets.QLabel(txt)
pane = create_pane(icons + [label, None],
True,
compact=True)
self.setCellWidget(row, 0, pane)
else:
item.setText(txt)
self.setItem(row, 0, item)
item = _item()
item.setText(release_str)
self.setItem(row, 1, item)
vh = self.verticalHeader()
vh.setVisible(True)
self.resizeColumnsToContents()
hh.setStretchLastSection(True)
self.update()
self.allow_selection = True
self.selectRow(self.version_index)
self.allow_selection = False
self.variant = variant
def set_package_name(self, package_name):
package_paths = self.context_model.packages_path
self.packages = {}
self.clear()
rows = []
busy_cursor = QtGui.QCursor(QtCore.Qt.WaitCursor)
QtWidgets.QApplication.setOverrideCursor(busy_cursor)
try:
packages = list(
iter_packages(name=str(package_name), paths=package_paths))
except RezError:
packages = []
if not packages:
self.setEnabled(False)
self.package_name = None
QtWidgets.QApplication.restoreOverrideCursor()
return
for i, package in enumerate(
sorted(packages, key=lambda x: x.version, reverse=True)):
version_str = str(package.version) + ' '
path_str = package.uri + " "
release_str = get_timestamp_str(package.timestamp) \
if package.timestamp else '-'
enabled = self.callback(package) if self.callback else True
rows.append((enabled, version_str, path_str, release_str))
self.packages[i] = package
QtWidgets.QApplication.restoreOverrideCursor()
self.setRowCount(len(rows))
first_selectable_row = -1
for i, row in enumerate(rows):
enabled, version_str = row[:2]
row = row[2:]
item = QtWidgets.QTableWidgetItem(version_str)
item.setTextAlignment(QtCore.Qt.AlignRight
| QtCore.Qt.AlignVCenter)
self.setVerticalHeaderItem(i, item)
for j in range(len(row)):
item = QtWidgets.QTableWidgetItem(row[j])
if enabled:
if first_selectable_row == -1:
first_selectable_row = i
else:
item.setFlags(QtCore.Qt.NoItemFlags)
self.setItem(i, j, item)
self.setHorizontalHeaderLabels(["path", "released"])
self.resizeRowsToContents()
self.resizeColumnsToContents()
vh = self.verticalHeader()
vh.setVisible(True)
hh = self.horizontalHeader()
hh.setStretchLastSection(True)
hh.setVisible(True)
self.package_name = package_name
self.setEnabled(True)
if first_selectable_row != -1:
self.selectRow(first_selectable_row)