def resolve_context(self, verbosity=0, max_fails=-1, timestamp=None, callback=None, buf=None, package_load_callback=None): """Update the current context by performing a re-resolve. The newly resolved context is only applied if it is a successful solve. Returns: `ResolvedContext` object, which may be a successful or failed solve. """ package_filter = PackageFilterList.from_pod(self.package_filter) context = ResolvedContext( self.request, package_paths=self.packages_path, package_filter=package_filter, verbosity=verbosity, max_fails=max_fails, timestamp=timestamp, buf=buf, callback=callback, package_load_callback=package_load_callback, caching=self.caching) if context.success: if self._context and self._context.load_path: context.set_load_path(self._context.load_path) self._set_context(context) self._modified = True return context
def create_build_context(self, variant, build_type, build_path): """Create a context to build the variant within.""" request = variant.get_requires(build_requires=True, private_build_requires=True) requests_str = ' '.join(map(str, request)) self._print("Resolving build environment: %s", requests_str) if build_type == BuildType.local: packages_path = self.package.config.packages_path else: packages_path = self.package.config.nonlocal_packages_path if self.package.config.is_overridden("package_filter"): from rez.package_filter import PackageFilterList data = self.package.config.package_filter package_filter = PackageFilterList.from_pod(data) else: package_filter = None context = ResolvedContext(request, package_paths=packages_path, package_filter=package_filter, building=True) if self.verbose: context.print_info() # save context before possible fail, so user can debug rxt_filepath = os.path.join(build_path, "build.rxt") context.save(rxt_filepath) if context.status != ResolverStatus.solved: raise BuildContextResolveError(context) return context, rxt_filepath
def resolve_context(self, verbosity=0, max_fails=-1, timestamp=None, callback=None, buf=None, package_load_callback=None): """Update the current context by performing a re-resolve. The newly resolved context is only applied if it is a successful solve. Returns: `ResolvedContext` object, which may be a successful or failed solve. """ package_filter = PackageFilterList.from_pod(self.package_filter) context = ResolvedContext( self.request, package_paths=self.packages_path, package_filter=package_filter, verbosity=verbosity, max_fails=max_fails, timestamp=timestamp, buf=buf, callback=callback, package_load_callback=package_load_callback) if context.success: if self._context and self._context.load_path: context.set_load_path(self._context.load_path) self._set_context(context) self._modified = True return context
def __init__(self, package_requests, package_paths, package_filter=None, timestamp=0, callback=None, building=False, verbosity=False, buf=None, package_load_callback=None, caching=True): """Create a Resolver. Args: package_requests: List of Requirement objects representing the request. package_paths: List of paths to search for pkgs. package_filter (`PackageFilterList`): Package filter. callback: See `Solver`. package_load_callback: If not None, this callable will be called prior to each package being loaded. It is passed a single `Package` object. building: True if we're resolving for a build. caching: If True, cache(s) may be used to speed the resolve. If False, caches will not be used. """ self.package_requests = package_requests self.package_paths = package_paths self.timestamp = timestamp self.callback = callback self.package_load_callback = package_load_callback self.building = building self.verbosity = verbosity self.caching = caching self.buf = buf # store hash of pre-timestamp-combined package filter. This is used in # the memcached key if package_filter: self.package_filter_hash = package_filter.hash else: self.package_filter_hash = '' # combine timestamp and package filter into single filter if self.timestamp: if package_filter: self.package_filter = package_filter.copy() else: self.package_filter = PackageFilterList() rule = TimestampRule.after(self.timestamp) self.package_filter.add_exclusion(rule) else: self.package_filter = package_filter self.status_ = ResolverStatus.pending self.resolved_packages_ = None self.failure_description = None self.graph_ = None self.from_cache = False self.memcached_servers = config.memcached_uri if config.resolve_caching else None self.solve_time = 0.0 # time spent solving self.load_time = 0.0 # time spent loading package resources self._print = config.debug_printer("resolve_memcache")
def test_empty_filter_list(self): """Test that empty filter list has no effect """ fltr = PackageFilterList() self._test( fltr, "pydad", ["1", "2", "3"] )
def create_build_context(self, variant, build_type, build_path): """Create a context to build the variant within.""" request = variant.get_requires(build_requires=True, private_build_requires=True) req_strs = map(str, request) quoted_req_strs = map(quote, req_strs) self._print("Resolving build environment: %s", ' '.join(quoted_req_strs)) if build_type == BuildType.local: packages_path = self.package.config.packages_path else: packages_path = self.package.config.nonlocal_packages_path # It is uncommon, but possible, to define the package filters in the # developer package. Example scenario: you may want to enable visiblity # of *.dev packages if the current package is *.dev also, for example # (assuming you have a production-time package filter which filters out # *.dev packages by default). # if self.package.config.is_overridden("package_filter"): from rez.package_filter import PackageFilterList data = self.package.config.package_filter package_filter = PackageFilterList.from_pod(data) else: package_filter = None # create the build context context = ResolvedContext(request, package_paths=packages_path, package_filter=package_filter, building=True) if self.verbose: context.print_info() # save context before possible fail, so user can debug rxt_filepath = os.path.join(build_path, "build.rxt") context.save(rxt_filepath) if context.status != ResolverStatus.solved: raise BuildContextResolveError(context) return context, rxt_filepath
def _test(self, fltr, pkg_family, expected_result): # convert from json if required if isinstance(fltr, dict): fltr = PackageFilter.from_pod(fltr) elif isinstance(fltr, list): fltr = PackageFilterList.from_pod(fltr) def filter_versions(fltr_): matching_versions = set() for pkg in iter_packages(pkg_family): if not fltr_.excludes(pkg): matching_versions.add(str(pkg.version)) self.assertEqual(matching_versions, set(expected_result)) # apply filter to all pkg versions filter_versions(fltr) # serialise to/from json and do it again data = fltr.to_pod() fltr2 = fltr.from_pod(data) filter_versions(fltr2)
def create_build_context(self, variant, build_type, build_path): """Create a context to build the variant within.""" request = variant.get_requires(build_requires=True, private_build_requires=True) req_strs = map(str, request) quoted_req_strs = map(quote, req_strs) self._print("Resolving build environment: %s", ' '.join(quoted_req_strs)) if build_type == BuildType.local: packages_path = self.package.config.packages_path else: packages_path = self.package.config.nonlocal_packages_path if self.package.config.is_overridden("package_filter"): from rez.package_filter import PackageFilterList data = self.package.config.package_filter package_filter = PackageFilterList.from_pod(data) else: package_filter = None context = ResolvedContext(request, package_paths=packages_path, package_filter=package_filter, building=True) if self.verbose: context.print_info() # save context before possible fail, so user can debug rxt_filepath = os.path.join(build_path, "build.rxt") context.save(rxt_filepath) if context.status != ResolverStatus.solved: raise BuildContextResolveError(context) return context, rxt_filepath
def _contextChanged(self, flags=0): self._set_stale(self.context_model.is_stale()) if flags & (ContextModel.PACKAGES_PATH_CHANGED | ContextModel.CONTEXT_CHANGED): # update icons new_icons = [] if self.variant.index is not None: package = self.variant.parent if package.num_variants > 1: txt = "1 of %d variants" % package.num_variants new_icons.append(("variant", txt)) if self.variant.is_local: new_icons.append(("local", "package is local")) package_paths = PackageSearchPath(self.context_model.packages_path) package_filter = PackageFilterList.from_pod(self.context_model.package_filter) # TODO: move this all into a thread, it's blocking up the GUI during context load if self.variant in package_paths: # find all >= version packages, so we can determine tick type ge_range = VersionRange.from_version(self.variant.version, ">=") packages = None try: it = package_paths.iter_packages(name=self.variant.name, range_=ge_range) packages = sorted(it, key=lambda x: x.version) except: pass # apply a tick icon if appropriate ticked = False if packages: # test if variant is latest package latest_pkg = packages[-1] if self.variant.version == latest_pkg.version: new_icons.append(("green_tick", "package is latest")) ticked = True range_ = None packages_ = None # test if variant is in request, and is latest possible if not ticked: range_ = None try: request = self.context().requested_packages(True) reqlist = RequirementList(request) if self.variant.name in reqlist.names: variant_ = reqlist.get(self.variant.name) if not variant_.conflict: range_ = variant_.range except: pass if range_ is not None: packages_ = [x for x in packages if x.version in range_] if packages_: latest_pkg = packages_[-1] if self.variant.version == latest_pkg.version: new_icons.append(("yellow_tick", "package is latest possible")) ticked = True packages2 = [x for x in (packages_ or packages) if x.version > self.variant.version] # test if variant is latest within package filter if (not ticked and packages2 and package_filter): if all(package_filter.excludes(x) for x in packages2): new_icons.append(("yellow_tick", "package is latest possible")) ticked = True # test if variant was latest package at time of resolve if not ticked and self.variant.timestamp: untimestamped_packages = [x for x in packages if not x.timestamp] if not untimestamped_packages: resolve_time = self.context().timestamp old_packages = [x for x in packages if x.timestamp <= resolve_time] if old_packages: latest_pkg = old_packages[-1] if self.variant.version == latest_pkg.version: new_icons.append( ("green_white_tick", "package was latest at time of resolve")) ticked = True # test if variant is in request, and was latest possible at # the time of resolve if (not ticked and self.variant.timestamp and range_ is not None and packages_ is not None): untimestamped_packages = any(x for x in packages_ if not x.timestamp) if not untimestamped_packages: resolve_time = self.context().timestamp old_packages = [x for x in packages_ if x.timestamp <= resolve_time] if old_packages: latest_pkg = old_packages[-1] if self.variant.version == latest_pkg.version: new_icons.append( ("yellow_white_tick", "package was latest possible at time of resolve")) ticked = True # test if variant is within package filter, and was latest # possible at the time of resolve if (not ticked and packages2 and package_filter and self.variant.timestamp): untimestamped_packages = any(x for x in (packages_ or packages) if not x.timestamp) if not untimestamped_packages: newer_package = any(x for x in packages2 if not package_filter.excludes(x) and x.timestamp <= resolve_time) if not newer_package: new_icons.append( ("yellow_white_tick", "package was latest possible at time of resolve")) ticked = True # bring in the old man if not ticked: new_icons.append( ("old_man", "newer packages are/were available")) else: new_icons.append(("error", "package is not in the search path")) self._set_icons(new_icons) if (not self.hide_locks) and (flags & (ContextModel.LOCKS_CHANGED | ContextModel.CONTEXT_CHANGED)): # update lock icon lock = self.context_model.get_patch_lock(self.variant.name) if lock is None: lock = self.context_model.default_patch_lock icon_name = "%s_faint" % lock.name else: icon_name = lock.name # update lock tooltip if lock == PatchLock.no_lock: desc = lock.description else: req = self._get_lock_requirement(lock) if req: if lock == PatchLock.lock: desc = "Exact version (%s)" % str(req) else: unit = lock.description.split()[0] desc = ("%s version updates only (%s.*)" % (unit.capitalize(), str(req))) else: desc = lock.description self._set_lock_icon(icon_name, desc.lower())
def __init__(self, context, package_requests, package_paths, package_filter=None, package_orderers=None, timestamp=0, callback=None, building=False, verbosity=False, buf=None, package_load_callback=None, caching=True, suppress_passive=False, print_stats=False): """Create a Resolver. Args: package_requests: List of Requirement objects representing the request. package_paths: List of paths to search for pkgs. package_filter (`PackageFilterList`): Package filter. package_orderers (list of `PackageOrder`): Custom package ordering. callback: See `Solver`. package_load_callback: If not None, this callable will be called prior to each package being loaded. It is passed a single `Package` object. building: True if we're resolving for a build. caching: If True, cache(s) may be used to speed the resolve. If False, caches will not be used. print_stats (bool): If true, print advanced solver stats at the end. """ self.context = context self.package_requests = package_requests self.package_paths = package_paths self.timestamp = timestamp self.callback = callback self.package_orderers = package_orderers self.package_load_callback = package_load_callback self.building = building self.verbosity = verbosity self.caching = caching self.buf = buf self.suppress_passive = suppress_passive self.print_stats = print_stats # store hash of package orderers. This is used in the memcached key if package_orderers: sha1s = ''.join(x.sha1 for x in package_orderers) self.package_orderers_hash = sha1(sha1s.encode("utf8")).hexdigest() else: self.package_orderers_hash = '' # store hash of pre-timestamp-combined package filter. This is used in # the memcached key if package_filter: self.package_filter_hash = package_filter.sha1 else: self.package_filter_hash = '' # combine timestamp and package filter into single filter if self.timestamp: if package_filter: self.package_filter = package_filter.copy() else: self.package_filter = PackageFilterList() rule = TimestampRule.after(self.timestamp) self.package_filter.add_exclusion(rule) else: self.package_filter = package_filter self.status_ = ResolverStatus.pending self.resolved_packages_ = None self.resolved_ephemerals_ = None self.failure_description = None self.graph_ = None self.from_cache = False self.memcached_servers = config.memcached_uri if config.resolve_caching else None self.solve_time = 0.0 # time spent solving self.load_time = 0.0 # time spent loading package resources self._print = config.debug_printer("resolve_memcache")
class Resolver(object): """The package resolver. The Resolver uses a combination of Solver(s) and cache(s) to resolve a package request as quickly as possible. """ def __init__(self, context, package_requests, package_paths, package_filter=None, package_orderers=None, timestamp=0, callback=None, building=False, verbosity=False, buf=None, package_load_callback=None, caching=True, suppress_passive=False, print_stats=False): """Create a Resolver. Args: package_requests: List of Requirement objects representing the request. package_paths: List of paths to search for pkgs. package_filter (`PackageFilterList`): Package filter. package_orderers (list of `PackageOrder`): Custom package ordering. callback: See `Solver`. package_load_callback: If not None, this callable will be called prior to each package being loaded. It is passed a single `Package` object. building: True if we're resolving for a build. caching: If True, cache(s) may be used to speed the resolve. If False, caches will not be used. print_stats (bool): If true, print advanced solver stats at the end. """ self.context = context self.package_requests = package_requests self.package_paths = package_paths self.timestamp = timestamp self.callback = callback self.package_orderers = package_orderers self.package_load_callback = package_load_callback self.building = building self.verbosity = verbosity self.caching = caching self.buf = buf self.suppress_passive = suppress_passive self.print_stats = print_stats # store hash of package orderers. This is used in the memcached key if package_orderers: sha1s = ''.join(x.sha1 for x in package_orderers) self.package_orderers_hash = sha1(sha1s.encode("utf8")).hexdigest() else: self.package_orderers_hash = '' # store hash of pre-timestamp-combined package filter. This is used in # the memcached key if package_filter: self.package_filter_hash = package_filter.sha1 else: self.package_filter_hash = '' # combine timestamp and package filter into single filter if self.timestamp: if package_filter: self.package_filter = package_filter.copy() else: self.package_filter = PackageFilterList() rule = TimestampRule.after(self.timestamp) self.package_filter.add_exclusion(rule) else: self.package_filter = package_filter self.status_ = ResolverStatus.pending self.resolved_packages_ = None self.resolved_ephemerals_ = None self.failure_description = None self.graph_ = None self.from_cache = False self.memcached_servers = config.memcached_uri if config.resolve_caching else None self.solve_time = 0.0 # time spent solving self.load_time = 0.0 # time spent loading package resources self._print = config.debug_printer("resolve_memcache") @pool_memcached_connections def solve(self): """Perform the solve. """ with log_duration(self._print, "memcache get (resolve) took %s"): solver_dict = self._get_cached_solve() if solver_dict: self.from_cache = True self._set_result(solver_dict) else: self.from_cache = False solver = self._solve() solver_dict = self._solver_to_dict(solver) self._set_result(solver_dict) with log_duration(self._print, "memcache set (resolve) took %s"): self._set_cached_solve(solver_dict) @property def status(self): """Return the current status of the resolve. Returns: ResolverStatus. """ return self.status_ @property def resolved_packages(self): """Get the list of resolved packages. Returns: List of `PackageVariant` objects, or None if the resolve has not completed. """ return self.resolved_packages_ @property def resolved_ephemerals(self): """Get the list of resolved ewphemerals. Returns: List of `Requirement` objects, or None if the resolve has not completed. """ return self.resolved_ephemerals_ @property def graph(self): """Return the resolve graph. The resolve graph shows unsuccessful as well as successful resolves. Returns: A pygraph.digraph object, or None if the solve has not completed. """ return self.graph_ def _get_variant(self, variant_handle): return get_variant(variant_handle, context=self.context) def _get_cached_solve(self): """Find a memcached resolve. If there is NOT a resolve timestamp: - fetch a non-timestamped memcache entry; - if no entry, then fail; - if packages have changed, then: - delete the entry; - fail; - if no packages in the entry have been released since, then - use the entry and return; - else: - delete the entry; - fail. If there IS a resolve timestamp (let us call this T): - fetch a non-timestamped memcache entry; - if entry then: - if no packages have changed, then: - if no packages in the entry have been released since: - if no packages in the entry were released after T, then - use the entry and return; - else: - delete the entry; - else: - delete the entry; - fetch a timestamped (T) memcache entry; - if no entry, then fail; - if packages have changed, then: - delete the entry; - fail; - else: - use the entry. This behaviour exists specifically so that resolves that use a timestamp but set that to the current time, can be reused by other resolves if nothing has changed. Older resolves however, can only be reused if the timestamp matches exactly (but this might happen a lot - consider a workflow where a work area is tied down to a particular timestamp in order to 'lock' it from any further software releases). """ if not (self.caching and self.memcached_servers): return None # these caches avoids some potentially repeated file stats variant_states = {} last_release_times = {} def _hit(data): solver_dict, _, _ = data return solver_dict def _miss(): self._print("No cache key retrieved") return None def _delete_cache_entry(key): with self._memcached_client() as client: client.delete(key) self._print("Discarded entry: %r", key) def _retrieve(timestamped): key = self._memcache_key(timestamped=timestamped) self._print("Retrieving memcache key: %r", key) with self._memcached_client() as client: data = client.get(key) return key, data def _packages_changed(key, data): solver_dict, _, variant_states_dict = data for variant_handle in solver_dict.get("variant_handles", []): variant = self._get_variant(variant_handle) old_state = variant_states_dict.get(variant.name) new_state = variant_states.get(variant) if new_state is None: try: repo = variant.resource._repository new_state = repo.get_variant_state_handle( variant.resource) except (IOError, OSError) as e: # if, ie a package file was deleted on disk, then # an IOError or OSError will be raised when we try to # read from it - assume that the packages have changed! self._print( "Error loading %r (assuming cached state " "changed): %s", variant.qualified_name, e) return True variant_states[variant] = new_state if old_state != new_state: self._print("%r has been modified", variant.qualified_name) return True return False def _releases_since_solve(key, data): _, release_times_dict, _ = data for package_name, release_time in release_times_dict.items(): time_ = last_release_times.get(package_name) if time_ is None: time_ = get_last_release_time(package_name, self.package_paths) last_release_times[package_name] = time_ if time_ != release_time: self._print( "A newer version of %r (%d) has been released since the " "resolve was cached (latest release in cache was %d) " "(entry: %r)", package_name, time_, release_time, key) return True return False def _timestamp_is_earlier(key, data): _, release_times_dict, _ = data for package_name, release_time in release_times_dict.items(): if self.timestamp < release_time: self._print( "Resolve timestamp (%d) is earlier than %r in " "solve (%d) (entry: %r)", self.timestamp, package_name, release_time, key) return True return False key, data = _retrieve(False) if self.timestamp: if data: if _packages_changed(key, data) or _releases_since_solve( key, data): _delete_cache_entry(key) elif not _timestamp_is_earlier(key, data): return _hit(data) key, data = _retrieve(True) if not data: return _miss() if _packages_changed(key, data): _delete_cache_entry(key) return _miss() else: return _hit(data) else: if not data: return _miss() if _packages_changed(key, data) or _releases_since_solve( key, data): _delete_cache_entry(key) return _miss() else: return _hit(data) @contextmanager def _memcached_client(self): with memcached_client(self.memcached_servers, debug=config.debug_memcache) as client: yield client def _set_cached_solve(self, solver_dict): """Store a solve to memcached. If there is NOT a resolve timestamp: - store the solve to a non-timestamped entry. If there IS a resolve timestamp (let us call this T): - if NO newer package in the solve has been released since T, - then store the solve to a non-timestamped entry; - else: - store the solve to a timestamped entry. """ if self.status_ != ResolverStatus.solved: return # don't cache failed solves if not (self.caching and self.memcached_servers): return # most recent release times get stored with solve result in the cache releases_since_solve = False release_times_dict = {} variant_states_dict = {} for variant in self.resolved_packages_: time_ = get_last_release_time(variant.name, self.package_paths) # don't cache if a release time isn't known if time_ == 0: self._print( "Did not send memcache key: a repository could " "not provide a most recent release time for %r", variant.name) return if self.timestamp and self.timestamp < time_: releases_since_solve = True release_times_dict[variant.name] = time_ repo = variant.resource._repository variant_states_dict[variant.name] = \ repo.get_variant_state_handle(variant.resource) timestamped = (self.timestamp and releases_since_solve) key = self._memcache_key(timestamped=timestamped) data = (solver_dict, release_times_dict, variant_states_dict) with self._memcached_client() as client: client.set(key, data) self._print("Sent memcache key: %r", key) def _memcache_key(self, timestamped=False): """Makes a key suitable as a memcache entry.""" request = tuple(map(str, self.package_requests)) repo_ids = [] for path in self.package_paths: repo = package_repository_manager.get_repository(path) repo_ids.append(repo.uid) t = [ "resolve", request, tuple(repo_ids), self.package_filter_hash, self.package_orderers_hash, self.building, config.prune_failed_graph ] if timestamped and self.timestamp: t.append(self.timestamp) return str(tuple(t)) def _solve(self): solver = Solver(package_requests=self.package_requests, package_paths=self.package_paths, context=self.context, package_filter=self.package_filter, package_orderers=self.package_orderers, callback=self.callback, package_load_callback=self.package_load_callback, building=self.building, verbosity=self.verbosity, prune_unfailed=config.prune_failed_graph, buf=self.buf, suppress_passive=self.suppress_passive, print_stats=self.print_stats) solver.solve() return solver def _set_result(self, solver_dict): self.status_ = solver_dict.get("status") self.graph_ = solver_dict.get("graph") self.solve_time = solver_dict.get("solve_time") self.load_time = solver_dict.get("load_time") self.failure_description = solver_dict.get("failure_description") self.resolved_packages_ = None self.resolved_ephemerals_ = None if self.status_ == ResolverStatus.solved: # convert solver.Variants to packages.Variants self.resolved_packages_ = [] for variant_handle in solver_dict.get("variant_handles", []): variant = self._get_variant(variant_handle) self.resolved_packages_.append(variant) self.resolved_ephemerals_ = [] for req_str in solver_dict.get("ephemerals", []): req = Requirement(req_str) self.resolved_ephemerals_.append(req) @classmethod def _solver_to_dict(cls, solver): graph_ = solver.get_graph() solve_time = solver.solve_time load_time = solver.load_time failure_description = None variant_handles = None ephemerals = None st = solver.status if st == SolverStatus.unsolved: status_ = ResolverStatus.aborted failure_description = solver.abort_reason elif st == SolverStatus.failed: status_ = ResolverStatus.failed failure_description = solver.failure_description() elif st == SolverStatus.solved: status_ = ResolverStatus.solved variant_handles = [] for solver_variant in solver.resolved_packages: variant_handle_dict = solver_variant.handle variant_handles.append(variant_handle_dict) ephemerals = [] for ephemeral in solver.resolved_ephemerals: ephemerals.append(str(ephemeral)) return dict(status=status_, graph=graph_, solve_time=solve_time, load_time=load_time, failure_description=failure_description, variant_handles=variant_handles, ephemerals=ephemerals)
def command(opts, parser, extra_arg_groups=None): from rez.resolved_context import ResolvedContext from rez.resolver import ResolverStatus from rez.package_filter import PackageFilterList, Rule from rez.utils.formatting import get_epoch_time_from_str from rez.config import config import select import sys import os import os.path command = opts.command if extra_arg_groups: if opts.command: parser.error("argument --command: not allowed with arguments after '--'") command = extra_arg_groups[0] or None context = None request = opts.PKG t = get_epoch_time_from_str(opts.time) if opts.time else None if opts.isolated: config.inherit_parent_environment = False if opts.inherited: config.inherit_parent_environment = True if opts.paths is None: pkg_paths = (config.nonlocal_packages_path if opts.no_local else None) else: pkg_paths = opts.paths.split(os.pathsep) pkg_paths = [os.path.expanduser(x) for x in pkg_paths if x] if opts.input: if opts.PKG and not opts.patch: parser.error("Cannot use --input and provide PKG(s), unless patching.") context = ResolvedContext.load(opts.input) if opts.patch: if context is None: from rez.status import status context = status.context if context is None: print("cannot patch: not in a context", file=sys.stderr) sys.exit(1) # modify the request in terms of the given patch request request = context.get_patched_request(request, strict=opts.strict, rank=opts.patch_rank) context = None if opts.self: from rez.utils._version import _rez_version request += ["bleeding_rez==%s" % _rez_version] request += ["python"] # Required by Rez if context is None: # create package filters if opts.no_filters: package_filter = PackageFilterList() else: package_filter = PackageFilterList.singleton.copy() for rule_str in (opts.exclude or []): rule = Rule.parse_rule(rule_str) package_filter.add_exclusion(rule) for rule_str in (opts.include or []): rule = Rule.parse_rule(rule_str) package_filter.add_inclusion(rule) # perform the resolve context = ResolvedContext(package_requests=request, timestamp=t, package_paths=pkg_paths, building=opts.build, package_filter=package_filter, add_implicit_packages=(not opts.no_implicit), verbosity=opts.verbose, max_fails=opts.max_fails, time_limit=opts.time_limit, caching=(not opts.no_cache), suppress_passive=opts.no_passive, print_stats=opts.stats) success = (context.status == ResolverStatus.solved) if not success: context.print_info(buf=sys.stderr) if opts.fail_graph: if context.graph: from rez.utils.graph_utils import view_graph g = context.graph(as_dot=True) view_graph(g) else: print("the failed resolve context did not generate a graph.", file=sys.stderr) if opts.output: if opts.output == '-': # print to stdout context.write_to_buffer(sys.stdout) else: context.save(opts.output) sys.exit(0 if success else 1) if not success: sys.exit(1) if opts.env: env = {} for pair in opts.env: key, value = pair.split("=") env[key.upper()] = value config.additional_environment = env # generally shells will behave as though the '-s' flag was not present when # no stdin is available. So here we replicate this behaviour. try: if opts.stdin and not select.select([sys.stdin], [], [], 0.0)[0]: opts.stdin = False except select.error: pass # because windows quiet = opts.quiet or bool(command) returncode, _, _ = context.execute_shell( shell=opts.shell, rcfile=opts.rcfile, norc=opts.norc, command=command, stdin=opts.stdin, quiet=quiet, start_new_session=opts.new_session, detached=opts.detached, pre_command=opts.pre_command, block=True) sys.exit(returncode)
class Resolver(object): """The package resolver. The Resolver uses a combination of Solver(s) and cache(s) to resolve a package request as quickly as possible. """ def __init__(self, package_requests, package_paths, package_filter=None, timestamp=0, callback=None, building=False, verbosity=False, buf=None, package_load_callback=None, caching=True): """Create a Resolver. Args: package_requests: List of Requirement objects representing the request. package_paths: List of paths to search for pkgs. package_filter (`PackageFilterList`): Package filter. callback: See `Solver`. package_load_callback: If not None, this callable will be called prior to each package being loaded. It is passed a single `Package` object. building: True if we're resolving for a build. caching: If True, cache(s) may be used to speed the resolve. If False, caches will not be used. """ self.package_requests = package_requests self.package_paths = package_paths self.timestamp = timestamp self.callback = callback self.package_load_callback = package_load_callback self.building = building self.verbosity = verbosity self.caching = caching self.buf = buf # store hash of pre-timestamp-combined package filter. This is used in # the memcached key if package_filter: self.package_filter_hash = package_filter.hash else: self.package_filter_hash = '' # combine timestamp and package filter into single filter if self.timestamp: if package_filter: self.package_filter = package_filter.copy() else: self.package_filter = PackageFilterList() rule = TimestampRule.after(self.timestamp) self.package_filter.add_exclusion(rule) else: self.package_filter = package_filter self.status_ = ResolverStatus.pending self.resolved_packages_ = None self.failure_description = None self.graph_ = None self.from_cache = False self.memcached_servers = config.memcached_uri if config.resolve_caching else None self.solve_time = 0.0 # time spent solving self.load_time = 0.0 # time spent loading package resources self._print = config.debug_printer("resolve_memcache") @pool_memcached_connections def solve(self): """Perform the solve. """ solver_dict = self._get_cached_solve() if solver_dict: self.from_cache = True self._set_result(solver_dict) else: self.from_cache = False solver = self._solve() solver_dict = self._solver_to_dict(solver) self._set_result(solver_dict) self._set_cached_solve(solver_dict) @property def status(self): """Return the current status of the resolve. Returns: ResolverStatus. """ return self.status_ @property def resolved_packages(self): """Get the list of resolved packages. Returns: List of `PackageVariant` objects, or None if the resolve has not completed. """ return self.resolved_packages_ @property def graph(self): """Return the resolve graph. The resolve graph shows unsuccessful as well as successful resolves. Returns: A pygraph.digraph object, or None if the solve has not completed. """ return self.graph_ def _get_cached_solve(self): """Find a memcached resolve. If there is NOT a resolve timestamp: - fetch a non-timestamped memcache entry; - if no entry, then fail; - if packages have changed, then: - delete the entry; - fail; - if no packages in the entry have been released since, then - use the entry and return; - else: - delete the entry; - fail. If there IS a resolve timestamp (let us call this T): - fetch a non-timestamped memcache entry; - if entry then: - if no packages have changed, then: - if no packages in the entry have been released since: - if no packages in the entry were released after T, then - use the entry and return; - else: - delete the entry; - else: - delete the entry; - fetch a timestamped (T) memcache entry; - if no entry, then fail; - if packages have changed, then: - delete the entry; - fail; - else: - use the entry. This behaviour exists specifically so that resolves that use use a timestamp but set that to the current time, can be reused by other resolves if nothing has changed. Older resolves however, can only be reused if the timestamp matches exactly (but this might happen a lot - consider a workflow where a work area is tied down to a particular timestamp in order to 'lock' it from any further software releases). """ if not (self.caching and self.memcached_servers): return None def _hit(data): solver_dict, _, _ = data return solver_dict def _miss(): self._print("No cache key retrieved") return None def _delete_cache_entry(key): with self._memcached_client() as client: client.delete(key) self._print("Discarded entry: %r", key) def _retrieve(timestamped): key = self._memcache_key(timestamped=timestamped) self._print("Retrieving memcache key: %r", key) with self._memcached_client() as client: data = client.get(key) return key, data def _packages_changed(key, data): solver_dict, _, variant_states_dict = data for variant_handle in solver_dict.get("variant_handles", []): variant = get_variant(variant_handle) old_state = variant_states_dict.get(variant.name) repo = variant.resource._repository new_state = repo.get_variant_state_handle(variant.resource) if old_state != new_state: self._print("%r has been modified", variant.qualified_name) return True return False def _releases_since_solve(key, data): _, release_times_dict, _ = data for package_name, release_time in release_times_dict.iteritems(): time_ = get_last_release_time(package_name, self.package_paths) if time_ != release_time: self._print( "A newer version of %r (%d) has been released since the " "resolve was cached (latest release in cache was %d) " "(entry: %r)", package_name, time_, release_time, key) return True return False def _timestamp_is_earlier(key, data): _, release_times_dict, _ = data for package_name, release_time in release_times_dict.iteritems(): if self.timestamp < release_time: self._print("Resolve timestamp (%d) is earlier than %r in " "solve (%d) (entry: %r)", self.timestamp, package_name, release_time, key) return True return False key, data = _retrieve(False) if self.timestamp: if data: if _packages_changed(key, data): _delete_cache_entry(key) elif _releases_since_solve(key, data): _delete_cache_entry(key) elif not _timestamp_is_earlier(key, data): return _hit(data) key, data = _retrieve(True) if not data: return _miss() if _packages_changed(key, data): _delete_cache_entry(key) return _miss() else: return _hit(data) else: if not data: return _miss() if _packages_changed(key, data): _delete_cache_entry(key) return _miss() if _releases_since_solve(key, data): _delete_cache_entry(key) return _miss() else: return _hit(data) @contextmanager def _memcached_client(self): with memcached_client(self.memcached_servers, debug=config.debug_memcache) as client: yield client def _set_cached_solve(self, solver_dict): """Store a solve to memcached. If there is NOT a resolve timestamp: - store the solve to a non-timestamped entry. If there IS a resolve timestamp (let us call this T): - if NO newer package in the solve has been released since T, - then store the solve to a non-timestamped entry; - else: - store the solve to a timestamped entry. """ if self.status_ != ResolverStatus.solved: return # don't cache failed solves if not (self.caching and self.memcached_servers): return # most recent release times get stored with solve result in the cache releases_since_solve = False release_times_dict = {} variant_states_dict = {} for variant in self.resolved_packages_: time_ = get_last_release_time(variant.name, self.package_paths) # don't cache if a release time isn't known if time_ == 0: self._print("Did not send memcache key: a repository could " "not provide a most recent release time for %r", variant.name) return if self.timestamp and self.timestamp < time_: releases_since_solve = True release_times_dict[variant.name] = time_ repo = variant.resource._repository variant_states_dict[variant.name] = \ repo.get_variant_state_handle(variant.resource) timestamped = (self.timestamp and releases_since_solve) key = self._memcache_key(timestamped=timestamped) data = (solver_dict, release_times_dict, variant_states_dict) with self._memcached_client() as client: client.set(key, data) self._print("Sent memcache key: %r", key) def _memcache_key(self, timestamped=False): """Makes a key suitable as a memcache entry.""" request = tuple(map(str, self.package_requests)) repo_ids = [] for path in self.package_paths: repo = package_repository_manager.get_repository(path) repo_ids.append(repo.uid) t = ["resolve", request, tuple(repo_ids), self.package_filter_hash, self.building, config.prune_failed_graph] if timestamped and self.timestamp: t.append(self.timestamp) return str(tuple(t)) def _solve(self): solver = Solver(package_requests=self.package_requests, package_paths=self.package_paths, package_filter=self.package_filter, callback=self.callback, package_load_callback=self.package_load_callback, building=self.building, verbosity=self.verbosity, prune_unfailed=config.prune_failed_graph, buf=self.buf) solver.solve() return solver def _set_result(self, solver_dict): self.status_ = solver_dict.get("status") self.graph_ = solver_dict.get("graph") self.solve_time = solver_dict.get("solve_time") self.load_time = solver_dict.get("load_time") self.failure_description = solver_dict.get("failure_description") self.resolved_packages_ = None if self.status_ == ResolverStatus.solved: # convert solver.Variants to packages.Variants self.resolved_packages_ = [] for variant_handle in solver_dict.get("variant_handles", []): variant = get_variant(variant_handle) self.resolved_packages_.append(variant) @classmethod def _solver_to_dict(cls, solver): graph_ = solver.get_graph() solve_time = solver.solve_time load_time = solver.load_time failure_description = None variant_handles = None st = solver.status if st == SolverStatus.unsolved: status_ = ResolverStatus.aborted failure_description = solver.abort_reason elif st == SolverStatus.failed: status_ = ResolverStatus.failed failure_description = solver.failure_description() elif st == SolverStatus.solved: status_ = ResolverStatus.solved variant_handles = [] for solver_variant in solver.resolved_packages: variant_handle_dict = solver_variant.userdata variant_handles.append(variant_handle_dict) return dict( status=status_, graph=graph_, solve_time=solve_time, load_time=load_time, failure_description=failure_description, variant_handles=variant_handles)
def command(opts, parser, extra_arg_groups=None): from rez.resolved_context import ResolvedContext from rez.resolver import ResolverStatus from rez.package_filter import PackageFilterList, Rule from rez.utils.formatting import get_epoch_time_from_str from rez.config import config import select import sys import os import os.path command = opts.command if extra_arg_groups: if opts.command: parser.error( "argument --command: not allowed with arguments after '--'") command = extra_arg_groups[0] or None context = None request = opts.PKG t = get_epoch_time_from_str(opts.time) if opts.time else None if opts.paths is None: pkg_paths = (config.nonlocal_packages_path if opts.no_local else None) else: pkg_paths = opts.paths.split(os.pathsep) pkg_paths = [os.path.expanduser(x) for x in pkg_paths if x] if opts.input: if opts.PKG: parser.error( "Cannot use --input and provide PKG(s) at the same time") context = ResolvedContext.load(opts.input) if context.status != ResolverStatus.solved: print >> sys.stderr, "cannot rez-env into a failed context" sys.exit(1) if opts.patch: # TODO: patching is lagging behind in options, and needs to be updated # anyway. if context is None: from rez.status import status context = status.context if context is None: print >> sys.stderr, "cannot patch: not in a context" sys.exit(1) request = context.get_patched_request(request, strict=opts.strict, rank=opts.patch_rank) context = None if context is None: # create package filters if opts.no_filters: package_filter = PackageFilterList() else: package_filter = PackageFilterList.singleton.copy() for rule_str in (opts.exclude or []): rule = Rule.parse_rule(rule_str) package_filter.add_exclusion(rule) for rule_str in (opts.include or []): rule = Rule.parse_rule(rule_str) package_filter.add_inclusion(rule) # perform the resolve context = ResolvedContext(package_requests=request, timestamp=t, package_paths=pkg_paths, building=opts.build, package_filter=package_filter, add_implicit_packages=(not opts.no_implicit), verbosity=opts.verbose, max_fails=opts.max_fails, time_limit=opts.time_limit, caching=(not opts.no_cache)) success = (context.status == ResolverStatus.solved) if not success: context.print_info(buf=sys.stderr) if opts.output: if opts.output == '-': # print to stdout context.write_to_buffer(sys.stdout) else: context.save(opts.output) sys.exit(0 if success else 1) if not success: sys.exit(1) # generally shells will behave as though the '-s' flag was not present when # no stdin is available. So here we replicate this behaviour. if opts.stdin and not select.select([sys.stdin], [], [], 0.0)[0]: opts.stdin = False quiet = opts.quiet or bool(command) returncode, _, _ = context.execute_shell( shell=opts.shell, rcfile=opts.rcfile, norc=opts.norc, command=command, stdin=opts.stdin, quiet=quiet, start_new_session=opts.new_session, detached=opts.detached, pre_command=opts.pre_command, block=True) sys.exit(returncode)
def _set_variant(self, variant, preloaded_packages=None): self.clear() hh = self.horizontalHeader() self.setHorizontalHeaderLabels(["path", "released"]) hh.setResizeMode(0, QtGui.QHeaderView.Interactive) hh.setStretchLastSection(True) hh.setVisible(True) package_paths = self.context_model.packages_path package_filter = PackageFilterList.from_pod( self.context_model.package_filter) if variant and variant.wrapped.location in package_paths: self.version_index = -1 self.reference_version_index = -1 reference_version = None range_ = None if self.reference_variant and self.reference_variant.name == variant.name: reference_version = self.reference_variant.version versions = sorted([reference_version, variant.version]) range_ = VersionRange.as_span(*versions) timestamp = self.context().timestamp if preloaded_packages is not None: if range_ is None: packages = preloaded_packages else: packages = [ x for x in preloaded_packages if x.version in range_ ] else: it = iter_packages(name=variant.name, paths=package_paths, range_=range_) packages = sorted(it, key=lambda x: x.version, reverse=True) self.setRowCount(len(packages)) brush = self.palette().brush(QtGui.QPalette.Active, QtGui.QPalette.Base) for row, package in enumerate(packages): version_str = str(package.version) + ' ' item = QtGui.QTableWidgetItem(version_str) item.setTextAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter) self.setVerticalHeaderItem(row, item) if package.version == variant.version: self.version_index = row update_font(item, bold=True) if reference_version is not None \ and package.version == reference_version: self.reference_version_index = row update_font(item, bold=True, italic=True) def _item(): item_ = QtGui.QTableWidgetItem() item_.setBackground( brush) # get rid of mouse-hover coloring return item_ if package.timestamp: release_str = get_timestamp_str(package.timestamp) in_future = (package.timestamp > timestamp) else: release_str = '-' in_future = False item = _item() txt = package.uri + " " icons = [] if in_future: icon = get_icon_widget( "clock_warning", "package did not exist at time of resolve") icons.append(icon) rule = package_filter.excludes(package) if rule: icon = get_icon_widget( "excluded", "package was excluded by rule %s" % str(rule)) icons.append(icon) if icons: label = QtGui.QLabel(txt) pane = create_pane(icons + [label, None], True, compact=True) self.setCellWidget(row, 0, pane) else: item.setText(txt) self.setItem(row, 0, item) item = _item() item.setText(release_str) self.setItem(row, 1, item) vh = self.verticalHeader() vh.setVisible(True) self.resizeColumnsToContents() hh.setStretchLastSection(True) self.update() self.allow_selection = True self.selectRow(self.version_index) self.allow_selection = False self.variant = variant
def command(opts, parser, extra_arg_groups=None): from rez.resolved_context import ResolvedContext from rez.resolver import ResolverStatus from rez.package_filter import PackageFilterList, Rule from rez.utils.formatting import get_epoch_time_from_str from rez.config import config import select import sys import os import os.path command = opts.command if extra_arg_groups: if opts.command: parser.error("argument --command: not allowed with arguments after '--'") command = extra_arg_groups[0] or None context = None request = opts.PKG t = get_epoch_time_from_str(opts.time) if opts.time else None if opts.paths is None: pkg_paths = (config.nonlocal_packages_path if opts.no_local else None) else: pkg_paths = opts.paths.split(os.pathsep) pkg_paths = [os.path.expanduser(x) for x in pkg_paths if x] if opts.input: if opts.PKG and not opts.patch: parser.error("Cannot use --input and provide PKG(s), unless patching.") context = ResolvedContext.load(opts.input) if opts.patch: if context is None: from rez.status import status context = status.context if context is None: print >> sys.stderr, "cannot patch: not in a context" sys.exit(1) # modify the request in terms of the given patch request request = context.get_patched_request(request, strict=opts.strict, rank=opts.patch_rank) context = None if context is None: # create package filters if opts.no_filters: package_filter = PackageFilterList() else: package_filter = PackageFilterList.singleton.copy() for rule_str in (opts.exclude or []): rule = Rule.parse_rule(rule_str) package_filter.add_exclusion(rule) for rule_str in (opts.include or []): rule = Rule.parse_rule(rule_str) package_filter.add_inclusion(rule) # perform the resolve context = ResolvedContext(package_requests=request, timestamp=t, package_paths=pkg_paths, building=opts.build, package_filter=package_filter, add_implicit_packages=(not opts.no_implicit), verbosity=opts.verbose, max_fails=opts.max_fails, time_limit=opts.time_limit, caching=(not opts.no_cache), suppress_passive=opts.no_passive, print_stats=opts.stats) success = (context.status == ResolverStatus.solved) if not success: context.print_info(buf=sys.stderr) if opts.fail_graph: if context.graph: from rez.utils.graph_utils import view_graph g = context.graph(as_dot=True) view_graph(g) else: print >> sys.stderr, \ "the failed resolve context did not generate a graph." if opts.output: if opts.output == '-': # print to stdout context.write_to_buffer(sys.stdout) else: context.save(opts.output) sys.exit(0 if success else 1) if not success: sys.exit(1) # generally shells will behave as though the '-s' flag was not present when # no stdin is available. So here we replicate this behaviour. try: if opts.stdin and not select.select([sys.stdin], [], [], 0.0)[0]: opts.stdin = False except select.error: pass # because windows quiet = opts.quiet or bool(command) returncode, _, _ = context.execute_shell( shell=opts.shell, rcfile=opts.rcfile, norc=opts.norc, command=command, stdin=opts.stdin, quiet=quiet, start_new_session=opts.new_session, detached=opts.detached, pre_command=opts.pre_command, block=True) sys.exit(returncode)
def _set_variant(self, variant, preloaded_packages=None): self.clear() hh = self.horizontalHeader() self.setHorizontalHeaderLabels(["path", "released"]) hh.setResizeMode(0, QtGui.QHeaderView.Interactive) hh.setStretchLastSection(True) hh.setVisible(True) package_paths = self.context_model.packages_path package_filter = PackageFilterList.from_pod(self.context_model.package_filter) if variant and variant.wrapped.location in package_paths: self.version_index = -1 self.reference_version_index = -1 reference_version = None range_ = None if self.reference_variant and self.reference_variant.name == variant.name: reference_version = self.reference_variant.version versions = sorted([reference_version, variant.version]) range_ = VersionRange.as_span(*versions) timestamp = self.context().timestamp if preloaded_packages is not None: if range_ is None: packages = preloaded_packages else: packages = [x for x in preloaded_packages if x.version in range_] else: it = iter_packages(name=variant.name, paths=package_paths, range_=range_) packages = sorted(it, key=lambda x: x.version, reverse=True) self.setRowCount(len(packages)) brush = self.palette().brush(QtGui.QPalette.Active, QtGui.QPalette.Base) for row, package in enumerate(packages): version_str = str(package.version) + ' ' item = QtGui.QTableWidgetItem(version_str) item.setTextAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter) self.setVerticalHeaderItem(row, item) if package.version == variant.version: self.version_index = row update_font(item, bold=True) if reference_version is not None \ and package.version == reference_version: self.reference_version_index = row update_font(item, bold=True, italic=True) def _item(): item_ = QtGui.QTableWidgetItem() item_.setBackground(brush) # get rid of mouse-hover coloring return item_ if package.timestamp: release_str = get_timestamp_str(package.timestamp) in_future = (package.timestamp > timestamp) else: release_str = '-' in_future = False item = _item() txt = package.uri + " " icons = [] if in_future: icon = get_icon_widget( "clock_warning", "package did not exist at time of resolve") icons.append(icon) rule = package_filter.excludes(package) if rule: icon = get_icon_widget( "excluded", "package was excluded by rule %s" % str(rule)) icons.append(icon) if icons: label = QtGui.QLabel(txt) pane = create_pane(icons + [label, None], True, compact=True) self.setCellWidget(row, 0, pane) else: item.setText(txt) self.setItem(row, 0, item) item = _item() item.setText(release_str) self.setItem(row, 1, item) vh = self.verticalHeader() vh.setVisible(True) self.resizeColumnsToContents() hh.setStretchLastSection(True) self.update() self.allow_selection = True self.selectRow(self.version_index) self.allow_selection = False self.variant = variant