class Resolver(object):
"""The package resolver.
The Resolver uses a combination of Solver(s) and cache(s) to resolve a
package request as quickly as possible.
"""
def __init__(self,
context,
package_requests,
package_paths,
package_filter=None,
package_orderers=None,
timestamp=0,
callback=None,
building=False,
verbosity=False,
buf=None,
package_load_callback=None,
caching=True,
suppress_passive=False,
print_stats=False):
"""Create a Resolver.
Args:
package_requests: List of Requirement objects representing the
request.
package_paths: List of paths to search for pkgs.
package_filter (`PackageFilterList`): Package filter.
package_orderers (list of `PackageOrder`): Custom package ordering.
callback: See `Solver`.
package_load_callback: If not None, this callable will be called
prior to each package being loaded. It is passed a single
`Package` object.
building: True if we're resolving for a build.
caching: If True, cache(s) may be used to speed the resolve. If
False, caches will not be used.
print_stats (bool): If true, print advanced solver stats at the end.
"""
self.context = context
self.package_requests = package_requests
self.package_paths = package_paths
self.timestamp = timestamp
self.callback = callback
self.package_orderers = package_orderers
self.package_load_callback = package_load_callback
self.building = building
self.verbosity = verbosity
self.caching = caching
self.buf = buf
self.suppress_passive = suppress_passive
self.print_stats = print_stats
# store hash of package orderers. This is used in the memcached key
if package_orderers:
sha1s = ''.join(x.sha1 for x in package_orderers)
self.package_orderers_hash = sha1(sha1s.encode("utf8")).hexdigest()
else:
self.package_orderers_hash = ''
# store hash of pre-timestamp-combined package filter. This is used in
# the memcached key
if package_filter:
self.package_filter_hash = package_filter.sha1
else:
self.package_filter_hash = ''
# combine timestamp and package filter into single filter
if self.timestamp:
if package_filter:
self.package_filter = package_filter.copy()
else:
self.package_filter = PackageFilterList()
rule = TimestampRule.after(self.timestamp)
self.package_filter.add_exclusion(rule)
else:
self.package_filter = package_filter
self.status_ = ResolverStatus.pending
self.resolved_packages_ = None
self.resolved_ephemerals_ = None
self.failure_description = None
self.graph_ = None
self.from_cache = False
self.memcached_servers = config.memcached_uri if config.resolve_caching else None
self.solve_time = 0.0 # time spent solving
self.load_time = 0.0 # time spent loading package resources
self._print = config.debug_printer("resolve_memcache")
@pool_memcached_connections
def solve(self):
"""Perform the solve.
"""
with log_duration(self._print, "memcache get (resolve) took %s"):
solver_dict = self._get_cached_solve()
if solver_dict:
self.from_cache = True
self._set_result(solver_dict)
else:
self.from_cache = False
solver = self._solve()
solver_dict = self._solver_to_dict(solver)
self._set_result(solver_dict)
with log_duration(self._print, "memcache set (resolve) took %s"):
self._set_cached_solve(solver_dict)
@property
def status(self):
"""Return the current status of the resolve.
Returns:
ResolverStatus.
"""
return self.status_
@property
def resolved_packages(self):
"""Get the list of resolved packages.
Returns:
List of `PackageVariant` objects, or None if the resolve has not
completed.
"""
return self.resolved_packages_
@property
def resolved_ephemerals(self):
"""Get the list of resolved ewphemerals.
Returns:
List of `Requirement` objects, or None if the resolve has not
completed.
"""
return self.resolved_ephemerals_
@property
def graph(self):
"""Return the resolve graph.
The resolve graph shows unsuccessful as well as successful resolves.
Returns:
A pygraph.digraph object, or None if the solve has not completed.
"""
return self.graph_
def _get_variant(self, variant_handle):
return get_variant(variant_handle, context=self.context)
def _get_cached_solve(self):
"""Find a memcached resolve.
If there is NOT a resolve timestamp:
- fetch a non-timestamped memcache entry;
- if no entry, then fail;
- if packages have changed, then:
- delete the entry;
- fail;
- if no packages in the entry have been released since, then
- use the entry and return;
- else:
- delete the entry;
- fail.
If there IS a resolve timestamp (let us call this T):
- fetch a non-timestamped memcache entry;
- if entry then:
- if no packages have changed, then:
- if no packages in the entry have been released since:
- if no packages in the entry were released after T, then
- use the entry and return;
- else:
- delete the entry;
- else:
- delete the entry;
- fetch a timestamped (T) memcache entry;
- if no entry, then fail;
- if packages have changed, then:
- delete the entry;
- fail;
- else:
- use the entry.
This behaviour exists specifically so that resolves that use a
timestamp but set that to the current time, can be reused by other
resolves if nothing has changed. Older resolves however, can only be
reused if the timestamp matches exactly (but this might happen a lot -
consider a workflow where a work area is tied down to a particular
timestamp in order to 'lock' it from any further software releases).
"""
if not (self.caching and self.memcached_servers):
return None
# these caches avoids some potentially repeated file stats
variant_states = {}
last_release_times = {}
def _hit(data):
solver_dict, _, _ = data
return solver_dict
def _miss():
self._print("No cache key retrieved")
return None
def _delete_cache_entry(key):
with self._memcached_client() as client:
client.delete(key)
self._print("Discarded entry: %r", key)
def _retrieve(timestamped):
key = self._memcache_key(timestamped=timestamped)
self._print("Retrieving memcache key: %r", key)
with self._memcached_client() as client:
data = client.get(key)
return key, data
def _packages_changed(key, data):
solver_dict, _, variant_states_dict = data
for variant_handle in solver_dict.get("variant_handles", []):
variant = self._get_variant(variant_handle)
old_state = variant_states_dict.get(variant.name)
new_state = variant_states.get(variant)
if new_state is None:
try:
repo = variant.resource._repository
new_state = repo.get_variant_state_handle(
variant.resource)
except (IOError, OSError) as e:
# if, ie a package file was deleted on disk, then
# an IOError or OSError will be raised when we try to
# read from it - assume that the packages have changed!
self._print(
"Error loading %r (assuming cached state "
"changed): %s", variant.qualified_name, e)
return True
variant_states[variant] = new_state
if old_state != new_state:
self._print("%r has been modified", variant.qualified_name)
return True
return False
def _releases_since_solve(key, data):
_, release_times_dict, _ = data
for package_name, release_time in release_times_dict.items():
time_ = last_release_times.get(package_name)
if time_ is None:
time_ = get_last_release_time(package_name,
self.package_paths)
last_release_times[package_name] = time_
if time_ != release_time:
self._print(
"A newer version of %r (%d) has been released since the "
"resolve was cached (latest release in cache was %d) "
"(entry: %r)", package_name, time_, release_time, key)
return True
return False
def _timestamp_is_earlier(key, data):
_, release_times_dict, _ = data
for package_name, release_time in release_times_dict.items():
if self.timestamp < release_time:
self._print(
"Resolve timestamp (%d) is earlier than %r in "
"solve (%d) (entry: %r)", self.timestamp, package_name,
release_time, key)
return True
return False
key, data = _retrieve(False)
if self.timestamp:
if data:
if _packages_changed(key, data) or _releases_since_solve(
key, data):
_delete_cache_entry(key)
elif not _timestamp_is_earlier(key, data):
return _hit(data)
key, data = _retrieve(True)
if not data:
return _miss()
if _packages_changed(key, data):
_delete_cache_entry(key)
return _miss()
else:
return _hit(data)
else:
if not data:
return _miss()
if _packages_changed(key, data) or _releases_since_solve(
key, data):
_delete_cache_entry(key)
return _miss()
else:
return _hit(data)
@contextmanager
def _memcached_client(self):
with memcached_client(self.memcached_servers,
debug=config.debug_memcache) as client:
yield client
def _set_cached_solve(self, solver_dict):
"""Store a solve to memcached.
If there is NOT a resolve timestamp:
- store the solve to a non-timestamped entry.
If there IS a resolve timestamp (let us call this T):
- if NO newer package in the solve has been released since T,
- then store the solve to a non-timestamped entry;
- else:
- store the solve to a timestamped entry.
"""
if self.status_ != ResolverStatus.solved:
return # don't cache failed solves
if not (self.caching and self.memcached_servers):
return
# most recent release times get stored with solve result in the cache
releases_since_solve = False
release_times_dict = {}
variant_states_dict = {}
for variant in self.resolved_packages_:
time_ = get_last_release_time(variant.name, self.package_paths)
# don't cache if a release time isn't known
if time_ == 0:
self._print(
"Did not send memcache key: a repository could "
"not provide a most recent release time for %r",
variant.name)
return
if self.timestamp and self.timestamp < time_:
releases_since_solve = True
release_times_dict[variant.name] = time_
repo = variant.resource._repository
variant_states_dict[variant.name] = \
repo.get_variant_state_handle(variant.resource)
timestamped = (self.timestamp and releases_since_solve)
key = self._memcache_key(timestamped=timestamped)
data = (solver_dict, release_times_dict, variant_states_dict)
with self._memcached_client() as client:
client.set(key, data)
self._print("Sent memcache key: %r", key)
def _memcache_key(self, timestamped=False):
"""Makes a key suitable as a memcache entry."""
request = tuple(map(str, self.package_requests))
repo_ids = []
for path in self.package_paths:
repo = package_repository_manager.get_repository(path)
repo_ids.append(repo.uid)
t = [
"resolve", request,
tuple(repo_ids), self.package_filter_hash,
self.package_orderers_hash, self.building,
config.prune_failed_graph
]
if timestamped and self.timestamp:
t.append(self.timestamp)
return str(tuple(t))
def _solve(self):
solver = Solver(package_requests=self.package_requests,
package_paths=self.package_paths,
context=self.context,
package_filter=self.package_filter,
package_orderers=self.package_orderers,
callback=self.callback,
package_load_callback=self.package_load_callback,
building=self.building,
verbosity=self.verbosity,
prune_unfailed=config.prune_failed_graph,
buf=self.buf,
suppress_passive=self.suppress_passive,
print_stats=self.print_stats)
solver.solve()
return solver
def _set_result(self, solver_dict):
self.status_ = solver_dict.get("status")
self.graph_ = solver_dict.get("graph")
self.solve_time = solver_dict.get("solve_time")
self.load_time = solver_dict.get("load_time")
self.failure_description = solver_dict.get("failure_description")
self.resolved_packages_ = None
self.resolved_ephemerals_ = None
if self.status_ == ResolverStatus.solved:
# convert solver.Variants to packages.Variants
self.resolved_packages_ = []
for variant_handle in solver_dict.get("variant_handles", []):
variant = self._get_variant(variant_handle)
self.resolved_packages_.append(variant)
self.resolved_ephemerals_ = []
for req_str in solver_dict.get("ephemerals", []):
req = Requirement(req_str)
self.resolved_ephemerals_.append(req)
@classmethod
def _solver_to_dict(cls, solver):
graph_ = solver.get_graph()
solve_time = solver.solve_time
load_time = solver.load_time
failure_description = None
variant_handles = None
ephemerals = None
st = solver.status
if st == SolverStatus.unsolved:
status_ = ResolverStatus.aborted
failure_description = solver.abort_reason
elif st == SolverStatus.failed:
status_ = ResolverStatus.failed
failure_description = solver.failure_description()
elif st == SolverStatus.solved:
status_ = ResolverStatus.solved
variant_handles = []
for solver_variant in solver.resolved_packages:
variant_handle_dict = solver_variant.handle
variant_handles.append(variant_handle_dict)
ephemerals = []
for ephemeral in solver.resolved_ephemerals:
ephemerals.append(str(ephemeral))
return dict(status=status_,
graph=graph_,
solve_time=solve_time,
load_time=load_time,
failure_description=failure_description,
variant_handles=variant_handles,
ephemerals=ephemerals)
def command(opts, parser, extra_arg_groups=None):
from rez.resolved_context import ResolvedContext
from rez.resolver import ResolverStatus
from rez.package_filter import PackageFilterList, Rule
from rez.utils.formatting import get_epoch_time_from_str
from rez.config import config
import select
import sys
import os
import os.path
command = opts.command
if extra_arg_groups:
if opts.command:
parser.error("argument --command: not allowed with arguments after '--'")
command = extra_arg_groups[0] or None
context = None
request = opts.PKG
t = get_epoch_time_from_str(opts.time) if opts.time else None
if opts.isolated:
config.inherit_parent_environment = False
if opts.inherited:
config.inherit_parent_environment = True
if opts.paths is None:
pkg_paths = (config.nonlocal_packages_path
if opts.no_local else None)
else:
pkg_paths = opts.paths.split(os.pathsep)
pkg_paths = [os.path.expanduser(x) for x in pkg_paths if x]
if opts.input:
if opts.PKG and not opts.patch:
parser.error("Cannot use --input and provide PKG(s), unless patching.")
context = ResolvedContext.load(opts.input)
if opts.patch:
if context is None:
from rez.status import status
context = status.context
if context is None:
print("cannot patch: not in a context", file=sys.stderr)
sys.exit(1)
# modify the request in terms of the given patch request
request = context.get_patched_request(request,
strict=opts.strict,
rank=opts.patch_rank)
context = None
if opts.self:
from rez.utils._version import _rez_version
request += ["bleeding_rez==%s" % _rez_version]
request += ["python"] # Required by Rez
if context is None:
# create package filters
if opts.no_filters:
package_filter = PackageFilterList()
else:
package_filter = PackageFilterList.singleton.copy()
for rule_str in (opts.exclude or []):
rule = Rule.parse_rule(rule_str)
package_filter.add_exclusion(rule)
for rule_str in (opts.include or []):
rule = Rule.parse_rule(rule_str)
package_filter.add_inclusion(rule)
# perform the resolve
context = ResolvedContext(package_requests=request,
timestamp=t,
package_paths=pkg_paths,
building=opts.build,
package_filter=package_filter,
add_implicit_packages=(not opts.no_implicit),
verbosity=opts.verbose,
max_fails=opts.max_fails,
time_limit=opts.time_limit,
caching=(not opts.no_cache),
suppress_passive=opts.no_passive,
print_stats=opts.stats)
success = (context.status == ResolverStatus.solved)
if not success:
context.print_info(buf=sys.stderr)
if opts.fail_graph:
if context.graph:
from rez.utils.graph_utils import view_graph
g = context.graph(as_dot=True)
view_graph(g)
else:
print("the failed resolve context did not generate a graph.", file=sys.stderr)
if opts.output:
if opts.output == '-': # print to stdout
context.write_to_buffer(sys.stdout)
else:
context.save(opts.output)
sys.exit(0 if success else 1)
if not success:
sys.exit(1)
if opts.env:
env = {}
for pair in opts.env:
key, value = pair.split("=")
env[key.upper()] = value
config.additional_environment = env
# generally shells will behave as though the '-s' flag was not present when
# no stdin is available. So here we replicate this behaviour.
try:
if opts.stdin and not select.select([sys.stdin], [], [], 0.0)[0]:
opts.stdin = False
except select.error:
pass # because windows
quiet = opts.quiet or bool(command)
returncode, _, _ = context.execute_shell(
shell=opts.shell,
rcfile=opts.rcfile,
norc=opts.norc,
command=command,
stdin=opts.stdin,
quiet=quiet,
start_new_session=opts.new_session,
detached=opts.detached,
pre_command=opts.pre_command,
block=True)
sys.exit(returncode)
def command(opts, parser, extra_arg_groups=None):
from rez.resolved_context import ResolvedContext
from rez.resolver import ResolverStatus
from rez.package_filter import PackageFilterList, Rule
from rez.utils.formatting import get_epoch_time_from_str
from rez.config import config
import select
import sys
import os
import os.path
command = opts.command
if extra_arg_groups:
if opts.command:
parser.error(
"argument --command: not allowed with arguments after '--'")
command = extra_arg_groups[0] or None
context = None
request = opts.PKG
t = get_epoch_time_from_str(opts.time) if opts.time else None
if opts.paths is None:
pkg_paths = (config.nonlocal_packages_path if opts.no_local else None)
else:
pkg_paths = opts.paths.split(os.pathsep)
pkg_paths = [os.path.expanduser(x) for x in pkg_paths if x]
if opts.input:
if opts.PKG:
parser.error(
"Cannot use --input and provide PKG(s) at the same time")
context = ResolvedContext.load(opts.input)
if context.status != ResolverStatus.solved:
print >> sys.stderr, "cannot rez-env into a failed context"
sys.exit(1)
if opts.patch:
# TODO: patching is lagging behind in options, and needs to be updated
# anyway.
if context is None:
from rez.status import status
context = status.context
if context is None:
print >> sys.stderr, "cannot patch: not in a context"
sys.exit(1)
request = context.get_patched_request(request,
strict=opts.strict,
rank=opts.patch_rank)
context = None
if context is None:
# create package filters
if opts.no_filters:
package_filter = PackageFilterList()
else:
package_filter = PackageFilterList.singleton.copy()
for rule_str in (opts.exclude or []):
rule = Rule.parse_rule(rule_str)
package_filter.add_exclusion(rule)
for rule_str in (opts.include or []):
rule = Rule.parse_rule(rule_str)
package_filter.add_inclusion(rule)
# perform the resolve
context = ResolvedContext(package_requests=request,
timestamp=t,
package_paths=pkg_paths,
building=opts.build,
package_filter=package_filter,
add_implicit_packages=(not opts.no_implicit),
verbosity=opts.verbose,
max_fails=opts.max_fails,
time_limit=opts.time_limit,
caching=(not opts.no_cache))
success = (context.status == ResolverStatus.solved)
if not success:
context.print_info(buf=sys.stderr)
if opts.output:
if opts.output == '-': # print to stdout
context.write_to_buffer(sys.stdout)
else:
context.save(opts.output)
sys.exit(0 if success else 1)
if not success:
sys.exit(1)
# generally shells will behave as though the '-s' flag was not present when
# no stdin is available. So here we replicate this behaviour.
if opts.stdin and not select.select([sys.stdin], [], [], 0.0)[0]:
opts.stdin = False
quiet = opts.quiet or bool(command)
returncode, _, _ = context.execute_shell(
shell=opts.shell,
rcfile=opts.rcfile,
norc=opts.norc,
command=command,
stdin=opts.stdin,
quiet=quiet,
start_new_session=opts.new_session,
detached=opts.detached,
pre_command=opts.pre_command,
block=True)
sys.exit(returncode)
class Resolver(object):
"""The package resolver.
The Resolver uses a combination of Solver(s) and cache(s) to resolve a
package request as quickly as possible.
"""
def __init__(self, package_requests, package_paths, package_filter=None,
timestamp=0, callback=None, building=False, verbosity=False,
buf=None, package_load_callback=None, caching=True):
"""Create a Resolver.
Args:
package_requests: List of Requirement objects representing the
request.
package_paths: List of paths to search for pkgs.
package_filter (`PackageFilterList`): Package filter.
callback: See `Solver`.
package_load_callback: If not None, this callable will be called
prior to each package being loaded. It is passed a single
`Package` object.
building: True if we're resolving for a build.
caching: If True, cache(s) may be used to speed the resolve. If
False, caches will not be used.
"""
self.package_requests = package_requests
self.package_paths = package_paths
self.timestamp = timestamp
self.callback = callback
self.package_load_callback = package_load_callback
self.building = building
self.verbosity = verbosity
self.caching = caching
self.buf = buf
# store hash of pre-timestamp-combined package filter. This is used in
# the memcached key
if package_filter:
self.package_filter_hash = package_filter.hash
else:
self.package_filter_hash = ''
# combine timestamp and package filter into single filter
if self.timestamp:
if package_filter:
self.package_filter = package_filter.copy()
else:
self.package_filter = PackageFilterList()
rule = TimestampRule.after(self.timestamp)
self.package_filter.add_exclusion(rule)
else:
self.package_filter = package_filter
self.status_ = ResolverStatus.pending
self.resolved_packages_ = None
self.failure_description = None
self.graph_ = None
self.from_cache = False
self.memcached_servers = config.memcached_uri if config.resolve_caching else None
self.solve_time = 0.0 # time spent solving
self.load_time = 0.0 # time spent loading package resources
self._print = config.debug_printer("resolve_memcache")
@pool_memcached_connections
def solve(self):
"""Perform the solve.
"""
solver_dict = self._get_cached_solve()
if solver_dict:
self.from_cache = True
self._set_result(solver_dict)
else:
self.from_cache = False
solver = self._solve()
solver_dict = self._solver_to_dict(solver)
self._set_result(solver_dict)
self._set_cached_solve(solver_dict)
@property
def status(self):
"""Return the current status of the resolve.
Returns:
ResolverStatus.
"""
return self.status_
@property
def resolved_packages(self):
"""Get the list of resolved packages.
Returns:
List of `PackageVariant` objects, or None if the resolve has not
completed.
"""
return self.resolved_packages_
@property
def graph(self):
"""Return the resolve graph.
The resolve graph shows unsuccessful as well as successful resolves.
Returns:
A pygraph.digraph object, or None if the solve has not completed.
"""
return self.graph_
def _get_cached_solve(self):
"""Find a memcached resolve.
If there is NOT a resolve timestamp:
- fetch a non-timestamped memcache entry;
- if no entry, then fail;
- if packages have changed, then:
- delete the entry;
- fail;
- if no packages in the entry have been released since, then
- use the entry and return;
- else:
- delete the entry;
- fail.
If there IS a resolve timestamp (let us call this T):
- fetch a non-timestamped memcache entry;
- if entry then:
- if no packages have changed, then:
- if no packages in the entry have been released since:
- if no packages in the entry were released after T, then
- use the entry and return;
- else:
- delete the entry;
- else:
- delete the entry;
- fetch a timestamped (T) memcache entry;
- if no entry, then fail;
- if packages have changed, then:
- delete the entry;
- fail;
- else:
- use the entry.
This behaviour exists specifically so that resolves that use use a
timestamp but set that to the current time, can be reused by other
resolves if nothing has changed. Older resolves however, can only be
reused if the timestamp matches exactly (but this might happen a lot -
consider a workflow where a work area is tied down to a particular
timestamp in order to 'lock' it from any further software releases).
"""
if not (self.caching and self.memcached_servers):
return None
def _hit(data):
solver_dict, _, _ = data
return solver_dict
def _miss():
self._print("No cache key retrieved")
return None
def _delete_cache_entry(key):
with self._memcached_client() as client:
client.delete(key)
self._print("Discarded entry: %r", key)
def _retrieve(timestamped):
key = self._memcache_key(timestamped=timestamped)
self._print("Retrieving memcache key: %r", key)
with self._memcached_client() as client:
data = client.get(key)
return key, data
def _packages_changed(key, data):
solver_dict, _, variant_states_dict = data
for variant_handle in solver_dict.get("variant_handles", []):
variant = get_variant(variant_handle)
old_state = variant_states_dict.get(variant.name)
repo = variant.resource._repository
new_state = repo.get_variant_state_handle(variant.resource)
if old_state != new_state:
self._print("%r has been modified", variant.qualified_name)
return True
return False
def _releases_since_solve(key, data):
_, release_times_dict, _ = data
for package_name, release_time in release_times_dict.iteritems():
time_ = get_last_release_time(package_name, self.package_paths)
if time_ != release_time:
self._print(
"A newer version of %r (%d) has been released since the "
"resolve was cached (latest release in cache was %d) "
"(entry: %r)", package_name, time_, release_time, key)
return True
return False
def _timestamp_is_earlier(key, data):
_, release_times_dict, _ = data
for package_name, release_time in release_times_dict.iteritems():
if self.timestamp < release_time:
self._print("Resolve timestamp (%d) is earlier than %r in "
"solve (%d) (entry: %r)", self.timestamp,
package_name, release_time, key)
return True
return False
key, data = _retrieve(False)
if self.timestamp:
if data:
if _packages_changed(key, data):
_delete_cache_entry(key)
elif _releases_since_solve(key, data):
_delete_cache_entry(key)
elif not _timestamp_is_earlier(key, data):
return _hit(data)
key, data = _retrieve(True)
if not data:
return _miss()
if _packages_changed(key, data):
_delete_cache_entry(key)
return _miss()
else:
return _hit(data)
else:
if not data:
return _miss()
if _packages_changed(key, data):
_delete_cache_entry(key)
return _miss()
if _releases_since_solve(key, data):
_delete_cache_entry(key)
return _miss()
else:
return _hit(data)
@contextmanager
def _memcached_client(self):
with memcached_client(self.memcached_servers,
debug=config.debug_memcache) as client:
yield client
def _set_cached_solve(self, solver_dict):
"""Store a solve to memcached.
If there is NOT a resolve timestamp:
- store the solve to a non-timestamped entry.
If there IS a resolve timestamp (let us call this T):
- if NO newer package in the solve has been released since T,
- then store the solve to a non-timestamped entry;
- else:
- store the solve to a timestamped entry.
"""
if self.status_ != ResolverStatus.solved:
return # don't cache failed solves
if not (self.caching and self.memcached_servers):
return
# most recent release times get stored with solve result in the cache
releases_since_solve = False
release_times_dict = {}
variant_states_dict = {}
for variant in self.resolved_packages_:
time_ = get_last_release_time(variant.name, self.package_paths)
# don't cache if a release time isn't known
if time_ == 0:
self._print("Did not send memcache key: a repository could "
"not provide a most recent release time for %r",
variant.name)
return
if self.timestamp and self.timestamp < time_:
releases_since_solve = True
release_times_dict[variant.name] = time_
repo = variant.resource._repository
variant_states_dict[variant.name] = \
repo.get_variant_state_handle(variant.resource)
timestamped = (self.timestamp and releases_since_solve)
key = self._memcache_key(timestamped=timestamped)
data = (solver_dict, release_times_dict, variant_states_dict)
with self._memcached_client() as client:
client.set(key, data)
self._print("Sent memcache key: %r", key)
def _memcache_key(self, timestamped=False):
"""Makes a key suitable as a memcache entry."""
request = tuple(map(str, self.package_requests))
repo_ids = []
for path in self.package_paths:
repo = package_repository_manager.get_repository(path)
repo_ids.append(repo.uid)
t = ["resolve",
request,
tuple(repo_ids),
self.package_filter_hash,
self.building,
config.prune_failed_graph]
if timestamped and self.timestamp:
t.append(self.timestamp)
return str(tuple(t))
def _solve(self):
solver = Solver(package_requests=self.package_requests,
package_paths=self.package_paths,
package_filter=self.package_filter,
callback=self.callback,
package_load_callback=self.package_load_callback,
building=self.building,
verbosity=self.verbosity,
prune_unfailed=config.prune_failed_graph,
buf=self.buf)
solver.solve()
return solver
def _set_result(self, solver_dict):
self.status_ = solver_dict.get("status")
self.graph_ = solver_dict.get("graph")
self.solve_time = solver_dict.get("solve_time")
self.load_time = solver_dict.get("load_time")
self.failure_description = solver_dict.get("failure_description")
self.resolved_packages_ = None
if self.status_ == ResolverStatus.solved:
# convert solver.Variants to packages.Variants
self.resolved_packages_ = []
for variant_handle in solver_dict.get("variant_handles", []):
variant = get_variant(variant_handle)
self.resolved_packages_.append(variant)
@classmethod
def _solver_to_dict(cls, solver):
graph_ = solver.get_graph()
solve_time = solver.solve_time
load_time = solver.load_time
failure_description = None
variant_handles = None
st = solver.status
if st == SolverStatus.unsolved:
status_ = ResolverStatus.aborted
failure_description = solver.abort_reason
elif st == SolverStatus.failed:
status_ = ResolverStatus.failed
failure_description = solver.failure_description()
elif st == SolverStatus.solved:
status_ = ResolverStatus.solved
variant_handles = []
for solver_variant in solver.resolved_packages:
variant_handle_dict = solver_variant.userdata
variant_handles.append(variant_handle_dict)
return dict(
status=status_,
graph=graph_,
solve_time=solve_time,
load_time=load_time,
failure_description=failure_description,
variant_handles=variant_handles)
def command(opts, parser, extra_arg_groups=None):
from rez.resolved_context import ResolvedContext
from rez.resolver import ResolverStatus
from rez.package_filter import PackageFilterList, Rule
from rez.utils.formatting import get_epoch_time_from_str
from rez.config import config
import select
import sys
import os
import os.path
command = opts.command
if extra_arg_groups:
if opts.command:
parser.error("argument --command: not allowed with arguments after '--'")
command = extra_arg_groups[0] or None
context = None
request = opts.PKG
t = get_epoch_time_from_str(opts.time) if opts.time else None
if opts.paths is None:
pkg_paths = (config.nonlocal_packages_path
if opts.no_local else None)
else:
pkg_paths = opts.paths.split(os.pathsep)
pkg_paths = [os.path.expanduser(x) for x in pkg_paths if x]
if opts.input:
if opts.PKG and not opts.patch:
parser.error("Cannot use --input and provide PKG(s), unless patching.")
context = ResolvedContext.load(opts.input)
if opts.patch:
if context is None:
from rez.status import status
context = status.context
if context is None:
print >> sys.stderr, "cannot patch: not in a context"
sys.exit(1)
# modify the request in terms of the given patch request
request = context.get_patched_request(request,
strict=opts.strict,
rank=opts.patch_rank)
context = None
if context is None:
# create package filters
if opts.no_filters:
package_filter = PackageFilterList()
else:
package_filter = PackageFilterList.singleton.copy()
for rule_str in (opts.exclude or []):
rule = Rule.parse_rule(rule_str)
package_filter.add_exclusion(rule)
for rule_str in (opts.include or []):
rule = Rule.parse_rule(rule_str)
package_filter.add_inclusion(rule)
# perform the resolve
context = ResolvedContext(package_requests=request,
timestamp=t,
package_paths=pkg_paths,
building=opts.build,
package_filter=package_filter,
add_implicit_packages=(not opts.no_implicit),
verbosity=opts.verbose,
max_fails=opts.max_fails,
time_limit=opts.time_limit,
caching=(not opts.no_cache),
suppress_passive=opts.no_passive,
print_stats=opts.stats)
success = (context.status == ResolverStatus.solved)
if not success:
context.print_info(buf=sys.stderr)
if opts.fail_graph:
if context.graph:
from rez.utils.graph_utils import view_graph
g = context.graph(as_dot=True)
view_graph(g)
else:
print >> sys.stderr, \
"the failed resolve context did not generate a graph."
if opts.output:
if opts.output == '-': # print to stdout
context.write_to_buffer(sys.stdout)
else:
context.save(opts.output)
sys.exit(0 if success else 1)
if not success:
sys.exit(1)
# generally shells will behave as though the '-s' flag was not present when
# no stdin is available. So here we replicate this behaviour.
try:
if opts.stdin and not select.select([sys.stdin], [], [], 0.0)[0]:
opts.stdin = False
except select.error:
pass # because windows
quiet = opts.quiet or bool(command)
returncode, _, _ = context.execute_shell(
shell=opts.shell,
rcfile=opts.rcfile,
norc=opts.norc,
command=command,
stdin=opts.stdin,
quiet=quiet,
start_new_session=opts.new_session,
detached=opts.detached,
pre_command=opts.pre_command,
block=True)
sys.exit(returncode)
