def _get_cached_root(self, variant):
path = self._get_hash_path(variant)
if not os.path.exists(path):
return (self.VARIANT_NOT_FOUND, '')
handle_dict = variant.handle.to_dict()
for name in os.listdir(path):
if name.endswith(".json"):
incname = os.path.splitext(name)[0]
json_filepath = os.path.join(path, name)
rootpath = os.path.join(path, incname)
copying_filepath = os.path.join(path, ".copying-" + incname)
try:
with open(json_filepath) as f:
data = json.loads(f.read())
except IOError as e:
if e.errno == errno.ENOENT:
# maybe got cleaned up by other process
continue
else:
raise
if data.get("handle") == handle_dict:
if os.path.exists(copying_filepath):
try:
st = os.stat(copying_filepath)
secs = time.time() - st.st_mtime
if secs > self._COPYING_TIME_MAX:
return (self.VARIANT_COPY_STALLED, rootpath)
except:
# maybe .copying file was deleted just now
pass
return (self.VARIANT_COPYING, rootpath)
else:
return (self.VARIANT_FOUND, rootpath)
return (self.VARIANT_NOT_FOUND, '')
def _validate(self, data):
# overridden settings take precedence. Note that `data` has already
# taken override into account at this point
if self.key in self.config.overrides:
return data
if not self.config.locked:
# next, env-var
value = os.getenv(self._env_var_name)
if value is not None:
return self._parse_env_var(value)
# next, JSON-encoded env-var
varname = self._env_var_name + "_JSON"
value = os.getenv(varname)
if value is not None:
from rez.utils import json
try:
return json.loads(value)
except ValueError:
raise ConfigurationError(
"Expected $%s to be JSON-encoded string." % varname
)
# next, data unchanged
if data is not None:
return data
# some settings have a programmatic default
attr = "_get_%s" % self.key
if hasattr(self.config, attr):
return getattr(self.config, attr)()
# setting is None
return None
def _validate(self, data):
# overridden settings take precedence. Note that `data` has already
# taken override into account at this point
if self.key in self.config.overrides:
return data
if not self.config.locked:
# next, env-var
value = os.getenv(self._env_var_name)
if value is not None:
return self._parse_env_var(value)
# next, JSON-encoded env-var
varname = self._env_var_name + "_JSON"
value = os.getenv(varname)
if value is not None:
from rez.utils import json
try:
return json.loads(value)
except ValueError:
raise ConfigurationError(
"Expected $%s to be JSON-encoded string." % varname
)
# next, data unchanged
if data is not None:
return data
# some settings have a programmatic default
attr = "_get_%s" % self.key
if hasattr(self.config, attr):
return getattr(self.config, attr)()
# setting is None
return None
def _run_daemon_step(self, state):
logger = state["logger"]
# pick a random pending variant to copy
pending_filenames = set(os.listdir(self._pending_dir))
pending_filenames -= set(state.get("copying", set()))
if not pending_filenames:
return False
i = random.randint(0, len(pending_filenames) - 1)
filename = list(pending_filenames)[i]
filepath = os.path.join(self._pending_dir, filename)
try:
with open(filepath) as f:
variant_handle_dict = json.loads(f.read())
except IOError as e:
if e.errno == errno.ENOENT:
return True # was probably deleted by another rez-pkg-cache proc
raise
variant = get_variant(variant_handle_dict)
# copy the variant and log activity
logger.info("Started caching of variant %s...", variant.uri)
t = time.time()
try:
rootpath, status = self.add_variant(variant)
except PackageCacheError as e:
# variant cannot be cached, so remove as a pending variant
logger.warning(str(e))
safe_remove(filepath)
return True
except Exception:
# This is probably an error during shutil.copytree (eg a perms fail).
# In this case, the variant will be in VARIANT_COPYING status, and
# will shortly transition to VARIANT_COPY_STALLED. Thus we can
# remove the pending variant, as there's nothing more we can do.
#
logger.exception("Failed to add variant to the cache")
safe_remove(filepath)
return True
secs = time.time() - t
if status == self.VARIANT_FOUND:
logger.info("Variant was already cached at %s", rootpath)
elif status == self.VARIANT_COPYING:
logger.info("Variant is already being copied to %s", rootpath)
elif status == self.VARIANT_COPY_STALLED:
logger.info("Variant is stalled copying to %s", rootpath)
else: # VARIANT_CREATED
logger.info("Cached variant to %s in %g seconds", rootpath, secs)
if status == self.VARIANT_COPYING:
# we cannot delete the pending file (another proc is copying the
# variant, so it's responsible); but we also have to ignore this
# variant from now on.
#
state.setdefault("copying", set()).add(filename)
else:
safe_remove(filepath)
return True
def get_variants(self):
"""Get variants and their current statuses from the cache.
Returns:
List of 3-tuple:
- `Variant`: The cached variant
- str: Local cache path for variant, if determined ('' otherwise)
- int: Status. One of:
- VARIANT_FOUND
- VARIANT_COPYING
- VARIANT_COPY_STALLED
- VARIANT_PENDING
"""
from rez.packages import get_variant
statuses = (self.VARIANT_FOUND, self.VARIANT_COPYING,
self.VARIANT_COPY_STALLED)
results = []
seen_variants = set()
# find variants in cache
for pkg_name in safe_listdir(self.path):
if pkg_name.startswith('.'):
continue # dirs for internal cache use
path1 = os.path.join(self.path, pkg_name)
for ver_str in safe_listdir(path1):
path2 = os.path.join(path1, ver_str)
for hash_str in safe_listdir(path2):
path3 = os.path.join(path2, hash_str)
for name in safe_listdir(path3):
if name.endswith(".json"):
with open(os.path.join(path3, name)) as f:
data = json.loads(f.read())
handle = data["handle"]
variant = get_variant(handle)
status, rootpath = self._get_cached_root(variant)
if status in statuses:
results.append((variant, rootpath, status))
seen_variants.add(variant)
# find pending variants
pending_filenames = os.listdir(self._pending_dir)
for name in pending_filenames:
filepath = os.path.join(self._pending_dir, name)
try:
with open(filepath) as f:
variant_handle_dict = json.loads(f.read())
except:
continue # maybe file was just deleted
variant = get_variant(variant_handle_dict)
if variant not in seen_variants:
results.append((variant, '', self.VARIANT_PENDING))
seen_variants.add(variant)
return results
def add_variants_async(self, variants):
"""Update the package cache by adding some or all of the given variants.
This method is called when a context is created or sourced. Variants
are then added to the cache in a separate process.
"""
# A prod install is necessary because add_variants_async works by
# starting a rez-pkg-cache proc, and this can only be done reliably in
# a prod install. On non-windows we could fork instead, but there would
# remain no good solution on windows.
#
if not system.is_production_rez_install:
raise PackageCacheError(
"PackageCache.add_variants_async is only supported in a "
"production rez installation.")
variants_ = []
# trim down to those variants that are cachable, and not already cached
for variant in variants:
if not variant.parent.is_cachable:
continue
status, _ = self._get_cached_root(variant)
if status == self.VARIANT_NOT_FOUND:
variants_.append(variant)
# if there are no variants to add, and no potential cleanup to do, then exit
if not variants_ and config.package_cache_clean_limit < 0:
return
# Write each variant out to a file in the 'pending' dir in the cache. A
# separate proc reads these files and then performs the actual variant
# copy. Note that these files are unique, in case two rez procs attempt
# to write out the same pending variant file at the same time.
#
pending_filenames = os.listdir(self._pending_dir)
for variant in variants_:
prefix = variant.parent.qualified_name + '-'
handle_dict = variant.handle.to_dict()
already_pending = False
# check if this variant is already pending
for filename in pending_filenames:
if filename.startswith(prefix):
filepath = os.path.join(self._pending_dir, filename)
try:
with open(filepath) as f:
data = json.loads(f.read())
except:
continue # maybe file was just deleted
if data == handle_dict:
already_pending = True
break
if already_pending:
continue
filename = prefix + uuid4().hex + ".json"
filepath = os.path.join(self._pending_dir, filename)
with open(filepath, 'w') as f:
f.write(json.dumps(handle_dict))
# configure executable
if platform.system() == "Windows":
kwargs = {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
else:
kwargs = {"preexec_fn": os.setsid}
exe = os.path.join(system.rez_bin_path, "rez-pkg-cache")
if not exe:
# this should not happen
raise RuntimeError("Did not find rez-pkg-cache executable")
# start caching subproc
args = [exe, "--daemon", self.path]
try:
with open(os.devnull, 'w') as devnull:
# don't suppress output if selftest running, easier to debug
if system.selftest_is_running:
out_target = None
else:
out_target = devnull
_ = subprocess.Popen([exe, "--daemon", self.path],
stdout=out_target,
stderr=out_target,
**kwargs)
except Exception as e:
print_warning(
"Failed to start package caching daemon (command: %s): %s",
' '.join(args), e)
