def _run_daemon_step(self, state):
logger = state["logger"]
# pick a random pending variant to copy
pending_filenames = set(os.listdir(self._pending_dir))
pending_filenames -= set(state.get("copying", set()))
if not pending_filenames:
return False
i = random.randint(0, len(pending_filenames) - 1)
filename = list(pending_filenames)[i]
filepath = os.path.join(self._pending_dir, filename)
try:
with open(filepath) as f:
variant_handle_dict = json.loads(f.read())
except IOError as e:
if e.errno == errno.ENOENT:
return True # was probably deleted by another rez-pkg-cache proc
raise
variant = get_variant(variant_handle_dict)
# copy the variant and log activity
logger.info("Started caching of variant %s...", variant.uri)
t = time.time()
try:
rootpath, status = self.add_variant(variant)
except PackageCacheError as e:
# variant cannot be cached, so remove as a pending variant
logger.warning(str(e))
safe_remove(filepath)
return True
except Exception:
# This is probably an error during shutil.copytree (eg a perms fail).
# In this case, the variant will be in VARIANT_COPYING status, and
# will shortly transition to VARIANT_COPY_STALLED. Thus we can
# remove the pending variant, as there's nothing more we can do.
#
logger.exception("Failed to add variant to the cache")
safe_remove(filepath)
return True
secs = time.time() - t
if status == self.VARIANT_FOUND:
logger.info("Variant was already cached at %s", rootpath)
elif status == self.VARIANT_COPYING:
logger.info("Variant is already being copied to %s", rootpath)
elif status == self.VARIANT_COPY_STALLED:
logger.info("Variant is stalled copying to %s", rootpath)
else: # VARIANT_CREATED
logger.info("Cached variant to %s in %g seconds", rootpath, secs)
if status == self.VARIANT_COPYING:
# we cannot delete the pending file (another proc is copying the
# variant, so it's responsible); but we also have to ignore this
# variant from now on.
#
state.setdefault("copying", set()).add(filename)
else:
safe_remove(filepath)
return True
def _get_variant(self, variant_handle):
return get_variant(variant_handle, context=self.context)
def get_variants(self):
"""Get variants and their current statuses from the cache.
Returns:
List of 3-tuple:
- `Variant`: The cached variant
- str: Local cache path for variant, if determined ('' otherwise)
- int: Status. One of:
- VARIANT_FOUND
- VARIANT_COPYING
- VARIANT_COPY_STALLED
- VARIANT_PENDING
"""
from rez.packages import get_variant
statuses = (self.VARIANT_FOUND, self.VARIANT_COPYING,
self.VARIANT_COPY_STALLED)
results = []
seen_variants = set()
# find variants in cache
for pkg_name in safe_listdir(self.path):
if pkg_name.startswith('.'):
continue # dirs for internal cache use
path1 = os.path.join(self.path, pkg_name)
for ver_str in safe_listdir(path1):
path2 = os.path.join(path1, ver_str)
for hash_str in safe_listdir(path2):
path3 = os.path.join(path2, hash_str)
for name in safe_listdir(path3):
if name.endswith(".json"):
with open(os.path.join(path3, name)) as f:
data = json.loads(f.read())
handle = data["handle"]
variant = get_variant(handle)
status, rootpath = self._get_cached_root(variant)
if status in statuses:
results.append((variant, rootpath, status))
seen_variants.add(variant)
# find pending variants
pending_filenames = os.listdir(self._pending_dir)
for name in pending_filenames:
filepath = os.path.join(self._pending_dir, name)
try:
with open(filepath) as f:
variant_handle_dict = json.loads(f.read())
except:
continue # maybe file was just deleted
variant = get_variant(variant_handle_dict)
if variant not in seen_variants:
results.append((variant, '', self.VARIANT_PENDING))
seen_variants.add(variant)
return results
