def _run_daemon_step(self, state): logger = state["logger"] # pick a random pending variant to copy pending_filenames = set(os.listdir(self._pending_dir)) pending_filenames -= set(state.get("copying", set())) if not pending_filenames: return False i = random.randint(0, len(pending_filenames) - 1) filename = list(pending_filenames)[i] filepath = os.path.join(self._pending_dir, filename) try: with open(filepath) as f: variant_handle_dict = json.loads(f.read()) except IOError as e: if e.errno == errno.ENOENT: return True # was probably deleted by another rez-pkg-cache proc raise variant = get_variant(variant_handle_dict) # copy the variant and log activity logger.info("Started caching of variant %s...", variant.uri) t = time.time() try: rootpath, status = self.add_variant(variant) except PackageCacheError as e: # variant cannot be cached, so remove as a pending variant logger.warning(str(e)) safe_remove(filepath) return True except Exception: # This is probably an error during shutil.copytree (eg a perms fail). # In this case, the variant will be in VARIANT_COPYING status, and # will shortly transition to VARIANT_COPY_STALLED. Thus we can # remove the pending variant, as there's nothing more we can do. # logger.exception("Failed to add variant to the cache") safe_remove(filepath) return True secs = time.time() - t if status == self.VARIANT_FOUND: logger.info("Variant was already cached at %s", rootpath) elif status == self.VARIANT_COPYING: logger.info("Variant is already being copied to %s", rootpath) elif status == self.VARIANT_COPY_STALLED: logger.info("Variant is stalled copying to %s", rootpath) else: # VARIANT_CREATED logger.info("Cached variant to %s in %g seconds", rootpath, secs) if status == self.VARIANT_COPYING: # we cannot delete the pending file (another proc is copying the # variant, so it's responsible); but we also have to ignore this # variant from now on. # state.setdefault("copying", set()).add(filename) else: safe_remove(filepath) return True
def _get_variant(self, variant_handle): return get_variant(variant_handle, context=self.context)
def get_variants(self): """Get variants and their current statuses from the cache. Returns: List of 3-tuple: - `Variant`: The cached variant - str: Local cache path for variant, if determined ('' otherwise) - int: Status. One of: - VARIANT_FOUND - VARIANT_COPYING - VARIANT_COPY_STALLED - VARIANT_PENDING """ from rez.packages import get_variant statuses = (self.VARIANT_FOUND, self.VARIANT_COPYING, self.VARIANT_COPY_STALLED) results = [] seen_variants = set() # find variants in cache for pkg_name in safe_listdir(self.path): if pkg_name.startswith('.'): continue # dirs for internal cache use path1 = os.path.join(self.path, pkg_name) for ver_str in safe_listdir(path1): path2 = os.path.join(path1, ver_str) for hash_str in safe_listdir(path2): path3 = os.path.join(path2, hash_str) for name in safe_listdir(path3): if name.endswith(".json"): with open(os.path.join(path3, name)) as f: data = json.loads(f.read()) handle = data["handle"] variant = get_variant(handle) status, rootpath = self._get_cached_root(variant) if status in statuses: results.append((variant, rootpath, status)) seen_variants.add(variant) # find pending variants pending_filenames = os.listdir(self._pending_dir) for name in pending_filenames: filepath = os.path.join(self._pending_dir, name) try: with open(filepath) as f: variant_handle_dict = json.loads(f.read()) except: continue # maybe file was just deleted variant = get_variant(variant_handle_dict) if variant not in seen_variants: results.append((variant, '', self.VARIANT_PENDING)) seen_variants.add(variant) return results