def _pull_valid_cache(self, cpv, ebuild_path, repo_path): try: ebuild_hash = eclass_cache.hashed_path(ebuild_path) # snag mtime since we use it later, and to trigger stat failure # if it doesn't exist ebuild_hash.mtime except FileNotFound: writemsg(_("!!! aux_get(): ebuild for " \ "'%s' does not exist at:\n") % (cpv,), noiselevel=-1) writemsg("!!! %s\n" % ebuild_path, noiselevel=-1) raise KeyError(cpv) # Pull pre-generated metadata from the metadata/cache/ # directory if it exists and is valid, otherwise fall # back to the normal writable cache. auxdbs = [] pregen_auxdb = self._pregen_auxdb.get(repo_path) if pregen_auxdb is not None: auxdbs.append(pregen_auxdb) ro_auxdb = self._ro_auxdb.get(repo_path) if ro_auxdb is not None: auxdbs.append(ro_auxdb) auxdbs.append(self.auxdb[repo_path]) eclass_db = self.repositories.get_repo_for_location( repo_path).eclass_db for auxdb in auxdbs: try: metadata = auxdb[cpv] except KeyError: continue except CacheError: if not auxdb.readonly: try: del auxdb[cpv] except (KeyError, CacheError): pass continue eapi = metadata.get('EAPI', '').strip() if not eapi: eapi = '0' metadata['EAPI'] = eapi if not eapi_is_supported(eapi): # Since we're supposed to be able to efficiently obtain the # EAPI from _parse_eapi_ebuild_head, we disregard cache entries # for unsupported EAPIs. continue if auxdb.validate_entry(metadata, ebuild_hash, eclass_db): break else: metadata = None return (metadata, ebuild_hash)
def _pull_valid_cache(self, cpv, ebuild_path, repo_path): try: ebuild_hash = eclass_cache.hashed_path(ebuild_path) # snag mtime since we use it later, and to trigger stat failure # if it doesn't exist ebuild_hash.mtime except FileNotFound: writemsg(_("!!! aux_get(): ebuild for " \ "'%s' does not exist at:\n") % (cpv,), noiselevel=-1) writemsg("!!! %s\n" % ebuild_path, noiselevel=-1) raise KeyError(cpv) # Pull pre-generated metadata from the metadata/cache/ # directory if it exists and is valid, otherwise fall # back to the normal writable cache. auxdbs = [] pregen_auxdb = self._pregen_auxdb.get(repo_path) if pregen_auxdb is not None: auxdbs.append(pregen_auxdb) ro_auxdb = self._ro_auxdb.get(repo_path) if ro_auxdb is not None: auxdbs.append(ro_auxdb) auxdbs.append(self.auxdb[repo_path]) eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db for auxdb in auxdbs: try: metadata = auxdb[cpv] except KeyError: continue except CacheError: if not auxdb.readonly: try: del auxdb[cpv] except (KeyError, CacheError): pass continue eapi = metadata.get('EAPI', '').strip() if not eapi: eapi = '0' metadata['EAPI'] = eapi if not eapi_is_supported(eapi): # Since we're supposed to be able to efficiently obtain the # EAPI from _parse_eapi_ebuild_head, we disregard cache entries # for unsupported EAPIs. continue if auxdb.validate_entry(metadata, ebuild_hash, eclass_db): break else: metadata = None return (metadata, ebuild_hash)
def action_metadata(settings, portdb, myopts, porttrees=None): if porttrees is None: porttrees = portdb.porttrees portage.writemsg_stdout("\n>>> Updating Portage cache\n") cachedir = os.path.normpath(settings.depcachedir) if cachedir in [ "/", "/bin", "/dev", "/etc", "/home", "/lib", "/opt", "/proc", "/root", "/sbin", "/sys", "/tmp", "/usr", "/var", ]: print( "!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY " + "ROOT DIRECTORY ON YOUR SYSTEM.", file=sys.stderr, ) print( "!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir, file=sys.stderr, ) sys.exit(73) if not os.path.exists(cachedir): os.makedirs(cachedir) auxdbkeys = portdb._known_keys class TreeData: __slots__ = ("dest_db", "eclass_db", "path", "src_db", "valid_nodes") def __init__(self, dest_db, eclass_db, path, src_db): self.dest_db = dest_db self.eclass_db = eclass_db self.path = path self.src_db = src_db self.valid_nodes = set() porttrees_data = [] for path in porttrees: src_db = portdb._pregen_auxdb.get(path) if src_db is None: # portdbapi does not populate _pregen_auxdb # when FEATURES=metadata-transfer is enabled src_db = portdb._create_pregen_cache(path) if src_db is not None: eclass_db = portdb.repositories.get_repo_for_location(path).eclass_db # Update eclass data which may be stale after sync. eclass_db.update_eclasses() porttrees_data.append(TreeData(portdb.auxdb[path], eclass_db, path, src_db)) porttrees = [tree_data.path for tree_data in porttrees_data] quiet = ( settings.get("TERM") == "dumb" or "--quiet" in myopts or not sys.stdout.isatty() ) onProgress = None if not quiet: progressBar = portage.output.TermProgressBar() progressHandler = ProgressHandler() onProgress = progressHandler.onProgress def display(): progressBar.set(progressHandler.curval, progressHandler.maxval) progressHandler.display = display def sigwinch_handler(signum, frame): lines, progressBar.term_columns = portage.output.get_term_size() signal.signal(signal.SIGWINCH, sigwinch_handler) # Temporarily override portdb.porttrees so portdb.cp_all() # will only return the relevant subset. portdb_porttrees = portdb.porttrees portdb.porttrees = porttrees try: cp_all = portdb.cp_all() finally: portdb.porttrees = portdb_porttrees curval = 0 maxval = len(cp_all) if onProgress is not None: onProgress(maxval, curval) # TODO: Display error messages, but do not interfere with the progress bar. # Here's how: # 1) erase the progress bar # 2) show the error message # 3) redraw the progress bar on a new line for cp in cp_all: for tree_data in porttrees_data: src_chf = tree_data.src_db.validation_chf dest_chf = tree_data.dest_db.validation_chf dest_chf_key = "_%s_" % dest_chf dest_chf_getter = operator.attrgetter(dest_chf) for cpv in portdb.cp_list(cp, mytree=tree_data.path): tree_data.valid_nodes.add(cpv) try: src = tree_data.src_db[cpv] except (CacheError, KeyError): continue ebuild_location = portdb.findname(cpv, mytree=tree_data.path) if ebuild_location is None: continue ebuild_hash = hashed_path(ebuild_location) try: if not tree_data.src_db.validate_entry( src, ebuild_hash, tree_data.eclass_db ): continue except CacheError: continue eapi = src.get("EAPI") if not eapi: eapi = "0" eapi_supported = eapi_is_supported(eapi) if not eapi_supported: continue dest = None try: dest = tree_data.dest_db[cpv] except (KeyError, CacheError): pass for d in (src, dest): if d is not None and d.get("EAPI") in ("", "0"): del d["EAPI"] if src_chf != "mtime": # src may contain an irrelevant _mtime_ which corresponds # to the time that the cache entry was written src.pop("_mtime_", None) if src_chf != dest_chf: # populate src entry with dest_chf_key # (the validity of the dest_chf that we generate from the # ebuild here relies on the fact that we already used # validate_entry to validate the ebuild with src_chf) src[dest_chf_key] = dest_chf_getter(ebuild_hash) if dest is not None: if not ( dest.get(dest_chf_key) == src[dest_chf_key] and tree_data.eclass_db.validate_and_rewrite_cache( dest["_eclasses_"], tree_data.dest_db.validation_chf, tree_data.dest_db.store_eclass_paths, ) is not None and set(dest["_eclasses_"]) == set(src["_eclasses_"]) ): dest = None else: # We don't want to skip the write unless we're really # sure that the existing cache is identical, so don't # trust _mtime_ and _eclasses_ alone. for k in auxdbkeys: if dest.get(k, "") != src.get(k, ""): dest = None break if dest is not None: # The existing data is valid and identical, # so there's no need to overwrite it. continue try: tree_data.dest_db[cpv] = src except CacheError: # ignore it; can't do anything about it. pass curval += 1 if onProgress is not None: onProgress(maxval, curval) if onProgress is not None: onProgress(maxval, curval) for tree_data in porttrees_data: try: dead_nodes = set(tree_data.dest_db) except CacheError as e: writemsg_level( "Error listing cache entries for " + "'%s': %s, continuing...\n" % (tree_data.path, e), level=logging.ERROR, noiselevel=-1, ) del e else: dead_nodes.difference_update(tree_data.valid_nodes) for cpv in dead_nodes: try: del tree_data.dest_db[cpv] except (KeyError, CacheError): pass if not quiet: # make sure the final progress is displayed progressHandler.display() print() signal.signal(signal.SIGWINCH, signal.SIG_DFL) portdb.flush_cache() sys.stdout.flush()
def action_metadata(settings, portdb, myopts, porttrees=None): if porttrees is None: porttrees = portdb.porttrees portage.writemsg_stdout("\n>>> Updating Portage cache\n") cachedir = os.path.normpath(settings.depcachedir) if cachedir in ["/", "/bin", "/dev", "/etc", "/home", "/lib", "/opt", "/proc", "/root", "/sbin", "/sys", "/tmp", "/usr", "/var"]: print("!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY " + \ "ROOT DIRECTORY ON YOUR SYSTEM.", file=sys.stderr) print("!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir, file=sys.stderr) sys.exit(73) if not os.path.exists(cachedir): os.makedirs(cachedir) auxdbkeys = portdb._known_keys class TreeData(object): __slots__ = ('dest_db', 'eclass_db', 'path', 'src_db', 'valid_nodes') def __init__(self, dest_db, eclass_db, path, src_db): self.dest_db = dest_db self.eclass_db = eclass_db self.path = path self.src_db = src_db self.valid_nodes = set() porttrees_data = [] for path in porttrees: src_db = portdb._pregen_auxdb.get(path) if src_db is None: # portdbapi does not populate _pregen_auxdb # when FEATURES=metadata-transfer is enabled src_db = portdb._create_pregen_cache(path) if src_db is not None: porttrees_data.append(TreeData(portdb.auxdb[path], portdb.repositories.get_repo_for_location(path).eclass_db, path, src_db)) porttrees = [tree_data.path for tree_data in porttrees_data] quiet = settings.get('TERM') == 'dumb' or \ '--quiet' in myopts or \ not sys.stdout.isatty() onProgress = None if not quiet: progressBar = portage.output.TermProgressBar() progressHandler = ProgressHandler() onProgress = progressHandler.onProgress def display(): progressBar.set(progressHandler.curval, progressHandler.maxval) progressHandler.display = display def sigwinch_handler(signum, frame): lines, progressBar.term_columns = \ portage.output.get_term_size() signal.signal(signal.SIGWINCH, sigwinch_handler) # Temporarily override portdb.porttrees so portdb.cp_all() # will only return the relevant subset. portdb_porttrees = portdb.porttrees portdb.porttrees = porttrees try: cp_all = portdb.cp_all() finally: portdb.porttrees = portdb_porttrees curval = 0 maxval = len(cp_all) if onProgress is not None: onProgress(maxval, curval) # TODO: Display error messages, but do not interfere with the progress bar. # Here's how: # 1) erase the progress bar # 2) show the error message # 3) redraw the progress bar on a new line for cp in cp_all: for tree_data in porttrees_data: src_chf = tree_data.src_db.validation_chf dest_chf = tree_data.dest_db.validation_chf dest_chf_key = '_%s_' % dest_chf dest_chf_getter = operator.attrgetter(dest_chf) for cpv in portdb.cp_list(cp, mytree=tree_data.path): tree_data.valid_nodes.add(cpv) try: src = tree_data.src_db[cpv] except (CacheError, KeyError): continue ebuild_location = portdb.findname(cpv, mytree=tree_data.path) if ebuild_location is None: continue ebuild_hash = hashed_path(ebuild_location) try: if not tree_data.src_db.validate_entry(src, ebuild_hash, tree_data.eclass_db): continue except CacheError: continue eapi = src.get('EAPI') if not eapi: eapi = '0' eapi_supported = eapi_is_supported(eapi) if not eapi_supported: continue dest = None try: dest = tree_data.dest_db[cpv] except (KeyError, CacheError): pass for d in (src, dest): if d is not None and d.get('EAPI') in ('', '0'): del d['EAPI'] if src_chf != 'mtime': # src may contain an irrelevant _mtime_ which corresponds # to the time that the cache entry was written src.pop('_mtime_', None) if src_chf != dest_chf: # populate src entry with dest_chf_key # (the validity of the dest_chf that we generate from the # ebuild here relies on the fact that we already used # validate_entry to validate the ebuild with src_chf) src[dest_chf_key] = dest_chf_getter(ebuild_hash) if dest is not None: if not (dest[dest_chf_key] == src[dest_chf_key] and \ tree_data.eclass_db.validate_and_rewrite_cache( dest['_eclasses_'], tree_data.dest_db.validation_chf, tree_data.dest_db.store_eclass_paths) is not None and \ set(dest['_eclasses_']) == set(src['_eclasses_'])): dest = None else: # We don't want to skip the write unless we're really # sure that the existing cache is identical, so don't # trust _mtime_ and _eclasses_ alone. for k in auxdbkeys: if dest.get(k, '') != src.get(k, ''): dest = None break if dest is not None: # The existing data is valid and identical, # so there's no need to overwrite it. continue try: tree_data.dest_db[cpv] = src except CacheError: # ignore it; can't do anything about it. pass curval += 1 if onProgress is not None: onProgress(maxval, curval) if onProgress is not None: onProgress(maxval, curval) for tree_data in porttrees_data: try: dead_nodes = set(tree_data.dest_db) except CacheError as e: writemsg_level("Error listing cache entries for " + \ "'%s': %s, continuing...\n" % (tree_data.path, e), level=logging.ERROR, noiselevel=-1) del e else: dead_nodes.difference_update(tree_data.valid_nodes) for cpv in dead_nodes: try: del tree_data.dest_db[cpv] except (KeyError, CacheError): pass if not quiet: # make sure the final progress is displayed progressHandler.display() print() signal.signal(signal.SIGWINCH, signal.SIG_DFL) portdb.flush_cache() sys.stdout.flush()