def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
try:
ebuild_hash = eclass_cache.hashed_path(ebuild_path)
# snag mtime since we use it later, and to trigger stat failure
# if it doesn't exist
ebuild_hash.mtime
except FileNotFound:
writemsg(_("!!! aux_get(): ebuild for " \
"'%s' does not exist at:\n") % (cpv,), noiselevel=-1)
writemsg("!!! %s\n" % ebuild_path, noiselevel=-1)
raise KeyError(cpv)
# Pull pre-generated metadata from the metadata/cache/
# directory if it exists and is valid, otherwise fall
# back to the normal writable cache.
auxdbs = []
pregen_auxdb = self._pregen_auxdb.get(repo_path)
if pregen_auxdb is not None:
auxdbs.append(pregen_auxdb)
ro_auxdb = self._ro_auxdb.get(repo_path)
if ro_auxdb is not None:
auxdbs.append(ro_auxdb)
auxdbs.append(self.auxdb[repo_path])
eclass_db = self.repositories.get_repo_for_location(
repo_path).eclass_db
for auxdb in auxdbs:
try:
metadata = auxdb[cpv]
except KeyError:
continue
except CacheError:
if not auxdb.readonly:
try:
del auxdb[cpv]
except (KeyError, CacheError):
pass
continue
eapi = metadata.get('EAPI', '').strip()
if not eapi:
eapi = '0'
metadata['EAPI'] = eapi
if not eapi_is_supported(eapi):
# Since we're supposed to be able to efficiently obtain the
# EAPI from _parse_eapi_ebuild_head, we disregard cache entries
# for unsupported EAPIs.
continue
if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
break
else:
metadata = None
return (metadata, ebuild_hash)
def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
try:
ebuild_hash = eclass_cache.hashed_path(ebuild_path)
# snag mtime since we use it later, and to trigger stat failure
# if it doesn't exist
ebuild_hash.mtime
except FileNotFound:
writemsg(_("!!! aux_get(): ebuild for " \
"'%s' does not exist at:\n") % (cpv,), noiselevel=-1)
writemsg("!!! %s\n" % ebuild_path, noiselevel=-1)
raise KeyError(cpv)
# Pull pre-generated metadata from the metadata/cache/
# directory if it exists and is valid, otherwise fall
# back to the normal writable cache.
auxdbs = []
pregen_auxdb = self._pregen_auxdb.get(repo_path)
if pregen_auxdb is not None:
auxdbs.append(pregen_auxdb)
ro_auxdb = self._ro_auxdb.get(repo_path)
if ro_auxdb is not None:
auxdbs.append(ro_auxdb)
auxdbs.append(self.auxdb[repo_path])
eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db
for auxdb in auxdbs:
try:
metadata = auxdb[cpv]
except KeyError:
continue
except CacheError:
if not auxdb.readonly:
try:
del auxdb[cpv]
except (KeyError, CacheError):
pass
continue
eapi = metadata.get('EAPI', '').strip()
if not eapi:
eapi = '0'
metadata['EAPI'] = eapi
if not eapi_is_supported(eapi):
# Since we're supposed to be able to efficiently obtain the
# EAPI from _parse_eapi_ebuild_head, we disregard cache entries
# for unsupported EAPIs.
continue
if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
break
else:
metadata = None
return (metadata, ebuild_hash)
def action_metadata(settings, portdb, myopts, porttrees=None):
if porttrees is None:
porttrees = portdb.porttrees
portage.writemsg_stdout("\n>>> Updating Portage cache\n")
cachedir = os.path.normpath(settings.depcachedir)
if cachedir in [
"/",
"/bin",
"/dev",
"/etc",
"/home",
"/lib",
"/opt",
"/proc",
"/root",
"/sbin",
"/sys",
"/tmp",
"/usr",
"/var",
]:
print(
"!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY "
+ "ROOT DIRECTORY ON YOUR SYSTEM.",
file=sys.stderr,
)
print(
"!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir,
file=sys.stderr,
)
sys.exit(73)
if not os.path.exists(cachedir):
os.makedirs(cachedir)
auxdbkeys = portdb._known_keys
class TreeData:
__slots__ = ("dest_db", "eclass_db", "path", "src_db", "valid_nodes")
def __init__(self, dest_db, eclass_db, path, src_db):
self.dest_db = dest_db
self.eclass_db = eclass_db
self.path = path
self.src_db = src_db
self.valid_nodes = set()
porttrees_data = []
for path in porttrees:
src_db = portdb._pregen_auxdb.get(path)
if src_db is None:
# portdbapi does not populate _pregen_auxdb
# when FEATURES=metadata-transfer is enabled
src_db = portdb._create_pregen_cache(path)
if src_db is not None:
eclass_db = portdb.repositories.get_repo_for_location(path).eclass_db
# Update eclass data which may be stale after sync.
eclass_db.update_eclasses()
porttrees_data.append(TreeData(portdb.auxdb[path], eclass_db, path, src_db))
porttrees = [tree_data.path for tree_data in porttrees_data]
quiet = (
settings.get("TERM") == "dumb" or "--quiet" in myopts or not sys.stdout.isatty()
)
onProgress = None
if not quiet:
progressBar = portage.output.TermProgressBar()
progressHandler = ProgressHandler()
onProgress = progressHandler.onProgress
def display():
progressBar.set(progressHandler.curval, progressHandler.maxval)
progressHandler.display = display
def sigwinch_handler(signum, frame):
lines, progressBar.term_columns = portage.output.get_term_size()
signal.signal(signal.SIGWINCH, sigwinch_handler)
# Temporarily override portdb.porttrees so portdb.cp_all()
# will only return the relevant subset.
portdb_porttrees = portdb.porttrees
portdb.porttrees = porttrees
try:
cp_all = portdb.cp_all()
finally:
portdb.porttrees = portdb_porttrees
curval = 0
maxval = len(cp_all)
if onProgress is not None:
onProgress(maxval, curval)
# TODO: Display error messages, but do not interfere with the progress bar.
# Here's how:
# 1) erase the progress bar
# 2) show the error message
# 3) redraw the progress bar on a new line
for cp in cp_all:
for tree_data in porttrees_data:
src_chf = tree_data.src_db.validation_chf
dest_chf = tree_data.dest_db.validation_chf
dest_chf_key = "_%s_" % dest_chf
dest_chf_getter = operator.attrgetter(dest_chf)
for cpv in portdb.cp_list(cp, mytree=tree_data.path):
tree_data.valid_nodes.add(cpv)
try:
src = tree_data.src_db[cpv]
except (CacheError, KeyError):
continue
ebuild_location = portdb.findname(cpv, mytree=tree_data.path)
if ebuild_location is None:
continue
ebuild_hash = hashed_path(ebuild_location)
try:
if not tree_data.src_db.validate_entry(
src, ebuild_hash, tree_data.eclass_db
):
continue
except CacheError:
continue
eapi = src.get("EAPI")
if not eapi:
eapi = "0"
eapi_supported = eapi_is_supported(eapi)
if not eapi_supported:
continue
dest = None
try:
dest = tree_data.dest_db[cpv]
except (KeyError, CacheError):
pass
for d in (src, dest):
if d is not None and d.get("EAPI") in ("", "0"):
del d["EAPI"]
if src_chf != "mtime":
# src may contain an irrelevant _mtime_ which corresponds
# to the time that the cache entry was written
src.pop("_mtime_", None)
if src_chf != dest_chf:
# populate src entry with dest_chf_key
# (the validity of the dest_chf that we generate from the
# ebuild here relies on the fact that we already used
# validate_entry to validate the ebuild with src_chf)
src[dest_chf_key] = dest_chf_getter(ebuild_hash)
if dest is not None:
if not (
dest.get(dest_chf_key) == src[dest_chf_key]
and tree_data.eclass_db.validate_and_rewrite_cache(
dest["_eclasses_"],
tree_data.dest_db.validation_chf,
tree_data.dest_db.store_eclass_paths,
)
is not None
and set(dest["_eclasses_"]) == set(src["_eclasses_"])
):
dest = None
else:
# We don't want to skip the write unless we're really
# sure that the existing cache is identical, so don't
# trust _mtime_ and _eclasses_ alone.
for k in auxdbkeys:
if dest.get(k, "") != src.get(k, ""):
dest = None
break
if dest is not None:
# The existing data is valid and identical,
# so there's no need to overwrite it.
continue
try:
tree_data.dest_db[cpv] = src
except CacheError:
# ignore it; can't do anything about it.
pass
curval += 1
if onProgress is not None:
onProgress(maxval, curval)
if onProgress is not None:
onProgress(maxval, curval)
for tree_data in porttrees_data:
try:
dead_nodes = set(tree_data.dest_db)
except CacheError as e:
writemsg_level(
"Error listing cache entries for "
+ "'%s': %s, continuing...\n" % (tree_data.path, e),
level=logging.ERROR,
noiselevel=-1,
)
del e
else:
dead_nodes.difference_update(tree_data.valid_nodes)
for cpv in dead_nodes:
try:
del tree_data.dest_db[cpv]
except (KeyError, CacheError):
pass
if not quiet:
# make sure the final progress is displayed
progressHandler.display()
print()
signal.signal(signal.SIGWINCH, signal.SIG_DFL)
portdb.flush_cache()
sys.stdout.flush()
def action_metadata(settings, portdb, myopts, porttrees=None):
if porttrees is None:
porttrees = portdb.porttrees
portage.writemsg_stdout("\n>>> Updating Portage cache\n")
cachedir = os.path.normpath(settings.depcachedir)
if cachedir in ["/", "/bin", "/dev", "/etc", "/home",
"/lib", "/opt", "/proc", "/root", "/sbin",
"/sys", "/tmp", "/usr", "/var"]:
print("!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY " + \
"ROOT DIRECTORY ON YOUR SYSTEM.", file=sys.stderr)
print("!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir, file=sys.stderr)
sys.exit(73)
if not os.path.exists(cachedir):
os.makedirs(cachedir)
auxdbkeys = portdb._known_keys
class TreeData(object):
__slots__ = ('dest_db', 'eclass_db', 'path', 'src_db', 'valid_nodes')
def __init__(self, dest_db, eclass_db, path, src_db):
self.dest_db = dest_db
self.eclass_db = eclass_db
self.path = path
self.src_db = src_db
self.valid_nodes = set()
porttrees_data = []
for path in porttrees:
src_db = portdb._pregen_auxdb.get(path)
if src_db is None:
# portdbapi does not populate _pregen_auxdb
# when FEATURES=metadata-transfer is enabled
src_db = portdb._create_pregen_cache(path)
if src_db is not None:
porttrees_data.append(TreeData(portdb.auxdb[path],
portdb.repositories.get_repo_for_location(path).eclass_db, path, src_db))
porttrees = [tree_data.path for tree_data in porttrees_data]
quiet = settings.get('TERM') == 'dumb' or \
'--quiet' in myopts or \
not sys.stdout.isatty()
onProgress = None
if not quiet:
progressBar = portage.output.TermProgressBar()
progressHandler = ProgressHandler()
onProgress = progressHandler.onProgress
def display():
progressBar.set(progressHandler.curval, progressHandler.maxval)
progressHandler.display = display
def sigwinch_handler(signum, frame):
lines, progressBar.term_columns = \
portage.output.get_term_size()
signal.signal(signal.SIGWINCH, sigwinch_handler)
# Temporarily override portdb.porttrees so portdb.cp_all()
# will only return the relevant subset.
portdb_porttrees = portdb.porttrees
portdb.porttrees = porttrees
try:
cp_all = portdb.cp_all()
finally:
portdb.porttrees = portdb_porttrees
curval = 0
maxval = len(cp_all)
if onProgress is not None:
onProgress(maxval, curval)
# TODO: Display error messages, but do not interfere with the progress bar.
# Here's how:
# 1) erase the progress bar
# 2) show the error message
# 3) redraw the progress bar on a new line
for cp in cp_all:
for tree_data in porttrees_data:
src_chf = tree_data.src_db.validation_chf
dest_chf = tree_data.dest_db.validation_chf
dest_chf_key = '_%s_' % dest_chf
dest_chf_getter = operator.attrgetter(dest_chf)
for cpv in portdb.cp_list(cp, mytree=tree_data.path):
tree_data.valid_nodes.add(cpv)
try:
src = tree_data.src_db[cpv]
except (CacheError, KeyError):
continue
ebuild_location = portdb.findname(cpv, mytree=tree_data.path)
if ebuild_location is None:
continue
ebuild_hash = hashed_path(ebuild_location)
try:
if not tree_data.src_db.validate_entry(src,
ebuild_hash, tree_data.eclass_db):
continue
except CacheError:
continue
eapi = src.get('EAPI')
if not eapi:
eapi = '0'
eapi_supported = eapi_is_supported(eapi)
if not eapi_supported:
continue
dest = None
try:
dest = tree_data.dest_db[cpv]
except (KeyError, CacheError):
pass
for d in (src, dest):
if d is not None and d.get('EAPI') in ('', '0'):
del d['EAPI']
if src_chf != 'mtime':
# src may contain an irrelevant _mtime_ which corresponds
# to the time that the cache entry was written
src.pop('_mtime_', None)
if src_chf != dest_chf:
# populate src entry with dest_chf_key
# (the validity of the dest_chf that we generate from the
# ebuild here relies on the fact that we already used
# validate_entry to validate the ebuild with src_chf)
src[dest_chf_key] = dest_chf_getter(ebuild_hash)
if dest is not None:
if not (dest[dest_chf_key] == src[dest_chf_key] and \
tree_data.eclass_db.validate_and_rewrite_cache(
dest['_eclasses_'], tree_data.dest_db.validation_chf,
tree_data.dest_db.store_eclass_paths) is not None and \
set(dest['_eclasses_']) == set(src['_eclasses_'])):
dest = None
else:
# We don't want to skip the write unless we're really
# sure that the existing cache is identical, so don't
# trust _mtime_ and _eclasses_ alone.
for k in auxdbkeys:
if dest.get(k, '') != src.get(k, ''):
dest = None
break
if dest is not None:
# The existing data is valid and identical,
# so there's no need to overwrite it.
continue
try:
tree_data.dest_db[cpv] = src
except CacheError:
# ignore it; can't do anything about it.
pass
curval += 1
if onProgress is not None:
onProgress(maxval, curval)
if onProgress is not None:
onProgress(maxval, curval)
for tree_data in porttrees_data:
try:
dead_nodes = set(tree_data.dest_db)
except CacheError as e:
writemsg_level("Error listing cache entries for " + \
"'%s': %s, continuing...\n" % (tree_data.path, e),
level=logging.ERROR, noiselevel=-1)
del e
else:
dead_nodes.difference_update(tree_data.valid_nodes)
for cpv in dead_nodes:
try:
del tree_data.dest_db[cpv]
except (KeyError, CacheError):
pass
if not quiet:
# make sure the final progress is displayed
progressHandler.display()
print()
signal.signal(signal.SIGWINCH, signal.SIG_DFL)
portdb.flush_cache()
sys.stdout.flush()
