def _iter_metadata_processes(self):
portdb = self._portdb
valid_pkgs = self._valid_pkgs
cp_set = self._cp_set
consumer = self._consumer
for cp in self._cp_iter:
if self._terminated_tasks:
break
cp_set.add(cp)
portage.writemsg_stdout("Processing %s\n" % cp)
# We iterate over portdb.porttrees, since it's common to
# tweak this attribute in order to adjust repo selection.
for mytree in portdb.porttrees:
repo = portdb.repositories.get_repo_for_location(mytree)
cpv_list = portdb.cp_list(cp, mytree=[repo.location])
for cpv in cpv_list:
if self._terminated_tasks:
break
valid_pkgs.add(cpv)
ebuild_path, repo_path = portdb.findname2(cpv, myrepo=repo.name)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s%s%s'" % (cpv, _repo_separator, repo.name))
metadata, ebuild_hash = portdb._pull_valid_cache(
cpv, ebuild_path, repo_path)
if metadata is not None:
if consumer is not None:
consumer(cpv, repo_path, metadata, ebuild_hash, True)
continue
yield EbuildMetadataPhase(cpv=cpv,
ebuild_hash=ebuild_hash,
portdb=portdb, repo_path=repo_path,
settings=portdb.doebuild_settings,
write_auxdb=self._write_auxdb)
def _iter_metadata_processes(self):
portdb = self._portdb
valid_pkgs = self._valid_pkgs
cp_set = self._cp_set
consumer = self._consumer
for cp in self._cp_iter:
cp_set.add(cp)
portage.writemsg_stdout("Processing %s\n" % cp)
cpv_list = portdb.cp_list(cp)
for cpv in cpv_list:
valid_pkgs.add(cpv)
ebuild_path, repo_path = portdb.findname2(cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % cpv)
metadata, st, emtime = portdb._pull_valid_cache(
cpv, ebuild_path, repo_path)
if metadata is not None:
if consumer is not None:
consumer(cpv, ebuild_path, repo_path, metadata)
continue
yield EbuildMetadataPhase(
cpv=cpv,
ebuild_path=ebuild_path,
ebuild_mtime=emtime,
metadata_callback=portdb._metadata_callback,
portdb=portdb,
repo_path=repo_path,
settings=portdb.doebuild_settings)
def _metadata_process(self, cpv, ebuild_path, repo_path):
"""
Create an EbuildMetadataPhase instance to generate metadata for the
give ebuild.
@rtype: EbuildMetadataPhase
@returns: A new EbuildMetadataPhase instance, or None if the
metadata cache is already valid.
"""
metadata, ebuild_hash = self._pull_valid_cache(cpv, ebuild_path,
repo_path)
if metadata is not None:
return None
process = EbuildMetadataPhase(
cpv=cpv,
ebuild_hash=ebuild_hash,
metadata_callback=self._metadata_callback,
portdb=self,
repo_path=repo_path,
settings=self.doebuild_settings)
return process
def aux_get(self, mycpv, mylist, mytree=None, myrepo=None):
"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error'
cache_me = False
if myrepo is not None:
mytree = self.treemap.get(myrepo)
if mytree is None:
raise KeyError(myrepo)
if mytree is not None and len(self.porttrees) == 1 \
and mytree == self.porttrees[0]:
# mytree matches our only tree, so it's safe to
# ignore mytree and cache the result
mytree = None
myrepo = None
if mytree is None:
cache_me = True
if mytree is None and not self._known_keys.intersection(
mylist).difference(self._aux_cache_keys):
aux_cache = self._aux_cache.get(mycpv)
if aux_cache is not None:
return [aux_cache.get(x, "") for x in mylist]
cache_me = True
try:
cat, pkg = mycpv.split("/", 1)
except ValueError:
# Missing slash. Can't find ebuild so raise KeyError.
raise KeyError(mycpv)
myebuild, mylocation = self.findname2(mycpv, mytree)
if not myebuild:
writemsg("!!! aux_get(): %s\n" % \
_("ebuild not found for '%s'") % mycpv, noiselevel=1)
raise KeyError(mycpv)
mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
doregen = mydata is None
if doregen:
if myebuild in self._broken_ebuilds:
raise KeyError(mycpv)
proc = EbuildMetadataPhase(cpv=mycpv,
ebuild_hash=ebuild_hash, portdb=self,
repo_path=mylocation, scheduler=self._event_loop,
settings=self.doebuild_settings)
proc.start()
proc.wait()
if proc.returncode != os.EX_OK:
self._broken_ebuilds.add(myebuild)
raise KeyError(mycpv)
mydata = proc.metadata
mydata["repository"] = self.repositories.get_name_for_location(mylocation)
mydata["_mtime_"] = ebuild_hash.mtime
eapi = mydata.get("EAPI")
if not eapi:
eapi = "0"
mydata["EAPI"] = eapi
if eapi_is_supported(eapi):
mydata["INHERITED"] = " ".join(mydata.get("_eclasses_", []))
#finally, we look at our internal cache entry and return the requested data.
returnme = [mydata.get(x, "") for x in mylist]
if cache_me:
aux_cache = {}
for x in self._aux_cache_keys:
aux_cache[x] = mydata.get(x, "")
self._aux_cache[mycpv] = aux_cache
return returnme
def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None):
"""
Asynchronous form form of aux_get.
@param mycpv: cpv for an ebuild
@type mycpv: str
@param mylist: list of metadata keys
@type mylist: list
@param mytree: The canonical path of the tree in which the ebuild
is located, or None for automatic lookup
@type mytree: str
@param myrepo: name of the repo in which the ebuild is located,
or None for automatic lookup
@type myrepo: str
@param loop: event loop (defaults to global event loop)
@type loop: EventLoop
@return: list of metadata values
@rtype: asyncio.Future (or compatible)
"""
# Don't default to self._event_loop here, since that creates a
# local event loop for thread safety, and that could easily lead
# to simultaneous instantiation of multiple event loops here.
# Callers of this method certainly want the same event loop to
# be used for all calls.
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
cache_me = False
if myrepo is not None:
mytree = self.treemap.get(myrepo)
if mytree is None:
future.set_exception(PortageKeyError(myrepo))
return future
if mytree is not None and len(self.porttrees) == 1 \
and mytree == self.porttrees[0]:
# mytree matches our only tree, so it's safe to
# ignore mytree and cache the result
mytree = None
myrepo = None
if mytree is None:
cache_me = True
if mytree is None and not self._known_keys.intersection(
mylist).difference(self._aux_cache_keys):
aux_cache = self._aux_cache.get(mycpv)
if aux_cache is not None:
future.set_result([aux_cache.get(x, "") for x in mylist])
return future
cache_me = True
try:
cat, pkg = mycpv.split("/", 1)
except ValueError:
# Missing slash. Can't find ebuild so raise PortageKeyError.
future.set_exception(PortageKeyError(mycpv))
return future
myebuild, mylocation = self.findname2(mycpv, mytree)
if not myebuild:
writemsg("!!! aux_get(): %s\n" % \
_("ebuild not found for '%s'") % mycpv, noiselevel=1)
future.set_exception(PortageKeyError(mycpv))
return future
mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
if mydata is not None:
self._aux_get_return(
future, mycpv, mylist, myebuild, ebuild_hash,
mydata, mylocation, cache_me, None)
return future
if myebuild in self._broken_ebuilds:
future.set_exception(PortageKeyError(mycpv))
return future
proc = EbuildMetadataPhase(cpv=mycpv,
ebuild_hash=ebuild_hash, portdb=self,
repo_path=mylocation, scheduler=loop,
settings=self.doebuild_settings)
proc.addExitListener(functools.partial(self._aux_get_return,
future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation,
cache_me))
future.add_done_callback(functools.partial(self._aux_get_cancel, proc))
proc.start()
return future
def aux_get(self, mycpv, mylist, mytree=None, myrepo=None):
"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise PortageKeyError if error'
cache_me = False
if myrepo is not None:
mytree = self.treemap.get(myrepo)
if mytree is None:
raise PortageKeyError(myrepo)
if mytree is not None and len(self.porttrees) == 1 \
and mytree == self.porttrees[0]:
# mytree matches our only tree, so it's safe to
# ignore mytree and cache the result
mytree = None
myrepo = None
if mytree is None:
cache_me = True
if mytree is None and not self._known_keys.intersection(
mylist).difference(self._aux_cache_keys):
aux_cache = self._aux_cache.get(mycpv)
if aux_cache is not None:
return [aux_cache.get(x, "") for x in mylist]
cache_me = True
try:
cat, pkg = mycpv.split("/", 1)
except ValueError:
# Missing slash. Can't find ebuild so raise PortageKeyError.
raise PortageKeyError(mycpv)
myebuild, mylocation = self.findname2(mycpv, mytree)
if not myebuild:
writemsg("!!! aux_get(): %s\n" % \
_("ebuild not found for '%s'") % mycpv, noiselevel=1)
raise PortageKeyError(mycpv)
mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
doregen = mydata is None
if doregen:
if myebuild in self._broken_ebuilds:
raise PortageKeyError(mycpv)
proc = EbuildMetadataPhase(cpv=mycpv,
ebuild_hash=ebuild_hash, portdb=self,
repo_path=mylocation, scheduler=self._event_loop,
settings=self.doebuild_settings)
proc.start()
proc.wait()
if proc.returncode != os.EX_OK:
self._broken_ebuilds.add(myebuild)
raise PortageKeyError(mycpv)
mydata = proc.metadata
mydata["repository"] = self.repositories.get_name_for_location(mylocation)
mydata["_mtime_"] = ebuild_hash.mtime
eapi = mydata.get("EAPI")
if not eapi:
eapi = "0"
mydata["EAPI"] = eapi
if eapi_is_supported(eapi):
mydata["INHERITED"] = " ".join(mydata.get("_eclasses_", []))
#finally, we look at our internal cache entry and return the requested data.
returnme = [mydata.get(x, "") for x in mylist]
if cache_me and self.frozen:
aux_cache = {}
for x in self._aux_cache_keys:
aux_cache[x] = mydata.get(x, "")
self._aux_cache[mycpv] = aux_cache
return returnme
def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None):
"""
Asynchronous form form of aux_get.
@param mycpv: cpv for an ebuild
@type mycpv: str
@param mylist: list of metadata keys
@type mylist: list
@param mytree: The canonical path of the tree in which the ebuild
is located, or None for automatic lookup
@type mytree: str
@param myrepo: name of the repo in which the ebuild is located,
or None for automatic lookup
@type myrepo: str
@param loop: event loop (defaults to global event loop)
@type loop: EventLoop
@return: list of metadata values
@rtype: asyncio.Future (or compatible)
"""
# Don't default to self._event_loop here, since that creates a
# local event loop for thread safety, and that could easily lead
# to simultaneous instantiation of multiple event loops here.
# Callers of this method certainly want the same event loop to
# be used for all calls.
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
cache_me = False
if myrepo is not None:
mytree = self.treemap.get(myrepo)
if mytree is None:
future.set_exception(PortageKeyError(myrepo))
return future
if mytree is not None and len(self.porttrees) == 1 \
and mytree == self.porttrees[0]:
# mytree matches our only tree, so it's safe to
# ignore mytree and cache the result
mytree = None
myrepo = None
if mytree is None:
cache_me = True
if mytree is None and not self._known_keys.intersection(
mylist).difference(self._aux_cache_keys):
aux_cache = self._aux_cache.get(mycpv)
if aux_cache is not None:
future.set_result([aux_cache.get(x, "") for x in mylist])
return future
cache_me = True
try:
cat, pkg = mycpv.split("/", 1)
except ValueError:
# Missing slash. Can't find ebuild so raise PortageKeyError.
future.set_exception(PortageKeyError(mycpv))
return future
myebuild, mylocation = self.findname2(mycpv, mytree)
if not myebuild:
writemsg("!!! aux_get(): %s\n" % \
_("ebuild not found for '%s'") % mycpv, noiselevel=1)
future.set_exception(PortageKeyError(mycpv))
return future
mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
if mydata is not None:
self._aux_get_return(
future, mycpv, mylist, myebuild, ebuild_hash,
mydata, mylocation, cache_me, None)
return future
if myebuild in self._broken_ebuilds:
future.set_exception(PortageKeyError(mycpv))
return future
proc = EbuildMetadataPhase(cpv=mycpv,
ebuild_hash=ebuild_hash, portdb=self,
repo_path=mylocation, scheduler=loop,
settings=self.doebuild_settings)
proc.addExitListener(functools.partial(self._aux_get_return,
future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation,
cache_me))
future.add_done_callback(functools.partial(self._aux_get_cancel, proc))
proc.start()
return future
def aux_get(self, mycpv, mylist, mytree=None):
"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error'
cache_me = False
if not mytree:
cache_me = True
if not mytree and not self._known_keys.intersection(mylist).difference(
self._aux_cache_keys):
aux_cache = self._aux_cache.get(mycpv)
if aux_cache is not None:
return [aux_cache.get(x, "") for x in mylist]
cache_me = True
global auxdbkeys, auxdbkeylen
try:
cat, pkg = mycpv.split("/", 1)
except ValueError:
# Missing slash. Can't find ebuild so raise KeyError.
raise KeyError(mycpv)
myebuild, mylocation = self.findname2(mycpv, mytree)
if not myebuild:
writemsg("!!! aux_get(): %s\n" % \
_("ebuild not found for '%s'") % mycpv, noiselevel=1)
raise KeyError(mycpv)
mydata, st, emtime = self._pull_valid_cache(mycpv, myebuild,
mylocation)
doregen = mydata is None
if doregen:
if myebuild in self._broken_ebuilds:
raise KeyError(mycpv)
if not self._have_root_eclass_dir:
raise KeyError(mycpv)
self.doebuild_settings.setcpv(mycpv)
eapi = None
if eapi is None and \
'parse-eapi-ebuild-head' in self.doebuild_settings.features:
eapi = portage._parse_eapi_ebuild_head(
codecs.open(_unicode_encode(myebuild,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace'))
if eapi is not None:
self.doebuild_settings.configdict['pkg']['EAPI'] = eapi
if eapi is not None and not portage.eapi_is_supported(eapi):
mydata = self._metadata_callback(mycpv, myebuild, mylocation,
{'EAPI': eapi}, emtime)
else:
sched = TaskScheduler()
proc = EbuildMetadataPhase(
cpv=mycpv,
ebuild_path=myebuild,
ebuild_mtime=emtime,
metadata_callback=self._metadata_callback,
portdb=self,
repo_path=mylocation,
scheduler=sched.sched_iface,
settings=self.doebuild_settings)
sched.add(proc)
sched.run()
if proc.returncode != os.EX_OK:
self._broken_ebuilds.add(myebuild)
raise KeyError(mycpv)
mydata = proc.metadata
# do we have a origin repository name for the current package
mydata["repository"] = self._repository_map.get(mylocation, "")
mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", []))
mydata["_mtime_"] = st[stat.ST_MTIME]
eapi = mydata.get("EAPI")
if not eapi:
eapi = "0"
mydata["EAPI"] = eapi
if not eapi_is_supported(eapi):
for k in set(mydata).difference(("_mtime_", "_eclasses_")):
mydata[k] = ""
mydata["EAPI"] = "-" + eapi.lstrip("-")
#finally, we look at our internal cache entry and return the requested data.
returnme = [mydata.get(x, "") for x in mylist]
if cache_me:
aux_cache = {}
for x in self._aux_cache_keys:
aux_cache[x] = mydata.get(x, "")
self._aux_cache[mycpv] = aux_cache
return returnme
