def _iter_metadata_processes(self): portdb = self._portdb valid_pkgs = self._valid_pkgs cp_set = self._cp_set consumer = self._consumer for cp in self._cp_iter: if self._terminated_tasks: break cp_set.add(cp) portage.writemsg_stdout("Processing %s\n" % cp) # We iterate over portdb.porttrees, since it's common to # tweak this attribute in order to adjust repo selection. for mytree in portdb.porttrees: repo = portdb.repositories.get_repo_for_location(mytree) cpv_list = portdb.cp_list(cp, mytree=[repo.location]) for cpv in cpv_list: if self._terminated_tasks: break valid_pkgs.add(cpv) ebuild_path, repo_path = portdb.findname2(cpv, myrepo=repo.name) if ebuild_path is None: raise AssertionError("ebuild not found for '%s%s%s'" % (cpv, _repo_separator, repo.name)) metadata, ebuild_hash = portdb._pull_valid_cache( cpv, ebuild_path, repo_path) if metadata is not None: if consumer is not None: consumer(cpv, repo_path, metadata, ebuild_hash, True) continue yield EbuildMetadataPhase(cpv=cpv, ebuild_hash=ebuild_hash, portdb=portdb, repo_path=repo_path, settings=portdb.doebuild_settings, write_auxdb=self._write_auxdb)
def _iter_metadata_processes(self): portdb = self._portdb valid_pkgs = self._valid_pkgs cp_set = self._cp_set consumer = self._consumer for cp in self._cp_iter: cp_set.add(cp) portage.writemsg_stdout("Processing %s\n" % cp) cpv_list = portdb.cp_list(cp) for cpv in cpv_list: valid_pkgs.add(cpv) ebuild_path, repo_path = portdb.findname2(cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % cpv) metadata, st, emtime = portdb._pull_valid_cache( cpv, ebuild_path, repo_path) if metadata is not None: if consumer is not None: consumer(cpv, ebuild_path, repo_path, metadata) continue yield EbuildMetadataPhase( cpv=cpv, ebuild_path=ebuild_path, ebuild_mtime=emtime, metadata_callback=portdb._metadata_callback, portdb=portdb, repo_path=repo_path, settings=portdb.doebuild_settings)
def _metadata_process(self, cpv, ebuild_path, repo_path): """ Create an EbuildMetadataPhase instance to generate metadata for the give ebuild. @rtype: EbuildMetadataPhase @returns: A new EbuildMetadataPhase instance, or None if the metadata cache is already valid. """ metadata, ebuild_hash = self._pull_valid_cache(cpv, ebuild_path, repo_path) if metadata is not None: return None process = EbuildMetadataPhase( cpv=cpv, ebuild_hash=ebuild_hash, metadata_callback=self._metadata_callback, portdb=self, repo_path=repo_path, settings=self.doebuild_settings) return process
def aux_get(self, mycpv, mylist, mytree=None, myrepo=None): "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc." 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error' cache_me = False if myrepo is not None: mytree = self.treemap.get(myrepo) if mytree is None: raise KeyError(myrepo) if mytree is not None and len(self.porttrees) == 1 \ and mytree == self.porttrees[0]: # mytree matches our only tree, so it's safe to # ignore mytree and cache the result mytree = None myrepo = None if mytree is None: cache_me = True if mytree is None and not self._known_keys.intersection( mylist).difference(self._aux_cache_keys): aux_cache = self._aux_cache.get(mycpv) if aux_cache is not None: return [aux_cache.get(x, "") for x in mylist] cache_me = True try: cat, pkg = mycpv.split("/", 1) except ValueError: # Missing slash. Can't find ebuild so raise KeyError. raise KeyError(mycpv) myebuild, mylocation = self.findname2(mycpv, mytree) if not myebuild: writemsg("!!! aux_get(): %s\n" % \ _("ebuild not found for '%s'") % mycpv, noiselevel=1) raise KeyError(mycpv) mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation) doregen = mydata is None if doregen: if myebuild in self._broken_ebuilds: raise KeyError(mycpv) proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, repo_path=mylocation, scheduler=self._event_loop, settings=self.doebuild_settings) proc.start() proc.wait() if proc.returncode != os.EX_OK: self._broken_ebuilds.add(myebuild) raise KeyError(mycpv) mydata = proc.metadata mydata["repository"] = self.repositories.get_name_for_location(mylocation) mydata["_mtime_"] = ebuild_hash.mtime eapi = mydata.get("EAPI") if not eapi: eapi = "0" mydata["EAPI"] = eapi if eapi_is_supported(eapi): mydata["INHERITED"] = " ".join(mydata.get("_eclasses_", [])) #finally, we look at our internal cache entry and return the requested data. returnme = [mydata.get(x, "") for x in mylist] if cache_me: aux_cache = {} for x in self._aux_cache_keys: aux_cache[x] = mydata.get(x, "") self._aux_cache[mycpv] = aux_cache return returnme
def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None): """ Asynchronous form form of aux_get. @param mycpv: cpv for an ebuild @type mycpv: str @param mylist: list of metadata keys @type mylist: list @param mytree: The canonical path of the tree in which the ebuild is located, or None for automatic lookup @type mytree: str @param myrepo: name of the repo in which the ebuild is located, or None for automatic lookup @type myrepo: str @param loop: event loop (defaults to global event loop) @type loop: EventLoop @return: list of metadata values @rtype: asyncio.Future (or compatible) """ # Don't default to self._event_loop here, since that creates a # local event loop for thread safety, and that could easily lead # to simultaneous instantiation of multiple event loops here. # Callers of this method certainly want the same event loop to # be used for all calls. loop = asyncio._wrap_loop(loop) future = loop.create_future() cache_me = False if myrepo is not None: mytree = self.treemap.get(myrepo) if mytree is None: future.set_exception(PortageKeyError(myrepo)) return future if mytree is not None and len(self.porttrees) == 1 \ and mytree == self.porttrees[0]: # mytree matches our only tree, so it's safe to # ignore mytree and cache the result mytree = None myrepo = None if mytree is None: cache_me = True if mytree is None and not self._known_keys.intersection( mylist).difference(self._aux_cache_keys): aux_cache = self._aux_cache.get(mycpv) if aux_cache is not None: future.set_result([aux_cache.get(x, "") for x in mylist]) return future cache_me = True try: cat, pkg = mycpv.split("/", 1) except ValueError: # Missing slash. Can't find ebuild so raise PortageKeyError. future.set_exception(PortageKeyError(mycpv)) return future myebuild, mylocation = self.findname2(mycpv, mytree) if not myebuild: writemsg("!!! aux_get(): %s\n" % \ _("ebuild not found for '%s'") % mycpv, noiselevel=1) future.set_exception(PortageKeyError(mycpv)) return future mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation) if mydata is not None: self._aux_get_return( future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation, cache_me, None) return future if myebuild in self._broken_ebuilds: future.set_exception(PortageKeyError(mycpv)) return future proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, repo_path=mylocation, scheduler=loop, settings=self.doebuild_settings) proc.addExitListener(functools.partial(self._aux_get_return, future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation, cache_me)) future.add_done_callback(functools.partial(self._aux_get_cancel, proc)) proc.start() return future
def aux_get(self, mycpv, mylist, mytree=None, myrepo=None): "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc." 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise PortageKeyError if error' cache_me = False if myrepo is not None: mytree = self.treemap.get(myrepo) if mytree is None: raise PortageKeyError(myrepo) if mytree is not None and len(self.porttrees) == 1 \ and mytree == self.porttrees[0]: # mytree matches our only tree, so it's safe to # ignore mytree and cache the result mytree = None myrepo = None if mytree is None: cache_me = True if mytree is None and not self._known_keys.intersection( mylist).difference(self._aux_cache_keys): aux_cache = self._aux_cache.get(mycpv) if aux_cache is not None: return [aux_cache.get(x, "") for x in mylist] cache_me = True try: cat, pkg = mycpv.split("/", 1) except ValueError: # Missing slash. Can't find ebuild so raise PortageKeyError. raise PortageKeyError(mycpv) myebuild, mylocation = self.findname2(mycpv, mytree) if not myebuild: writemsg("!!! aux_get(): %s\n" % \ _("ebuild not found for '%s'") % mycpv, noiselevel=1) raise PortageKeyError(mycpv) mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation) doregen = mydata is None if doregen: if myebuild in self._broken_ebuilds: raise PortageKeyError(mycpv) proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, repo_path=mylocation, scheduler=self._event_loop, settings=self.doebuild_settings) proc.start() proc.wait() if proc.returncode != os.EX_OK: self._broken_ebuilds.add(myebuild) raise PortageKeyError(mycpv) mydata = proc.metadata mydata["repository"] = self.repositories.get_name_for_location(mylocation) mydata["_mtime_"] = ebuild_hash.mtime eapi = mydata.get("EAPI") if not eapi: eapi = "0" mydata["EAPI"] = eapi if eapi_is_supported(eapi): mydata["INHERITED"] = " ".join(mydata.get("_eclasses_", [])) #finally, we look at our internal cache entry and return the requested data. returnme = [mydata.get(x, "") for x in mylist] if cache_me and self.frozen: aux_cache = {} for x in self._aux_cache_keys: aux_cache[x] = mydata.get(x, "") self._aux_cache[mycpv] = aux_cache return returnme
def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None): """ Asynchronous form form of aux_get. @param mycpv: cpv for an ebuild @type mycpv: str @param mylist: list of metadata keys @type mylist: list @param mytree: The canonical path of the tree in which the ebuild is located, or None for automatic lookup @type mytree: str @param myrepo: name of the repo in which the ebuild is located, or None for automatic lookup @type myrepo: str @param loop: event loop (defaults to global event loop) @type loop: EventLoop @return: list of metadata values @rtype: asyncio.Future (or compatible) """ # Don't default to self._event_loop here, since that creates a # local event loop for thread safety, and that could easily lead # to simultaneous instantiation of multiple event loops here. # Callers of this method certainly want the same event loop to # be used for all calls. loop = asyncio._wrap_loop(loop) future = loop.create_future() cache_me = False if myrepo is not None: mytree = self.treemap.get(myrepo) if mytree is None: future.set_exception(PortageKeyError(myrepo)) return future if mytree is not None and len(self.porttrees) == 1 \ and mytree == self.porttrees[0]: # mytree matches our only tree, so it's safe to # ignore mytree and cache the result mytree = None myrepo = None if mytree is None: cache_me = True if mytree is None and not self._known_keys.intersection( mylist).difference(self._aux_cache_keys): aux_cache = self._aux_cache.get(mycpv) if aux_cache is not None: future.set_result([aux_cache.get(x, "") for x in mylist]) return future cache_me = True try: cat, pkg = mycpv.split("/", 1) except ValueError: # Missing slash. Can't find ebuild so raise PortageKeyError. future.set_exception(PortageKeyError(mycpv)) return future myebuild, mylocation = self.findname2(mycpv, mytree) if not myebuild: writemsg("!!! aux_get(): %s\n" % \ _("ebuild not found for '%s'") % mycpv, noiselevel=1) future.set_exception(PortageKeyError(mycpv)) return future mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation) if mydata is not None: self._aux_get_return( future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation, cache_me, None) return future if myebuild in self._broken_ebuilds: future.set_exception(PortageKeyError(mycpv)) return future proc = EbuildMetadataPhase(cpv=mycpv, ebuild_hash=ebuild_hash, portdb=self, repo_path=mylocation, scheduler=loop, settings=self.doebuild_settings) proc.addExitListener(functools.partial(self._aux_get_return, future, mycpv, mylist, myebuild, ebuild_hash, mydata, mylocation, cache_me)) future.add_done_callback(functools.partial(self._aux_get_cancel, proc)) proc.start() return future
def aux_get(self, mycpv, mylist, mytree=None): "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc." 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error' cache_me = False if not mytree: cache_me = True if not mytree and not self._known_keys.intersection(mylist).difference( self._aux_cache_keys): aux_cache = self._aux_cache.get(mycpv) if aux_cache is not None: return [aux_cache.get(x, "") for x in mylist] cache_me = True global auxdbkeys, auxdbkeylen try: cat, pkg = mycpv.split("/", 1) except ValueError: # Missing slash. Can't find ebuild so raise KeyError. raise KeyError(mycpv) myebuild, mylocation = self.findname2(mycpv, mytree) if not myebuild: writemsg("!!! aux_get(): %s\n" % \ _("ebuild not found for '%s'") % mycpv, noiselevel=1) raise KeyError(mycpv) mydata, st, emtime = self._pull_valid_cache(mycpv, myebuild, mylocation) doregen = mydata is None if doregen: if myebuild in self._broken_ebuilds: raise KeyError(mycpv) if not self._have_root_eclass_dir: raise KeyError(mycpv) self.doebuild_settings.setcpv(mycpv) eapi = None if eapi is None and \ 'parse-eapi-ebuild-head' in self.doebuild_settings.features: eapi = portage._parse_eapi_ebuild_head( codecs.open(_unicode_encode(myebuild, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace')) if eapi is not None: self.doebuild_settings.configdict['pkg']['EAPI'] = eapi if eapi is not None and not portage.eapi_is_supported(eapi): mydata = self._metadata_callback(mycpv, myebuild, mylocation, {'EAPI': eapi}, emtime) else: sched = TaskScheduler() proc = EbuildMetadataPhase( cpv=mycpv, ebuild_path=myebuild, ebuild_mtime=emtime, metadata_callback=self._metadata_callback, portdb=self, repo_path=mylocation, scheduler=sched.sched_iface, settings=self.doebuild_settings) sched.add(proc) sched.run() if proc.returncode != os.EX_OK: self._broken_ebuilds.add(myebuild) raise KeyError(mycpv) mydata = proc.metadata # do we have a origin repository name for the current package mydata["repository"] = self._repository_map.get(mylocation, "") mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", [])) mydata["_mtime_"] = st[stat.ST_MTIME] eapi = mydata.get("EAPI") if not eapi: eapi = "0" mydata["EAPI"] = eapi if not eapi_is_supported(eapi): for k in set(mydata).difference(("_mtime_", "_eclasses_")): mydata[k] = "" mydata["EAPI"] = "-" + eapi.lstrip("-") #finally, we look at our internal cache entry and return the requested data. returnme = [mydata.get(x, "") for x in mylist] if cache_me: aux_cache = {} for x in self._aux_cache_keys: aux_cache[x] = mydata.get(x, "") self._aux_cache[mycpv] = aux_cache return returnme