def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. self.cachepath = \ self.core.setup.cfp.get("packages", "cache", default=os.path.join(self.data, 'cache')) #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` #: object used to generate #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects for #: this plugin. self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, self) #: We cache #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: objects in ``collections`` so that calling :func:`Refresh` #: or :func:`Reload` can tell the collection objects to clean #: up their cache, but we don't actually use the cache to #: return a ``Collection`` object when one is requested, #: because that prevents new machines from working, since a #: ``Collection`` object gets created by #: :func:`get_additional_data`, which is called for all #: clients at server startup and various other times. (It #: would also prevent machines that change groups from working #: properly; e.g., if you reinstall a machine with a new OS, #: then returning a cached ``Collection`` object would give #: the wrong sources to that client.) These are keyed by the #: collection #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. self.collections = dict() #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` #: Unlike :attr:`collections`, this _is_ used to return a #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict()
def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) self.sentinels = set() self.cachepath = os.path.join(self.data, 'cache') self.keypath = os.path.join(self.data, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.core.setup)
def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) self.sentinels = set() self.cachepath = self.core.setup.cfp.get("packages", "cache", default=os.path.join(self.data, "cache")) self.keypath = self.core.setup.cfp.get("packages", "keycache", default=os.path.join(self.data, "keys")) if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) self.sources = PackagesSources( os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.core.setup )
class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.ClientRunHooks): """ Packages resolves Package entries on the Bcfg2 server in order to present a complete list of Package entries to the client in order to determine the completeness of the client configuration. It does so by delegating control of package version information to a number of backends, which may parse repository metadata directly or defer to package manager libraries for truly dynamic resolution. .. private-include: _build_packages""" #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] #: Packages exposes two additional XML-RPC calls, :func:`Refresh` #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. self.cachepath = \ self.core.setup.cfp.get("packages", "cache", default=os.path.join(self.data, 'cache')) #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) # warn about deprecated magic groups if self.core.setup.cfp.getboolean("packages", "magic_groups", default=False): self.logger.warning("Packages: Magic groups are deprecated and " "will be removed in a future release") self.logger.warning("You can disable magic groups by setting " "magic_groups=0 in [packages] in bcfg2.conf") # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` #: object used to generate #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects for #: this plugin. self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.core.setup) #: We cache #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: objects in ``collections`` so that calling :func:`Refresh` #: or :func:`Reload` can tell the collection objects to clean #: up their cache, but we don't actually use the cache to #: return a ``Collection`` object when one is requested, #: because that prevents new machines from working, since a #: ``Collection`` object gets created by #: :func:`get_additional_data`, which is called for all #: clients at server startup and various other times. (It #: would also prevent machines that change groups from working #: properly; e.g., if you reinstall a machine with a new OS, #: then returning a cached ``Collection`` object would give #: the wrong sources to that client.) These are keyed by the #: collection #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. self.collections = dict() #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` #: Unlike :attr:`collections`, this _is_ used to return a #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict() # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def toggle_debug(self): rv = Bcfg2.Server.Plugin.Plugin.toggle_debug(self) self.sources.toggle_debug() for collection in self.collections.values(): collection.toggle_debug() return rv toggle_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.toggle_debug.__doc__ @property def disableResolver(self): """ Report the state of the resolver. This can be disabled in the configuration. Note that disabling metadata (see :attr:`disableMetaData`) implies disabling the resolver. This property cannot be set. """ if self.disableMetaData: # disabling metadata without disabling the resolver Breaks # Things return True try: return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled", which are not handled according to the # Python docs but appear to be handled properly by # ConfigParser in at least some versions return self.core.setup.cfp.get( "packages", "resolver", default="enabled").lower() == "disabled" @property def disableMetaData(self): """ Report whether or not metadata processing is enabled. This property cannot be set. """ try: return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled" return self.core.setup.cfp.get( "packages", "metadata", default="enabled").lower() == "disabled" def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. :param entry: The base entry to bind. This will be modified in place. :type entry: lxml.etree._Element :param metadata: The client to create the config for. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ attrib = dict(encoding='ascii', owner='root', group='root', type='file', perms='0644', important='true') collection = self.get_collection(metadata) entry.text = collection.get_config() for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) def HandleEntry(self, entry, metadata): """ Bind configuration entries. ``HandleEntry`` handles entries two different ways: * All ``Package`` entries have their ``version`` and ``type`` attributes set according to the appropriate :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object for this client. * ``Path`` entries are delegated to :func:`create_config` :param entry: The entry to bind :type entry: lxml.etree._Element :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: lxml.etree._Element - The fully bound entry """ if entry.tag == 'Package': collection = self.get_collection(metadata) entry.set( 'version', self.core.setup.cfp.get("packages", "version", default="auto")) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) return entry def HandlesEntry(self, entry, metadata): """ Determine if the given entry can be handled. Packages handles two kinds of entries: * ``Package`` entries are handled if the client has any sources at all. * ``Path`` entries are handled if they match the paths that are handled by a backend that can produce client configurations, e.g., :attr:`YUM_CONFIG_DEFAULT`, :attr:`APT_CONFIG_DEFAULT`, or the overridden value of either of those from the configuration. :param entry: The entry to bind :type entry: lxml.etree._Element :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: bool - Whether or not this plugin can handle the entry :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ if entry.tag == 'Package': if self.core.setup.cfp.getboolean("packages", "magic_groups", default=False): collection = self.get_collection(metadata) if collection.magic_groups_match(): return True else: return True elif entry.tag == 'Path': # managed entries for yum/apt configs if (entry.get("name") == \ self.core.setup.cfp.get("packages", "yum_config", default=YUM_CONFIG_DEFAULT) or entry.get("name") == \ self.core.setup.cfp.get("packages", "apt_config", default=APT_CONFIG_DEFAULT)): return True return False @Bcfg2.Server.Plugin.track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: #. Given the full list of all packages that apply to this client from the specification, calls :func:`_build_packages` to resolve dependencies, determine unknown packages (i.e., those that are not in any repository that applies to this client), and build a complete package list. #. Calls :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.build_extra_structures` to add any other extra data required by the backend (e.g., GPG keys) :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param structures: A list of lxml.etree._Element objects describing the structures (i.e., bundles) for this client. This can be modified in place. :type structures: list of lxml.etree._Element objects :returns: None """ collection = self.get_collection(metadata) indep = lxml.etree.Element('Independent') self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) @Bcfg2.Server.Plugin.track_statistics() def _build_packages(self, metadata, independent, structures, collection=None): """ Perform dependency resolution and build the complete list of packages that need to be included in the specification by :func:`validate_structures`, based on the initial list of packages. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param independent: The XML tag to add package entries generated by dependency resolution to. This will be modified in place. :type independent: lxml.etree._Element :param structures: A list of lxml.etree._Element objects describing the structures (i.e., bundles) for this client :type structures: list of lxml.etree._Element objects :param collection: The collection of sources for this client. If none is given, one will be created with :func:`get_collection` :type collection: Bcfg2.Server.Plugins.Packages.Collection.Collection """ if self.disableResolver: # Config requests no resolver return if collection is None: collection = self.get_collection(metadata) # initial is the set of packages that are explicitly specified # in the configuration initial = set() # base is the set of initial packages with groups expanded base = set() # essential pkgs are those marked as such by the distribution essential = collection.get_essential() to_remove = [] groups = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) to_remove.append(pkg) else: self.logger.error( "Packages: Malformed Package: %s" % lxml.etree.tostring( pkg, xml_declaration=False).decode('UTF-8')) gpkgs = collection.get_groups(groups) for pkgs in gpkgs.values(): base.update(pkgs) base.update(initial | essential) for el in to_remove: el.getparent().remove(el) packages, unknown = collection.complete(base) if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d initial, %d complete, %d new" % (len(initial), len(packages), len(newpkgs))) newpkgs.sort() collection.packages_to_entry(newpkgs, independent) @Bcfg2.Server.Plugin.track_statistics() def Refresh(self): """ Packages.Refresh() => True|False Reload configuration specification and download sources """ self._load_config(force_update=True) return True @Bcfg2.Server.Plugin.track_statistics() def Reload(self): """ Packages.Refresh() => True|False Reload configuration specification and sources """ self._load_config() return True def _load_config(self, force_update=False): """ Load the configuration data and setup sources :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ self._load_sources(force_update) self._load_gpg_keys(force_update) def _load_sources(self, force_update): """ Load sources from the config, downloading if necessary. :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ cachefiles = set() for collection in list(self.collections.values()): cachefiles.update(collection.cachefiles) if not self.disableMetaData: collection.setup_data(force_update) # clear Collection caches self.clients = dict() self.collections = dict() for source in self.sources.entries: cachefiles.add(source.cachefile) if not self.disableMetaData: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): if cfile not in cachefiles: try: if os.path.isdir(cfile): shutil.rmtree(cfile) else: os.unlink(cfile) except OSError: err = sys.exc_info()[1] self.logger.error("Packages: Could not remove cache file " "%s: %s" % (cfile, err)) def _load_gpg_keys(self, force_update): """ Load GPG keys from the config, downloading if necessary. :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ keyfiles = [] keys = [] for source in self.sources.entries: for key in source.gpgkeys: localfile = os.path.join(self.keypath, os.path.basename(key.rstrip("/"))) if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) response = urlopen(key) open(localfile, 'w').write(response.read()) keys.append(key) for kfile in glob.glob(os.path.join(self.keypath, "*")): if kfile not in keyfiles: os.unlink(kfile) @Bcfg2.Server.Plugin.track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object for this client. :param metadata: The client metadata to get a Collection for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: An instance of the appropriate subclass of :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` that contains all relevant sources that apply to the given client """ if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object return Collection(metadata, [], self.cachepath, self.data, self.core.fam) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] sclasses = set() relevant = list() for source in self.sources.entries: if source.applies(metadata): relevant.append(source) sclasses.update([source.__class__]) if len(sclasses) > 1: self.logger.warning("Packages: Multiple source types found for " "%s: %s" % ",".join([s.__name__ for s in sclasses])) cclass = Collection elif len(sclasses) == 0: self.logger.error("Packages: No sources found for %s" % metadata.hostname) cclass = Collection else: cclass = get_collection_class(sclasses.pop().__name__.replace( "Source", "")) if self.debug_flag: self.logger.error("Packages: Using %s for Collection of sources " "for %s" % (cclass.__name__, metadata.hostname)) collection = cclass(metadata, relevant, self.cachepath, self.data, self.core.fam, debug=self.debug_flag) ckey = collection.cachekey self.clients[metadata.hostname] = ckey self.collections[ckey] = collection return collection def get_additional_data(self, metadata): """ Return additional data for the given client. This will be a dict containing a single key, ``sources``, whose value is a list of data returned from :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, namely, a list of :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` data. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ collection = self.get_collection(metadata) return dict(sources=collection.get_additional_data()) def end_client_run(self, metadata): """ Hook to clear the cache for this client in :attr:`clients`, which must persist only the duration of a client run. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ if metadata.hostname in self.clients: del self.clients[metadata.hostname] def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` once statistics are processed to ensure that a stray cached :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object is not built during statistics and preserved until a subsequent client run. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ self.end_client_run(metadata)
class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.ClientRunHooks): """ Packages resolves Package entries on the Bcfg2 server in order to present a complete list of Package entries to the client in order to determine the completeness of the client configuration. It does so by delegating control of package version information to a number of backends, which may parse repository metadata directly or defer to package manager libraries for truly dynamic resolution. .. private-include: _build_packages""" options = [ Bcfg2.Options.Option(cf=("packages", "backends"), dest="packages_backends", help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac']), Bcfg2.Options.PathOption(cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", default='<repository>/Packages/cache'), Bcfg2.Options.Option(cf=("packages", "resolver"), dest="packages_resolver", help="Disable the Packages resolver", type=packages_boolean, default=True), Bcfg2.Options.Option(cf=("packages", "metadata"), dest="packages_metadata", help="Disable all Packages metadata processing", type=packages_boolean, default=True), Bcfg2.Options.Option(cf=("packages", "version"), dest="packages_version", help="Set default Package entry version", default="auto", choices=["auto", "any"]), Bcfg2.Options.PathOption( cf=("packages", "yum_config"), help="The default path for generated yum configs", default="/etc/yum.repos.d/bcfg2.repo"), Bcfg2.Options.PathOption( cf=("packages", "apt_config"), help="The default path for generated apt configs", default= "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list") ] #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] #: Packages exposes two additional XML-RPC calls, :func:`Refresh` #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core): Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. self.cachepath = Bcfg2.Options.setup.packages_cache #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` #: object used to generate #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects for #: this plugin. self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, self) #: We cache #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: objects in ``collections`` so that calling :func:`Refresh` #: or :func:`Reload` can tell the collection objects to clean #: up their cache, but we don't actually use the cache to #: return a ``Collection`` object when one is requested, #: because that prevents new machines from working, since a #: ``Collection`` object gets created by #: :func:`get_additional_data`, which is called for all #: clients at server startup and various other times. (It #: would also prevent machines that change groups from working #: properly; e.g., if you reinstall a machine with a new OS, #: then returning a cached ``Collection`` object would give #: the wrong sources to that client.) These are keyed by the #: collection #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. self.collections = Bcfg2.Server.Cache.Cache("Packages", "collections") #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` #: Unlike :attr:`collections`, this _is_ used to return a #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = Bcfg2.Server.Cache.Cache("Packages", "cache") # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def set_debug(self, debug): rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) self.sources.set_debug(debug) for collection in self.collections.values(): collection.set_debug(debug) return rv set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. :param entry: The base entry to bind. This will be modified in place. :type entry: lxml.etree._Element :param metadata: The client to create the config for. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ attrib = dict(encoding='ascii', owner='root', group='root', type='file', mode='0644', important='true') collection = self.get_collection(metadata) entry.text = collection.get_config() for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) def get_config(self, metadata): """ Get yum/apt config, as a string, for the specified client. :param metadata: The client to create the config for. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ return self.get_collection(metadata).get_config() def HandleEntry(self, entry, metadata): """ Bind configuration entries. ``HandleEntry`` handles entries two different ways: * All ``Package`` entries have their ``version`` and ``type`` attributes set according to the appropriate :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object for this client. * ``Path`` entries are delegated to :func:`create_config` :param entry: The entry to bind :type entry: lxml.etree._Element :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: lxml.etree._Element - The fully bound entry """ if entry.tag == 'Package': collection = self.get_collection(metadata) entry.set('version', Bcfg2.Options.setup.packages_version) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) return entry def HandlesEntry(self, entry, metadata): """ Determine if the given entry can be handled. Packages handles two kinds of entries: * ``Package`` entries are handled if the client has any sources at all. * ``Path`` entries are handled if they match the paths that are handled by a backend that can produce client configurations, e.g., :attr:`YUM_CONFIG_DEFAULT`, :attr:`APT_CONFIG_DEFAULT`, or the overridden value of either of those from the configuration. :param entry: The entry to bind :type entry: lxml.etree._Element :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: bool - Whether or not this plugin can handle the entry :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ if entry.tag == 'Package': return True elif entry.tag == 'Path': # managed entries for yum/apt configs if entry.get("name") in [ Bcfg2.Options.setup.apt_config, Bcfg2.Options.setup.yum_config ]: return True return False @track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: #. Given the full list of all packages that apply to this client from the specification, calls :func:`_build_packages` to resolve dependencies, determine unknown packages (i.e., those that are not in any repository that applies to this client), and build a complete package list. #. Calls :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.build_extra_structures` to add any other extra data required by the backend (e.g., GPG keys) :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param structures: A list of lxml.etree._Element objects describing the structures (i.e., bundles) for this client. This can be modified in place. :type structures: list of lxml.etree._Element objects :returns: None """ collection = self.get_collection(metadata) indep = lxml.etree.Element('Independent', name=self.__class__.__name__) self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) @track_statistics() def _build_packages( self, metadata, independent, # pylint: disable=R0914 structures, collection=None): """ Perform dependency resolution and build the complete list of packages that need to be included in the specification by :func:`validate_structures`, based on the initial list of packages. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param independent: The XML tag to add package entries generated by dependency resolution to. This will be modified in place. :type independent: lxml.etree._Element :param structures: A list of lxml.etree._Element objects describing the structures (i.e., bundles) for this client :type structures: list of lxml.etree._Element objects :param collection: The collection of sources for this client. If none is given, one will be created with :func:`get_collection` :type collection: Bcfg2.Server.Plugins.Packages.Collection.Collection """ if (not Bcfg2.Options.setup.packages_metadata or not Bcfg2.Options.setup.packages_resolver): # Config requests no resolver. Note that disabling # metadata implies disabling the resolver. for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("group"): if pkg.get("type"): pkg.set("choose", pkg.get("type")) return if collection is None: collection = self.get_collection(metadata) initial = set() to_remove = [] groups = [] recommended = dict() for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) if pkg.get("recommended"): recommended[pkg.get("name")] = pkg.get("recommended") elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) to_remove.append(pkg) else: self.logger.error( "Packages: Malformed Package: %s" % lxml.etree.tostring( pkg, xml_declaration=False).decode('UTF-8')) # base is the set of initial packages explicitly given in the # specification, packages from expanded package groups, and # packages essential to the distribution base = set(initial) # remove package groups for el in to_remove: el.getparent().remove(el) groups.sort() # check for this set of groups in the group cache gcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_groups", collection.cachekey) gkey = hash(tuple(groups)) if gkey not in gcache: gcache[gkey] = collection.get_groups(groups) for pkgs in gcache[gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution base.update(collection.get_essential()) # check for this set of packages in the package cache pkey = hash(tuple(base)) pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", collection.cachekey) if pkey not in pcache: pcache[pkey] = collection.complete(base, recommended) packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d base, %d complete, %d new" % (len(base), len(packages), len(newpkgs))) newpkgs.sort() collection.packages_to_entry(newpkgs, independent) @track_statistics() def Refresh(self): """ Packages.Refresh() => True|False Reload configuration specification and download sources """ self._load_config(force_update=True) return True @track_statistics() def Reload(self): """ Packages.Refresh() => True|False Reload configuration specification and sources """ self._load_config() return True def child_reload(self, _=None): """ Reload the Packages configuration on a child process. """ self.Reload() def _load_config(self, force_update=False): """ Load the configuration data and setup sources :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ self._load_sources(force_update) self._load_gpg_keys(force_update) def _load_sources(self, force_update): """ Load sources from the config, downloading if necessary. :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ cachefiles = set() for collection in list(self.collections.values()): cachefiles.update(collection.cachefiles) if Bcfg2.Options.setup.packages_metadata: collection.setup_data(force_update) # clear Collection and package caches Bcfg2.Server.Cache.expire("Packages") for source in self.sources.entries: cachefiles.add(source.cachefile) if Bcfg2.Options.setup.packages_metadata: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): if cfile not in cachefiles: try: if os.path.isdir(cfile): shutil.rmtree(cfile) else: os.unlink(cfile) except OSError: err = sys.exc_info()[1] self.logger.error("Packages: Could not remove cache file " "%s: %s" % (cfile, err)) def _load_gpg_keys(self, force_update): """ Load GPG keys from the config, downloading if necessary. :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ keyfiles = [] keys = [] for source in self.sources.entries: for key in source.gpgkeys: localfile = os.path.join(self.keypath, os.path.basename(key.rstrip("/"))) if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) try: open(localfile, 'w').write(urlopen(key).read()) keys.append(key) except (URLError, HTTPError): err = sys.exc_info()[1] self.logger.error( "Packages: Error downloading %s: %s" % (key, err)) except IOError: err = sys.exc_info()[1] self.logger.error("Packages: Error writing %s to %s: " "%s" % (key, localfile, err)) except: err = sys.exc_info()[1] self.logger.error("Packages: Unknown error fetching " "%s: %s" % (key, err)) for kfile in glob.glob(os.path.join(self.keypath, "*")): if kfile not in keyfiles: os.unlink(kfile) @track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object for this client. :param metadata: The client metadata to get a Collection for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: An instance of the appropriate subclass of :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` that contains all relevant sources that apply to the given client """ if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object return Collection(metadata, [], self.cachepath, self.data) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] sclasses = set() relevant = list() for source in self.sources.entries: if source.applies(metadata): relevant.append(source) sclasses.update([source.__class__]) if len(sclasses) > 1: self.logger.warning( "Packages: Multiple source types found for " "%s: %s" % (metadata.hostname, ",".join([s.__name__ for s in sclasses]))) cclass = Collection elif len(sclasses) == 0: self.logger.error("Packages: No sources found for %s" % metadata.hostname) cclass = Collection else: cclass = get_collection_class(sclasses.pop().__name__.replace( "Source", "")) if self.debug_flag: self.logger.error("Packages: Using %s for Collection of sources " "for %s" % (cclass.__name__, metadata.hostname)) collection = cclass(metadata, relevant, self.cachepath, self.data, debug=self.debug_flag) ckey = collection.cachekey if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection return collection def get_additional_data(self, metadata): """ Return additional data for the given client. This will be an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict` containing two keys: * ``sources``, whose value is a list of data returned from :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, namely, a list of :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` data; and * ``get_config``, whose value is the :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` function, which can be used to get the Packages config for other systems. This uses an OnDemandDict instead of just a normal dict because loading a source collection can be a fairly time-consuming process, particularly for the first time. As a result, when all metadata objects are built at once (such as after the server is restarted, or far more frequently if Metadata caching is disabled), this function would be a major bottleneck if we tried to build all collections at the same time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ def get_sources(): """ getter for the 'sources' key of the OnDemandDict returned by this function. This delays calling get_collection() until it's absolutely necessary. """ return self.get_collection(metadata).get_additional_data() return OnDemandDict(sources=get_sources, get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in :attr:`clients`, which must persist only the duration of a client run. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ self.clients.expire(metadata.hostname) def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` once statistics are processed to ensure that a stray cached :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object is not built during statistics and preserved until a subsequent client run. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ self.end_client_run(metadata)
class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.GoalValidator): name = 'Packages' conflicts = ['Pkgmgr'] experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) self.sentinels = set() self.cachepath = os.path.join(self.data, 'cache') self.keypath = os.path.join(self.data, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.core.setup) def toggle_debug(self): Bcfg2.Server.Plugin.Plugin.toggle_debug(self) self.sources.toggle_debug() @property def disableResolver(self): if self.disableMetaData: # disabling metadata without disabling the resolver Breaks # Things return True try: return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled", which are not handled according to the # Python docs but appear to be handled properly by # ConfigParser in at least some versions return self.core.setup.cfp.get("packages", "resolver", default="enabled").lower() == "disabled" @property def disableMetaData(self): try: return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled" return self.core.setup.cfp.get("packages", "metadata", default="enabled").lower() == "disabled" def create_config(self, entry, metadata): """ create yum/apt config for the specified host """ attrib = dict(encoding='ascii', owner='root', group='root', type='file', perms='0644', important='true') collection = self._get_collection(metadata) entry.text = collection.get_config() for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) def HandleEntry(self, entry, metadata): if entry.tag == 'Package': collection = self._get_collection(metadata) entry.set('version', self.core.setup.cfp.get("packages", "version", default="auto")) entry.set('type', collection.ptype) elif entry.tag == 'Path': if (entry.get("name") == self.core.setup.cfp.get("packages", "yum_config", default="") or entry.get("name") == self.core.setup.cfp.get("packages", "apt_config", default="")): self.create_config(entry, metadata) def HandlesEntry(self, entry, metadata): if entry.tag == 'Package': if self.core.setup.cfp.getboolean("packages", "magic_groups", default=True): collection = self._get_collection(metadata) if collection.magic_groups_match(): return True else: return True elif entry.tag == 'Path': # managed entries for yum/apt configs if (entry.get("name") == self.core.setup.cfp.get("packages", "yum_config", default="") or entry.get("name") == self.core.setup.cfp.get("packages", "apt_config", default="")): return True return False def validate_structures(self, metadata, structures): '''Ensure client configurations include all needed prerequisites Arguments: metadata - client metadata instance structures - a list of structure-stage entry combinations ''' collection = self._get_collection(metadata) indep = lxml.etree.Element('Independent') self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) def _build_packages(self, metadata, independent, structures, collection=None): """ build list of packages that need to be included in the specification by validate_structures() """ if self.disableResolver: # Config requests no resolver return if collection is None: collection = self._get_collection(metadata) # initial is the set of packages that are explicitly specified # in the configuration initial = set() # base is the set of initial packages with groups expanded base = set() # essential pkgs are those marked as such by the distribution essential = collection.get_essential() to_remove = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.add(pkg.get("name")) elif pkg.get("group"): try: if pkg.get("type"): gpkgs = collection.get_group(pkg.get("group"), ptype=pkg.get("type")) else: gpkgs = collection.get_group(pkg.get("group")) base.update(gpkgs) except TypeError: raise self.logger.error("Could not resolve group %s" % pkg.get("group")) to_remove.append(pkg) else: self.logger.error("Packages: Malformed Package: %s" % lxml.etree.tostring(pkg)) base.update(initial | essential) for el in to_remove: el.getparent().remove(el) packages, unknown = collection.complete(base) if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) newpkgs = list(packages.difference(initial)) self.debug_log("Packages: %d initial, %d complete, %d new" % (len(initial), len(packages), len(newpkgs))) newpkgs.sort() for pkg in newpkgs: lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, version=self.core.setup.cfp.get("packages", "version", default="auto"), type=collection.ptype, origin='Packages') def Refresh(self): '''Packages.Refresh() => True|False\nReload configuration specification and download sources\n''' self._load_config(force_update=True) return True def Reload(self): '''Packages.Refresh() => True|False\nReload configuration specification and sources\n''' self._load_config() return True def _load_config(self, force_update=False): ''' Load the configuration data and setup sources Keyword args: force_update Force downloading repo data ''' self._load_sources(force_update) self._load_gpg_keys(force_update) def _load_sources(self, force_update): """ Load sources from the config """ self.sentinels = set() cachefiles = set() for collection in list(Collection.collections.values()): cachefiles.update(collection.cachefiles) if not self.disableMetaData: collection.setup_data(force_update) self.sentinels.update(collection.basegroups) Collection.clear_cache() for source in self.sources: cachefiles.add(source.cachefile) if not self.disableMetaData: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): if cfile not in cachefiles: try: if os.path.isdir(cfile): shutil.rmtree(cfile) else: os.unlink(cfile) except OSError: err = sys.exc_info()[1] self.logger.error("Packages: Could not remove cache file " "%s: %s" % (cfile, err)) def _load_gpg_keys(self, force_update): """ Load gpg keys from the config """ keyfiles = [] keys = [] for source in self.sources: for key in source.gpgkeys: localfile = os.path.join(self.keypath, os.path.basename(key.rstrip("/"))) if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) response = urlopen(key) open(localfile, 'w').write(response.read()) keys.append(key) for kfile in glob.glob(os.path.join(self.keypath, "*")): if kfile not in keyfiles: os.unlink(kfile) def _get_collection(self, metadata): return Collection.factory(metadata, self.sources, self.data, debug=self.debug_flag) def get_additional_data(self, metadata): collection = self._get_collection(metadata) return dict(sources=collection.get_additional_data()) def validate_goals(self, metadata, _): """ we abuse the GoalValidator plugin since validate_goals() is the very last thing called during a client config run. so we use this to clear the collection cache for this client, which must persist only the duration of a client run """ if metadata.hostname in Collection.clients: del Collection.clients[metadata.hostname]
class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.ClientRunHooks): """ Packages resolves Package entries on the Bcfg2 server in order to present a complete list of Package entries to the client in order to determine the completeness of the client configuration. It does so by delegating control of package version information to a number of backends, which may parse repository metadata directly or defer to package manager libraries for truly dynamic resolution. .. private-include: _build_packages""" options = [ Bcfg2.Options.Option( cf=("packages", "backends"), dest="packages_backends", help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac', 'Pkgng']), Bcfg2.Options.PathOption( cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", default='<repository>/Packages/cache'), Bcfg2.Options.Option( cf=("packages", "resolver"), dest="packages_resolver", help="Disable the Packages resolver", type=packages_boolean, default=True), Bcfg2.Options.Option( cf=("packages", "metadata"), dest="packages_metadata", help="Disable all Packages metadata processing", type=packages_boolean, default=True), Bcfg2.Options.Option( cf=("packages", "version"), dest="packages_version", help="Set default Package entry version", default="auto", choices=["auto", "any"]), Bcfg2.Options.PathOption( cf=("packages", "yum_config"), help="The default path for generated yum configs", default="/etc/yum.repos.d/bcfg2.repo"), Bcfg2.Options.PathOption( cf=("packages", "apt_config"), help="The default path for generated apt configs", default="/etc/apt/sources.list.d/" "bcfg2-packages-generated-sources.list")] #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] #: Packages exposes two additional XML-RPC calls, :func:`Refresh` #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core): Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. self.cachepath = Bcfg2.Options.setup.packages_cache #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` #: object used to generate #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects for #: this plugin. self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, self) #: We cache #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: objects in ``collections`` so that calling :func:`Refresh` #: or :func:`Reload` can tell the collection objects to clean #: up their cache, but we don't actually use the cache to #: return a ``Collection`` object when one is requested, #: because that prevents new machines from working, since a #: ``Collection`` object gets created by #: :func:`get_additional_data`, which is called for all #: clients at server startup and various other times. (It #: would also prevent machines that change groups from working #: properly; e.g., if you reinstall a machine with a new OS, #: then returning a cached ``Collection`` object would give #: the wrong sources to that client.) These are keyed by the #: collection #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. self.collections = Bcfg2.Server.Cache.Cache("Packages", "collections") #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` #: Unlike :attr:`collections`, this _is_ used to return a #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = Bcfg2.Server.Cache.Cache("Packages", "cache") # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def set_debug(self, debug): rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) self.sources.set_debug(debug) for collection in self.collections.values(): collection.set_debug(debug) return rv set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. :param entry: The base entry to bind. This will be modified in place. :type entry: lxml.etree._Element :param metadata: The client to create the config for. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ attrib = dict(encoding='ascii', owner='root', group='root', type='file', mode='0644', important='true') collection = self.get_collection(metadata) entry.text = collection.get_config() for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) def get_config(self, metadata): """ Get yum/apt config, as a string, for the specified client. :param metadata: The client to create the config for. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ return self.get_collection(metadata).get_config() def HandleEntry(self, entry, metadata): """ Bind configuration entries. ``HandleEntry`` handles entries two different ways: * All ``Package`` entries have their ``version`` and ``type`` attributes set according to the appropriate :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object for this client. * ``Path`` entries are delegated to :func:`create_config` :param entry: The entry to bind :type entry: lxml.etree._Element :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: lxml.etree._Element - The fully bound entry """ if entry.tag == 'Package': collection = self.get_collection(metadata) entry.set('version', Bcfg2.Options.setup.packages_version) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) return entry def HandlesEntry(self, entry, metadata): """ Determine if the given entry can be handled. Packages handles two kinds of entries: * ``Package`` entries are handled if the client has any sources at all. * ``Path`` entries are handled if they match the paths that are handled by a backend that can produce client configurations, e.g., :attr:`YUM_CONFIG_DEFAULT`, :attr:`APT_CONFIG_DEFAULT`, or the overridden value of either of those from the configuration. :param entry: The entry to bind :type entry: lxml.etree._Element :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: bool - Whether or not this plugin can handle the entry :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ if entry.tag == 'Package': return True elif entry.tag == 'Path': # managed entries for yum/apt configs if entry.get("name") in [Bcfg2.Options.setup.apt_config, Bcfg2.Options.setup.yum_config]: return True return False @track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: #. Given the full list of all packages that apply to this client from the specification, calls :func:`_build_packages` to resolve dependencies, determine unknown packages (i.e., those that are not in any repository that applies to this client), and build a complete package list. #. Calls :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.build_extra_structures` to add any other extra data required by the backend (e.g., GPG keys) :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param structures: A list of lxml.etree._Element objects describing the structures (i.e., bundles) for this client. This can be modified in place. :type structures: list of lxml.etree._Element objects :returns: None """ collection = self.get_collection(metadata) indep = lxml.etree.Element('Independent', name=self.__class__.__name__) self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) @track_statistics() def _build_packages(self, metadata, independent, # pylint: disable=R0914 structures, collection=None): """ Perform dependency resolution and build the complete list of packages that need to be included in the specification by :func:`validate_structures`, based on the initial list of packages. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param independent: The XML tag to add package entries generated by dependency resolution to. This will be modified in place. :type independent: lxml.etree._Element :param structures: A list of lxml.etree._Element objects describing the structures (i.e., bundles) for this client :type structures: list of lxml.etree._Element objects :param collection: The collection of sources for this client. If none is given, one will be created with :func:`get_collection` :type collection: Bcfg2.Server.Plugins.Packages.Collection.Collection """ if (not Bcfg2.Options.setup.packages_metadata or not Bcfg2.Options.setup.packages_resolver): # Config requests no resolver. Note that disabling # metadata implies disabling the resolver. for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("group"): if pkg.get("type"): pkg.set("choose", pkg.get("type")) return if collection is None: collection = self.get_collection(metadata) initial = set() to_remove = [] groups = [] recommended = dict() for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) if pkg.get("recommended"): recommended[pkg.get("name")] = pkg.get("recommended") elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) to_remove.append(pkg) else: self.logger.error( "Packages: Malformed Package: %s" % lxml.etree.tostring( pkg, xml_declaration=False).decode('UTF-8')) # base is the set of initial packages explicitly given in the # specification, packages from expanded package groups, and # packages essential to the distribution base = set(initial) # remove package groups for el in to_remove: el.getparent().remove(el) groups.sort() # check for this set of groups in the group cache gcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_groups", collection.cachekey) gkey = hash(tuple(groups)) if gkey not in gcache: gcache[gkey] = collection.get_groups(groups) for pkgs in gcache[gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution base.update(collection.get_essential()) # check for this set of packages in the package cache pkey = hash(tuple(base)) pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", collection.cachekey) if pkey not in pcache: pcache[pkey] = collection.complete(base, recommended) packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d base, %d complete, %d new" % (len(base), len(packages), len(newpkgs))) newpkgs.sort() collection.packages_to_entry(newpkgs, independent) @track_statistics() def Refresh(self): """ Packages.Refresh() => True|False Reload configuration specification and download sources """ self._load_config(force_update=True) return True @track_statistics() def Reload(self): """ Packages.Refresh() => True|False Reload configuration specification and sources """ self._load_config() return True def child_reload(self, _=None): """ Reload the Packages configuration on a child process. """ self.Reload() def _load_config(self, force_update=False): """ Load the configuration data and setup sources :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ self._load_sources(force_update) self._load_gpg_keys(force_update) def _load_sources(self, force_update): """ Load sources from the config, downloading if necessary. :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ cachefiles = set() for collection in list(self.collections.values()): cachefiles.update(collection.cachefiles) if Bcfg2.Options.setup.packages_metadata: collection.setup_data(force_update) # clear Collection and package caches Bcfg2.Server.Cache.expire("Packages") for source in self.sources.entries: cachefiles.add(source.cachefile) if Bcfg2.Options.setup.packages_metadata: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): if cfile not in cachefiles: try: if os.path.isdir(cfile): shutil.rmtree(cfile) else: os.unlink(cfile) except OSError: err = sys.exc_info()[1] self.logger.error("Packages: Could not remove cache file " "%s: %s" % (cfile, err)) def _load_gpg_keys(self, force_update): """ Load GPG keys from the config, downloading if necessary. :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ keyfiles = [] keys = [] for source in self.sources.entries: for key in source.gpgkeys: localfile = os.path.join(self.keypath, os.path.basename(key.rstrip("/"))) if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) try: open(localfile, 'w').write(urlopen(key).read()) keys.append(key) except (URLError, HTTPError): err = sys.exc_info()[1] self.logger.error("Packages: Error downloading %s: %s" % (key, err)) except IOError: err = sys.exc_info()[1] self.logger.error("Packages: Error writing %s to %s: " "%s" % (key, localfile, err)) for kfile in glob.glob(os.path.join(self.keypath, "*")): if kfile not in keyfiles: os.unlink(kfile) @track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object for this client. :param metadata: The client metadata to get a Collection for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: An instance of the appropriate subclass of :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` that contains all relevant sources that apply to the given client """ if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object return Collection(metadata, [], self.cachepath, self.data) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] sclasses = set() relevant = list() for source in self.sources.entries: if source.applies(metadata): relevant.append(source) sclasses.update([source.__class__]) if len(sclasses) > 1: self.logger.warning("Packages: Multiple source types found for " "%s: %s" % (metadata.hostname, ",".join([s.__name__ for s in sclasses]))) cclass = Collection elif len(sclasses) == 0: self.logger.error("Packages: No sources found for %s" % metadata.hostname) cclass = Collection else: cclass = get_collection_class( sclasses.pop().__name__.replace("Source", "")) if self.debug_flag: self.logger.error("Packages: Using %s for Collection of sources " "for %s" % (cclass.__name__, metadata.hostname)) collection = cclass(metadata, relevant, self.cachepath, self.data, debug=self.debug_flag) ckey = collection.cachekey if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection return collection def get_additional_data(self, metadata): """ Return additional data for the given client. This will be an :class:`Bcfg2.Server.Plugin.OnDemandDict` containing two keys: * ``sources``, whose value is a list of data returned from :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, namely, a list of :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` data; and * ``get_config``, whose value is the :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` function, which can be used to get the Packages config for other systems. This uses an OnDemandDict instead of just a normal dict because loading a source collection can be a fairly time-consuming process, particularly for the first time. As a result, when all metadata objects are built at once (such as after the server is restarted, or far more frequently if Metadata caching is disabled), this function would be a major bottleneck if we tried to build all collections at the same time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ def get_sources(): """ getter for the 'sources' key of the OnDemandDict returned by this function. This delays calling get_collection() until it's absolutely necessary. """ return self.get_collection(metadata).get_additional_data() return Bcfg2.Server.Plugin.OnDemandDict( sources=get_sources, get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in :attr:`clients`, which must persist only the duration of a client run. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ self.clients.expire(metadata.hostname) def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` once statistics are processed to ensure that a stray cached :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object is not built during statistics and preserved until a subsequent client run. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ self.end_client_run(metadata)
class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector): name = 'Packages' conflicts = ['Pkgmgr'] experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) self.sentinels = set() self.cachepath = os.path.join(self.data, 'cache') self.keypath = os.path.join(self.data, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) # set up config files self.config = PackagesConfig(self) self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.config) def toggle_debug(self): Bcfg2.Server.Plugin.Plugin.toggle_debug(self) self.sources.toggle_debug() @property def disableResolver(self): try: return not self.config.getboolean("global", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled", which are not handled according to the # Python docs but appear to be handled properly by # ConfigParser in at least some versions return self.config.get("global", "resolver", default="enabled").lower() == "disabled" @property def disableMetaData(self): try: return not self.config.getboolean("global", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled" return self.config.get("global", "metadata", default="enabled").lower() == "disabled" def create_config(self, entry, metadata): """ create yum/apt config for the specified host """ attrib = { 'encoding': 'ascii', 'owner': 'root', 'group': 'root', 'type': 'file', 'perms': '0644' } collection = self._get_collection(metadata) entry.text = collection.get_config() for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) def HandleEntry(self, entry, metadata): if entry.tag == 'Package': collection = self._get_collection(metadata) entry.set('version', 'auto') entry.set('type', collection.ptype) elif entry.tag == 'Path': if (entry.get("name") == self.config.get( "global", "yum_config", default="") or entry.get("name") == self.config.get("global", "apt_config", default="")): self.create_config(entry, metadata) def HandlesEntry(self, entry, metadata): if entry.tag == 'Package': if self.config.getboolean("global", "magic_groups", default=True) == True: collection = self._get_collection(metadata) if collection.magic_groups_match(): return True else: return True elif entry.tag == 'Path': # managed entries for yum/apt configs if (entry.get("name") == self.config.get( "global", "yum_config", default="") or entry.get("name") == self.config.get("global", "apt_config", default="")): return True return False def validate_structures(self, metadata, structures): '''Ensure client configurations include all needed prerequisites Arguments: metadata - client metadata instance structures - a list of structure-stage entry combinations ''' collection = self._get_collection(metadata) indep = lxml.etree.Element('Independent') self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) def _build_packages(self, metadata, independent, structures, collection=None): """ build list of packages that need to be included in the specification by validate_structures() """ if self.disableResolver: # Config requests no resolver return if collection is None: collection = self._get_collection(metadata) # initial is the set of packages that are explicitly specified # in the configuration initial = set() # base is the set of initial packages with groups expanded base = set() to_remove = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.add(pkg.get("name")) elif pkg.get("group"): try: if pkg.get("type"): gpkgs = collection.get_group(pkg.get("group"), ptype=pkg.get("type")) else: gpkgs = collection.get_group(pkg.get("group")) base.update(gpkgs) except TypeError: raise self.logger.error("Could not resolve group %s" % pkg.get("group")) to_remove.append(pkg) else: self.logger.error("Packages: Malformed Package: %s" % lxml.etree.tostring(pkg)) base.update(initial) for el in to_remove: el.getparent().remove(el) packages, unknown = collection.complete(base) if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) newpkgs = list(packages.difference(initial)) self.debug_log("Packages: %d initial, %d complete, %d new" % (len(initial), len(packages), len(newpkgs))) newpkgs.sort() for pkg in newpkgs: lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, version='auto', type=collection.ptype, origin='Packages') def Refresh(self): '''Packages.Refresh() => True|False\nReload configuration specification and download sources\n''' self._load_config(force_update=True) return True def Reload(self): '''Packages.Refresh() => True|False\nReload configuration specification and sources\n''' self._load_config() return True def _load_config(self, force_update=False): ''' Load the configuration data and setup sources Keyword args: force_update Force downloading repo data ''' self._load_sources(force_update) self._load_gpg_keys(force_update) def _load_sources(self, force_update): """ Load sources from the config """ self.sentinels = set() cachefiles = set() for collection in list(Collection.collections.values()): cachefiles.update(collection.cachefiles) if not self.disableMetaData: collection.setup_data(force_update) self.sentinels.update(collection.basegroups) Collection.clear_cache() for source in self.sources: cachefiles.add(source.cachefile) if not self.disableMetaData: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): if cfile not in cachefiles: try: if os.path.isdir(cfile): shutil.rmtree(cfile) else: os.unlink(cfile) except OSError: err = sys.exc_info()[1] self.logger.error("Packages: Could not remove cache file " "%s: %s" % (cfile, err)) def _load_gpg_keys(self, force_update): """ Load gpg keys from the config """ keyfiles = [] keys = [] for source in self.sources: for key in source.gpgkeys: localfile = os.path.join(self.keypath, os.path.basename(key.rstrip("/"))) if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) response = urlopen(key) open(localfile, 'w').write(response.read()) keys.append(key) for kfile in glob.glob(os.path.join(self.keypath, "*")): if kfile not in keyfiles: os.unlink(kfile) def _get_collection(self, metadata): return Collection.factory(metadata, self.sources, self.data, debug=self.debug_flag) def get_additional_data(self, metadata): collection = self._get_collection(metadata) return dict(sources=collection.get_additional_data())
class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.ClientRunHooks): """ Packages resolves Package entries on the Bcfg2 server in order to present a complete list of Package entries to the client in order to determine the completeness of the client configuration. It does so by delegating control of package version information to a number of backends, which may parse repository metadata directly or defer to package manager libraries for truly dynamic resolution. .. private-include: _build_packages""" #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] #: Packages exposes two additional XML-RPC calls, :func:`Refresh` #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. self.cachepath = \ self.core.setup.cfp.get("packages", "cache", default=os.path.join(self.data, 'cache')) #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) # warn about deprecated magic groups if self.core.setup.cfp.getboolean("packages", "magic_groups", default=False): self.logger.warning("Packages: Magic groups are deprecated and " "will be removed in a future release") self.logger.warning("You can disable magic groups by setting " "magic_groups=0 in [packages] in bcfg2.conf") # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` #: object used to generate #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects for #: this plugin. self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.core.setup) #: We cache #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: objects in ``collections`` so that calling :func:`Refresh` #: or :func:`Reload` can tell the collection objects to clean #: up their cache, but we don't actually use the cache to #: return a ``Collection`` object when one is requested, #: because that prevents new machines from working, since a #: ``Collection`` object gets created by #: :func:`get_additional_data`, which is called for all #: clients at server startup and various other times. (It #: would also prevent machines that change groups from working #: properly; e.g., if you reinstall a machine with a new OS, #: then returning a cached ``Collection`` object would give #: the wrong sources to that client.) These are keyed by the #: collection #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. self.collections = dict() #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` #: Unlike :attr:`collections`, this _is_ used to return a #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict() # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def toggle_debug(self): rv = Bcfg2.Server.Plugin.Plugin.toggle_debug(self) self.sources.toggle_debug() for collection in self.collections.values(): collection.toggle_debug() return rv toggle_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.toggle_debug.__doc__ @property def disableResolver(self): """ Report the state of the resolver. This can be disabled in the configuration. Note that disabling metadata (see :attr:`disableMetaData`) implies disabling the resolver. This property cannot be set. """ if self.disableMetaData: # disabling metadata without disabling the resolver Breaks # Things return True try: return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled", which are not handled according to the # Python docs but appear to be handled properly by # ConfigParser in at least some versions return self.core.setup.cfp.get( "packages", "resolver", default="enabled").lower() == "disabled" @property def disableMetaData(self): """ Report whether or not metadata processing is enabled. This property cannot be set. """ try: return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled" return self.core.setup.cfp.get( "packages", "metadata", default="enabled").lower() == "disabled" def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. :param entry: The base entry to bind. This will be modified in place. :type entry: lxml.etree._Element :param metadata: The client to create the config for. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ attrib = dict(encoding='ascii', owner='root', group='root', type='file', mode='0644', important='true') collection = self.get_collection(metadata) entry.text = collection.get_config() for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) def HandleEntry(self, entry, metadata): """ Bind configuration entries. ``HandleEntry`` handles entries two different ways: * All ``Package`` entries have their ``version`` and ``type`` attributes set according to the appropriate :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object for this client. * ``Path`` entries are delegated to :func:`create_config` :param entry: The entry to bind :type entry: lxml.etree._Element :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: lxml.etree._Element - The fully bound entry """ if entry.tag == 'Package': collection = self.get_collection(metadata) entry.set('version', self.core.setup.cfp.get("packages", "version", default="auto")) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) return entry def HandlesEntry(self, entry, metadata): """ Determine if the given entry can be handled. Packages handles two kinds of entries: * ``Package`` entries are handled if the client has any sources at all. * ``Path`` entries are handled if they match the paths that are handled by a backend that can produce client configurations, e.g., :attr:`YUM_CONFIG_DEFAULT`, :attr:`APT_CONFIG_DEFAULT`, or the overridden value of either of those from the configuration. :param entry: The entry to bind :type entry: lxml.etree._Element :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: bool - Whether or not this plugin can handle the entry :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ if entry.tag == 'Package': if self.core.setup.cfp.getboolean("packages", "magic_groups", default=False): collection = self.get_collection(metadata) if collection.magic_groups_match(): return True else: return True elif entry.tag == 'Path': # managed entries for yum/apt configs if (entry.get("name") == \ self.core.setup.cfp.get("packages", "yum_config", default=YUM_CONFIG_DEFAULT) or entry.get("name") == \ self.core.setup.cfp.get("packages", "apt_config", default=APT_CONFIG_DEFAULT)): return True return False @Bcfg2.Server.Plugin.track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: #. Given the full list of all packages that apply to this client from the specification, calls :func:`_build_packages` to resolve dependencies, determine unknown packages (i.e., those that are not in any repository that applies to this client), and build a complete package list. #. Calls :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.build_extra_structures` to add any other extra data required by the backend (e.g., GPG keys) :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param structures: A list of lxml.etree._Element objects describing the structures (i.e., bundles) for this client. This can be modified in place. :type structures: list of lxml.etree._Element objects :returns: None """ collection = self.get_collection(metadata) indep = lxml.etree.Element('Independent') self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) @Bcfg2.Server.Plugin.track_statistics() def _build_packages(self, metadata, independent, structures, collection=None): """ Perform dependency resolution and build the complete list of packages that need to be included in the specification by :func:`validate_structures`, based on the initial list of packages. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :param independent: The XML tag to add package entries generated by dependency resolution to. This will be modified in place. :type independent: lxml.etree._Element :param structures: A list of lxml.etree._Element objects describing the structures (i.e., bundles) for this client :type structures: list of lxml.etree._Element objects :param collection: The collection of sources for this client. If none is given, one will be created with :func:`get_collection` :type collection: Bcfg2.Server.Plugins.Packages.Collection.Collection """ if self.disableResolver: # Config requests no resolver return if collection is None: collection = self.get_collection(metadata) # initial is the set of packages that are explicitly specified # in the configuration initial = set() # base is the set of initial packages with groups expanded base = set() # essential pkgs are those marked as such by the distribution essential = collection.get_essential() to_remove = [] groups = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) to_remove.append(pkg) else: self.logger.error( "Packages: Malformed Package: %s" % lxml.etree.tostring( pkg, xml_declaration=False).decode('UTF-8')) gpkgs = collection.get_groups(groups) for pkgs in gpkgs.values(): base.update(pkgs) base.update(initial | essential) for el in to_remove: el.getparent().remove(el) packages, unknown = collection.complete(base) if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d initial, %d complete, %d new" % (len(initial), len(packages), len(newpkgs))) newpkgs.sort() collection.packages_to_entry(newpkgs, independent) @Bcfg2.Server.Plugin.track_statistics() def Refresh(self): """ Packages.Refresh() => True|False Reload configuration specification and download sources """ self._load_config(force_update=True) return True @Bcfg2.Server.Plugin.track_statistics() def Reload(self): """ Packages.Refresh() => True|False Reload configuration specification and sources """ self._load_config() return True def _load_config(self, force_update=False): """ Load the configuration data and setup sources :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ self._load_sources(force_update) self._load_gpg_keys(force_update) def _load_sources(self, force_update): """ Load sources from the config, downloading if necessary. :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ cachefiles = set() for collection in list(self.collections.values()): cachefiles.update(collection.cachefiles) if not self.disableMetaData: collection.setup_data(force_update) # clear Collection caches self.clients = dict() self.collections = dict() for source in self.sources.entries: cachefiles.add(source.cachefile) if not self.disableMetaData: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): if cfile not in cachefiles: try: if os.path.isdir(cfile): shutil.rmtree(cfile) else: os.unlink(cfile) except OSError: err = sys.exc_info()[1] self.logger.error("Packages: Could not remove cache file " "%s: %s" % (cfile, err)) def _load_gpg_keys(self, force_update): """ Load GPG keys from the config, downloading if necessary. :param force_update: Ignore all locally cached and downloaded data and fetch the metadata anew from the upstream repository. :type force_update: bool """ keyfiles = [] keys = [] for source in self.sources.entries: for key in source.gpgkeys: localfile = os.path.join(self.keypath, os.path.basename(key.rstrip("/"))) if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) try: open(localfile, 'w').write(urlopen(key).read()) keys.append(key) except HTTPError: err = sys.exc_info()[1] self.logger.error("Packages: Error downloading %s: %s" % (key, err)) except IOError: err = sys.exc_info()[1] self.logger.error("Packages: Error writing %s to %s: " "%s" % (key, localfile, err)) except: err = sys.exc_info()[1] self.logger.error("Packages: Unknown error fetching " "%s: %s" % (key, err)) for kfile in glob.glob(os.path.join(self.keypath, "*")): if kfile not in keyfiles: os.unlink(kfile) @Bcfg2.Server.Plugin.track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object for this client. :param metadata: The client metadata to get a Collection for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: An instance of the appropriate subclass of :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` that contains all relevant sources that apply to the given client """ if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object return Collection(metadata, [], self.cachepath, self.data, self.core.fam) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] sclasses = set() relevant = list() for source in self.sources.entries: if source.applies(metadata): relevant.append(source) sclasses.update([source.__class__]) if len(sclasses) > 1: self.logger.warning("Packages: Multiple source types found for " "%s: %s" % ",".join([s.__name__ for s in sclasses])) cclass = Collection elif len(sclasses) == 0: self.logger.error("Packages: No sources found for %s" % metadata.hostname) cclass = Collection else: cclass = get_collection_class( sclasses.pop().__name__.replace("Source", "")) if self.debug_flag: self.logger.error("Packages: Using %s for Collection of sources " "for %s" % (cclass.__name__, metadata.hostname)) collection = cclass(metadata, relevant, self.cachepath, self.data, self.core.fam, debug=self.debug_flag) ckey = collection.cachekey self.clients[metadata.hostname] = ckey self.collections[ckey] = collection return collection def get_additional_data(self, metadata): """ Return additional data for the given client. This will be a dict containing a single key, ``sources``, whose value is a list of data returned from :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, namely, a list of :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` data. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ collection = self.get_collection(metadata) return dict(sources=collection.get_additional_data()) def end_client_run(self, metadata): """ Hook to clear the cache for this client in :attr:`clients`, which must persist only the duration of a client run. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ if metadata.hostname in self.clients: del self.clients[metadata.hostname] def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` once statistics are processed to ensure that a stray cached :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` object is not built during statistics and preserved until a subsequent client run. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ self.end_client_run(metadata)