def BuildStructures(self, metadata): """Build structures for client described by metadata.""" ret = lxml.etree.Element("Independent", version='2.0') fragments = reduce(lambda x, y: x + y, [base.Match(metadata) for base in list(self.entries.values())], []) [ret.append(copy.copy(frag)) for frag in fragments] return [ret]
def BuildStructures(self, metadata): """Build structures for client described by metadata.""" ret = lxml.etree.Element("Independent", version='2.0') fragments = reduce( lambda x, y: x + y, [base.Match(metadata) for base in list(self.entries.values())], []) [ret.append(copy.copy(frag)) for frag in fragments] return [ret]
def GetStructures(self, metadata): """Get all structures for client specified by metadata.""" structures = reduce(lambda x, y: x + y, [struct.BuildStructures(metadata) for struct in self.structures], []) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] missing = [b for b in metadata.bundles if b not in sbundles] if missing: self.logger.error("Client %s configuration missing bundles: %s" % (metadata.hostname, ':'.join(missing))) return structures
def GetStructures(self, metadata): """Get all structures for client specified by metadata.""" structures = reduce( lambda x, y: x + y, [struct.BuildStructures(metadata) for struct in self.structures], []) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] missing = [b for b in metadata.bundles if b not in sbundles] if missing: self.logger.error("Client %s configuration missing bundles: %s" % (metadata.hostname, ':'.join(missing))) return structures
def GetStructures(self, metadata): """ Get all structures (i.e., bundles) for the given client :param metadata: Client metadata to get structures for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: list of :class:`lxml.etree._Element` objects """ structures = reduce(lambda x, y: x + y, [struct.BuildStructures(metadata) for struct in self.structures], []) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] missing = [b for b in metadata.bundles if b not in sbundles] if missing: self.logger.error("Client %s configuration missing bundles: %s" % (metadata.hostname, ':'.join(missing))) return structures
class Inotify(Pseudo, pyinotify.ProcessEvent): """ File monitor backend with `inotify <http://inotify.aiken.cz/>`_ support. """ __rmi__ = Pseudo.__rmi__ + ["list_watches", "list_paths"] #: Inotify is the best FAM backend, so it gets a very high #: priority __priority__ = 99 # pylint: disable=E1101 #: Map pyinotify event constants to FAM :ref:`event codes #: <development-fam-event-codes>`. The mapping is not #: terrifically exact. action_map = {pyinotify.IN_CREATE: 'created', pyinotify.IN_DELETE: 'deleted', pyinotify.IN_MODIFY: 'changed', pyinotify.IN_MOVED_FROM: 'deleted', pyinotify.IN_MOVED_TO: 'created'} # pylint: enable=E1101 #: The pyinotify event mask. We only ask for events that are #: listed in :attr:`action_map` mask = reduce(lambda x, y: x | y, action_map.keys()) def __init__(self, ignore=None, debug=False): Pseudo.__init__(self, ignore=ignore, debug=debug) pyinotify.ProcessEvent.__init__(self) #: inotify can't set useful monitors directly on files, only #: on directories, so when a monitor is added on a file we add #: its parent directory to ``event_filter`` and then only #: produce events on a file in that directory if the file is #: listed in ``event_filter``. Keys are directories -- the #: parent directories of individual files that are monitored #: -- and values are lists of full paths to files in each #: directory that events *should* be produced for. An event #: on a file whose parent directory is in ``event_filter`` but #: which is not itself listed will be silently suppressed. self.event_filter = dict() #: inotify doesn't like monitoring a path twice, so we keep a #: dict of :class:`pyinotify.Watch` objects, keyed by monitor #: path, to avoid trying to create duplicate monitors. #: (Duplicates can happen if an object accidentally requests #: duplicate monitors, or if two files in a single directory #: are both individually monitored, since inotify can't set #: monitors on the files but only on the parent directories.) self.watches_by_path = dict() #: The :class:`pyinotify.ThreadedNotifier` object. This is #: created in :func:`start` after the server is done #: daemonizing. self.notifier = None #: The :class:`pyinotify.WatchManager` object. This is created #: in :func:`start` after the server is done daemonizing. self.watchmgr = None #: The queue used to record monitors that are added before #: :func:`start` has been called and :attr:`notifier` and #: :attr:`watchmgr` are created. self.add_q = [] def start(self): """ The inotify notifier and manager objects in :attr:`notifier` and :attr:`watchmgr` must be created by the daemonized process, so they are created in ``start()``. Before those objects are created, monitors are added to :attr:`add_q`, and are created once the :class:`pyinotify.ThreadedNotifier` and :class:`pyinotify.WatchManager` objects are created.""" Pseudo.start(self) self.watchmgr = pyinotify.WatchManager() self.notifier = pyinotify.ThreadedNotifier(self.watchmgr, self) self.notifier.start() for monitor in self.add_q: self.AddMonitor(*monitor) self.add_q = [] def fileno(self): if self.started: return self.watchmgr.get_fd() else: return None fileno.__doc__ = Pseudo.fileno.__doc__ def process_default(self, ievent): """ Process all inotify events received. This process a :class:`pyinotify._Event` object, creates a :class:`Bcfg2.Server.FileMonitor.Event` object from it, and adds that event to :attr:`events`. :param ievent: Event to be processed :type ievent: pyinotify._Event """ action = ievent.maskname for amask, aname in self.action_map.items(): if ievent.mask & amask: action = aname break else: # event action is not in the mask, and thus is not # something we care about self.debug_log("Ignoring event %s for %s" % (action, ievent.pathname)) return try: watch = self.watchmgr.watches[ievent.wd] except KeyError: self.logger.error("Error handling event %s for %s: " "Watch %s not found" % (action, ievent.pathname, ievent.wd)) return # FAM-style file monitors return the full path to the parent # directory that is being watched, relative paths to anything # contained within the directory. since we can't use inotify # to watch files directly, we have to sort of guess at whether # this watch was actually added on a file (and thus is in # self.event_filter because we're filtering out other events # on the directory) or was added directly on a directory. if (watch.path == ievent.pathname or ievent.wd in self.event_filter): path = ievent.pathname else: # relative path path = os.path.basename(ievent.pathname) # figure out the handleID. start with the path of the event; # that should catch events on files that are watched directly. # (we have to watch the directory that a file is in, so this # lets us handle events on different files in the same # directory -- and thus under the same watch -- with different # objects.) If the path to the event doesn't have a handler, # use the path of the watch itself. handleID = ievent.pathname if handleID not in self.handles: handleID = watch.path evt = Event(handleID, path, action) if (ievent.wd not in self.event_filter or ievent.pathname in self.event_filter[ievent.wd]): self.events.append(evt) def AddMonitor(self, path, obj, handleID=None): # strip trailing slashes path = path.rstrip("/") if not self.started: self.add_q.append((path, obj)) return path if not os.path.isdir(path): # inotify is a little wonky about watching files. for # instance, if you watch /tmp/foo, and then do 'mv # /tmp/bar /tmp/foo', it processes that as a deletion of # /tmp/foo (which it technically _is_, but that's rather # useless -- we care that /tmp/foo changed, not that it # was first deleted and then created). In order to # effectively watch a file, we have to watch the directory # it's in, and filter out events for other files in the # same directory that are not similarly watched. # watch_transient_file requires a Processor _class_, not # an object, so we can't have this object handle events, # which is Wrong, so we can't use that function. watch_path = os.path.dirname(path) is_dir = False else: watch_path = path is_dir = True # see if this path is already being watched try: watchdir = self.watches_by_path[watch_path] except KeyError: if not os.path.exists(watch_path): raise OSError(errno.ENOENT, "No such file or directory: '%s'" % path) watchdir = self.watchmgr.add_watch(watch_path, self.mask, quiet=False)[watch_path] self.watches_by_path[watch_path] = watchdir produce_exists = True if not is_dir: if watchdir not in self.event_filter: self.event_filter[watchdir] = [path] elif path not in self.event_filter[watchdir]: self.event_filter[watchdir].append(path) else: # we've been asked to watch a file that we're already # watching, so we don't need to produce 'exists' # events produce_exists = False # inotify doesn't produce initial 'exists' events, so we # inherit from Pseudo to produce those if produce_exists: return Pseudo.AddMonitor(self, path, obj, handleID=path) else: self.handles[path] = obj return path AddMonitor.__doc__ = Pseudo.AddMonitor.__doc__ def shutdown(self): if self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ def list_watches(self): """ XML-RPC that returns a list of current inotify watches for debugging purposes. """ return list(self.watches_by_path.keys()) def list_paths(self): """ XML-RPC that returns a list of paths that are handled for debugging purposes. Because inotify doesn't like watching files, but prefers to watch directories, this will be different from :func:`Bcfg2.Server.FileMonitor.Inotify.Inotify.ListWatches`. For instance, if a plugin adds a monitor to ``/var/lib/bcfg2/Plugin/foo.xml``, :func:`ListPaths` will return ``/var/lib/bcfg2/Plugin/foo.xml``, while :func:`ListWatches` will return ``/var/lib/bcfg2/Plugin``. """ return list(self.handles.keys())
def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: # if no metadata is registered yet, defer if len(self.core.metadata.query.all()) == 0: self.__skn = False return self.__skn skn = [s.data.rstrip() for s in list(self.static.values())] mquery = self.core.metadata.query # build hostname cache names = dict() for cmeta in mquery.all(): names[cmeta.hostname] = set([cmeta.hostname]) names[cmeta.hostname].update(cmeta.aliases) newnames = set() newips = set() for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: newips.add(self.get_ipcache_entry(name)[0]) except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) names[cmeta.hostname].update(cmeta.addresses) names[cmeta.hostname].update(newips) # TODO: Only perform reverse lookups on IPs if an # option is set. if True: for ip in newips: try: names[cmeta.hostname].update( self.get_namecache_entry(ip)) except: # pylint: disable=W0702 continue names[cmeta.hostname] = sorted(names[cmeta.hostname]) pubkeys = [pubk for pubk in list(self.entries.keys()) if pubk.endswith('.pub')] pubkeys.sort() for pubkey in pubkeys: for entry in sorted(self.entries[pubkey].entries.values(), key=lambda e: (e.specific.hostname or e.specific.group)): specific = entry.specific hostnames = [] if specific.hostname and specific.hostname in names: hostnames = names[specific.hostname] elif specific.group: hostnames = \ reduce(lambda x, y: x + y, [names[cmeta.hostname] for cmeta in \ mquery.by_groups([specific.group])], []) elif specific.all: # a generic key for all hosts? really? hostnames = reduce(lambda x, y: x + y, list(names.values()), []) if not hostnames: if specific.hostname: key = specific.hostname ktype = "host" elif specific.group: key = specific.group ktype = "group" else: # user has added a global SSH key, but # have no clients yet. don't warn about # this. continue if key not in self.badnames: self.badnames[key] = True self.logger.info("Ignoring key for unknown %s %s" % (ktype, key)) continue skn.append("%s %s" % (','.join(hostnames), entry.data.rstrip())) self.__skn = "\n".join(skn) + "\n" return self.__skn
class Inotify(Pseudo, pyinotify.ProcessEvent): """ file monitor with inotify support """ __priority__ = 1 # pylint: disable=E1101 action_map = { pyinotify.IN_CREATE: 'created', pyinotify.IN_DELETE: 'deleted', pyinotify.IN_MODIFY: 'changed', pyinotify.IN_MOVED_FROM: 'deleted', pyinotify.IN_MOVED_TO: 'created' } # pylint: enable=E1101 mask = reduce(lambda x, y: x | y, action_map.keys()) def __init__(self, ignore=None, debug=False): Pseudo.__init__(self, ignore=ignore, debug=debug) pyinotify.ProcessEvent.__init__(self) self.event_filter = dict() self.watches_by_path = dict() # these are created in start() after the server is done forking self.notifier = None self.watchmgr = None self.add_q = [] def start(self): Pseudo.start(self) self.watchmgr = pyinotify.WatchManager() self.notifier = pyinotify.ThreadedNotifier(self.watchmgr, self) self.notifier.start() for monitor in self.add_q: self.AddMonitor(*monitor) self.add_q = [] def fileno(self): if self.started: return self.watchmgr.get_fd() else: return None def process_default(self, ievent): action = ievent.maskname for amask, aname in self.action_map.items(): if ievent.mask & amask: action = aname break try: watch = self.watchmgr.watches[ievent.wd] except KeyError: LOGGER.error("Error handling event for %s: Watch %s not found" % (ievent.pathname, ievent.wd)) return # FAM-style file monitors return the full path to the parent # directory that is being watched, relative paths to anything # contained within the directory. since we can't use inotify # to watch files directly, we have to sort of guess at whether # this watch was actually added on a file (and thus is in # self.event_filter because we're filtering out other events # on the directory) or was added directly on a directory. if (watch.path == ievent.pathname or ievent.wd in self.event_filter): path = ievent.pathname else: # relative path path = os.path.basename(ievent.pathname) # figure out the handleID. start with the path of the event; # that should catch events on files that are watched directly. # (we have to watch the directory that a file is in, so this # lets us handle events on different files in the same # directory -- and thus under the same watch -- with different # objects.) If the path to the event doesn't have a handler, # use the path of the watch itself. handleID = ievent.pathname if handleID not in self.handles: handleID = watch.path evt = Event(handleID, path, action) if (ievent.wd not in self.event_filter or ievent.pathname in self.event_filter[ievent.wd]): self.events.append(evt) def AddMonitor(self, path, obj, handleID=None): # strip trailing slashes path = path.rstrip("/") if not self.started: self.add_q.append((path, obj)) return path if not os.path.isdir(path): # inotify is a little wonky about watching files. for # instance, if you watch /tmp/foo, and then do 'mv # /tmp/bar /tmp/foo', it processes that as a deletion of # /tmp/foo (which it technically _is_, but that's rather # useless -- we care that /tmp/foo changed, not that it # was first deleted and then created). In order to # effectively watch a file, we have to watch the directory # it's in, and filter out events for other files in the # same directory that are not similarly watched. # watch_transient_file requires a Processor _class_, not # an object, so we can't have this object handle events, # which is Wrong, so we can't use that function. watch_path = os.path.dirname(path) is_dir = False else: watch_path = path is_dir = True # see if this path is already being watched try: watchdir = self.watches_by_path[watch_path] except KeyError: watchdir = self.watchmgr.add_watch(watch_path, self.mask, quiet=False)[watch_path] self.watches_by_path[watch_path] = watchdir produce_exists = True if not is_dir: if watchdir not in self.event_filter: self.event_filter[watchdir] = [path] elif path not in self.event_filter[watchdir]: self.event_filter[watchdir].append(path) else: # we've been asked to watch a file that we're already # watching, so we don't need to produce 'exists' # events produce_exists = False # inotify doesn't produce initial 'exists' events, so we # inherit from Pseudo to produce those if produce_exists: return Pseudo.AddMonitor(self, path, obj, handleID=path) else: self.handles[path] = obj return path def shutdown(self): if self.notifier: self.notifier.stop()
def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: # if no metadata is registered yet, defer if len(self.core.metadata.query.all()) == 0: self.__skn = False return self.__skn skn = [s.data.rstrip() for s in list(self.static.values())] mquery = self.core.metadata.query # build hostname cache names = dict() for cmeta in mquery.all(): names[cmeta.hostname] = set([cmeta.hostname]) names[cmeta.hostname].update(cmeta.aliases) newnames = set() newips = set() for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: newips.add(self.get_ipcache_entry(name)[0]) except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) names[cmeta.hostname].update(cmeta.addresses) names[cmeta.hostname].update(newips) # TODO: Only perform reverse lookups on IPs if an # option is set. if True: for ip in newips: try: names[cmeta.hostname].update( self.get_namecache_entry(ip)) except: # pylint: disable=W0702 continue names[cmeta.hostname] = sorted(names[cmeta.hostname]) pubkeys = [ pubk for pubk in list(self.entries.keys()) if pubk.endswith('.pub') ] pubkeys.sort() for pubkey in pubkeys: for entry in sorted(self.entries[pubkey].entries.values(), key=lambda e: (e.specific.hostname or e.specific.group)): specific = entry.specific hostnames = [] if specific.hostname and specific.hostname in names: hostnames = names[specific.hostname] elif specific.group: hostnames = \ reduce(lambda x, y: x + y, [names[cmeta.hostname] for cmeta in \ mquery.by_groups([specific.group])], []) elif specific.all: # a generic key for all hosts? really? hostnames = reduce(lambda x, y: x + y, list(names.values()), []) if not hostnames: if specific.hostname: key = specific.hostname ktype = "host" elif specific.group: key = specific.group ktype = "group" else: # user has added a global SSH key, but # have no clients yet. don't warn about # this. continue if key not in self.badnames: self.badnames[key] = True self.logger.info("Ignoring key for unknown %s %s" % (ktype, key)) continue skn.append("%s %s" % (','.join(hostnames), entry.data.rstrip())) self.__skn = "\n".join(skn) + "\n" return self.__skn