def run(self): log('Daemon started') wdds = [] notifiers = [] # read jobs from config file for section in self.config.sections(): log(section + ": " + self.config.get(section,'watch')) # get the basic config info mask = self._parseMask(self.config.get(section,'events').split(',')) folder = self.config.get(section,'watch') recursive = self.config.getboolean(section,'recursive') autoadd = self.config.getboolean(section,'autoadd') excluded = self.config.get(section,'excluded') # command = self.config.get(section,'command') # Exclude directories right away if 'excluded' regexp is set # Example https://github.com/seb-m/pyinotify/blob/master/python2/examples/exclude.py if excluded.strip() == '': # if 'excluded' is empty or whitespaces only excl = None else: excl = pyinotify.ExcludeFilter(excluded.split(',')) wm = pyinotify.WatchManager() handler = EventHandler(SCRIPT_LOCATION + '/backend/fixperm.sh $filename') wdds.append(wm.add_watch(folder, mask, rec=recursive, auto_add=autoadd, exclude_filter=excl)) # BUT we need a new ThreadNotifier so I can specify a different # EventHandler instance for each job # this means that each job has its own thread as well (I think) notifiers.append(pyinotify.ThreadedNotifier(wm, handler)) # now we need to start ALL the notifiers. # TODO: load test this ... is having a thread for each a problem? for notifier in notifiers: notifier.start()
def beacon(config): """ Watch the configured files Example Config .. code-block:: yaml beacons: inotify: - files: /path/to/file/or/dir: mask: - open - create - close_write recurse: True auto_add: True exclude: - /path/to/file/or/dir/exclude1 - /path/to/file/or/dir/exclude2 - /path/to/file/or/dir/regex[a-m]*$: regex: True - coalesce: True The mask list can contain the following events (the default mask is create, delete, and modify): * access - File accessed * attrib - File metadata changed * close_nowrite - Unwritable file closed * close_write - Writable file closed * create - File created in watched directory * delete - File deleted from watched directory * delete_self - Watched file or directory deleted * modify - File modified * moved_from - File moved out of watched directory * moved_to - File moved into watched directory * move_self - Watched file moved * open - File opened The mask can also contain the following options: * dont_follow - Don't dereference symbolic links * excl_unlink - Omit events for children after they have been unlinked * oneshot - Remove watch after one event * onlydir - Operate only if name is directory recurse: Recursively watch files in the directory auto_add: Automatically start watching files that are created in the watched directory exclude: Exclude directories or files from triggering events in the watched directory. Can use regex if regex is set to True coalesce: If this coalescing option is enabled, events are filtered based on their unicity, only unique events are enqueued, doublons are discarded. An event is unique when the combination of its fields (wd, mask, cookie, name) is unique among events of a same batch. After a batch of events is processed any events are accepted again. This option is top-level (at the same level as the path) and therefore affects all paths that are being watched. This is due to this option being at the Notifier level in pyinotify. """ _config = {} list(map(_config.update, config)) ret = [] notifier = _get_notifier(_config) wm = notifier._watch_manager # Read in existing events if notifier.check_events(1): notifier.read_events() notifier.process_events() queue = __context__["inotify.queue"] while queue: event = queue.popleft() _append = True # Find the matching path in config path = event.path while path != "/": if path in _config.get("files", {}): break path = os.path.dirname(path) excludes = _config["files"][path].get("exclude", "") if excludes and isinstance(excludes, list): for exclude in excludes: if isinstance(exclude, dict): _exclude = next(iter(exclude)) if exclude[_exclude].get("regex", False): try: if re.search(_exclude, event.pathname): _append = False except Exception: # pylint: disable=broad-except log.warning("Failed to compile regex: %s", _exclude) else: exclude = _exclude elif "*" in exclude: if fnmatch.fnmatch(event.pathname, exclude): _append = False else: if event.pathname.startswith(exclude): _append = False if _append: sub = { "tag": event.path, "path": event.pathname, "change": event.maskname, } ret.append(sub) else: log.info("Excluding %s from event for %s", event.pathname, path) # Get paths currently being watched current = set() for wd in wm.watches: current.add(wm.watches[wd].path) # Update existing watches and add new ones # TODO: make the config handle more options for path in _config.get("files", ()): if isinstance(_config["files"][path], dict): mask = _config["files"][path].get("mask", DEFAULT_MASK) if isinstance(mask, list): r_mask = 0 for sub in mask: r_mask |= _get_mask(sub) elif isinstance(mask, salt.ext.six.binary_type): r_mask = _get_mask(mask) else: r_mask = mask mask = r_mask rec = _config["files"][path].get("recurse", False) auto_add = _config["files"][path].get("auto_add", False) else: mask = DEFAULT_MASK rec = False auto_add = False if path in current: for wd in wm.watches: if path == wm.watches[wd].path: update = False if wm.watches[wd].mask != mask: update = True if wm.watches[wd].auto_add != auto_add: update = True if update: wm.update_watch(wd, mask=mask, rec=rec, auto_add=auto_add) elif os.path.exists(path): excludes = _config["files"][path].get("exclude", "") excl = None if isinstance(excludes, list): excl = [] for exclude in excludes: if isinstance(exclude, dict): excl.append(list(exclude)[0]) else: excl.append(exclude) excl = pyinotify.ExcludeFilter(excl) wm.add_watch(path, mask, rec=rec, auto_add=auto_add, exclude_filter=excl) # Return event data return ret
def beacon(config): ''' Watch the configured files Example Config .. code-block:: yaml beacons: inotify: /path/to/file/or/dir: mask: - open - create - close_write recurse: True auto_add: True exclude: - /path/to/file/or/dir/exclude1 - /path/to/file/or/dir/exclude2 - /path/to/file/or/dir/regex[a-m]*$: regex: True The mask list can contain the following events (the default mask is create, delete, and modify): * access - File accessed * attrib - File metadata changed * close_nowrite - Unwritable file closed * close_write - Writable file closed * create - File created in watched directory * delete - File deleted from watched directory * delete_self - Watched file or directory deleted * modify - File modified * moved_from - File moved out of watched directory * moved_to - File moved into watched directory * move_self - Watched file moved * open - File opened The mask can also contain the following options: * dont_follow - Don't dereference symbolic links * excl_unlink - Omit events for children after they have been unlinked * oneshot - Remove watch after one event * onlydir - Operate only if name is directory recurse: Recursively watch files in the directory auto_add: Automatically start watching files that are created in the watched directory exclude: Exclude directories or files from triggering events in the watched directory. Can use regex if regex is set to True ''' ret = [] notifier = _get_notifier() wm = notifier._watch_manager # Read in existing events if notifier.check_events(1): notifier.read_events() notifier.process_events() queue = __context__['inotify.queue'] while queue: event = queue.popleft() _append = True # Find the matching path in config path = event.path while path != '/': if path in config: break path = os.path.dirname(path) excludes = config[path].get('exclude', '') if excludes and isinstance(excludes, list): for exclude in excludes: if isinstance(exclude, dict): if exclude.values()[0].get('regex', False): try: if re.search(exclude.keys()[0], event.pathname): _append = False except Exception: log.warn('Failed to compile regex: {0}'.format( exclude.keys()[0])) else: exclude = exclude.keys()[0] elif '*' in exclude: if fnmatch.fnmatch(event.pathname, exclude): _append = False else: if event.pathname.startswith(exclude): _append = False if _append: sub = { 'tag': event.path, 'path': event.pathname, 'change': event.maskname } ret.append(sub) else: log.info('Excluding {0} from event for {1}'.format( event.pathname, path)) # Get paths currently being watched current = set() for wd in wm.watches: current.add(wm.watches[wd].path) # Update existing watches and add new ones # TODO: make the config handle more options for path in config: if isinstance(config[path], dict): mask = config[path].get('mask', DEFAULT_MASK) if isinstance(mask, list): r_mask = 0 for sub in mask: r_mask |= _get_mask(sub) elif isinstance(mask, salt.ext.six.binary_type): r_mask = _get_mask(mask) else: r_mask = mask mask = r_mask rec = config[path].get('recurse', False) auto_add = config[path].get('auto_add', False) else: mask = DEFAULT_MASK rec = False auto_add = False if path in current: for wd in wm.watches: if path == wm.watches[wd].path: update = False if wm.watches[wd].mask != mask: update = True if wm.watches[wd].auto_add != auto_add: update = True if update: wm.update_watch(wd, mask=mask, rec=rec, auto_add=auto_add) elif os.path.exists(path): excludes = config[path].get('exclude', '') excl = None if isinstance(excludes, list): excl = [] for exclude in excludes: if isinstance(exclude, dict): excl.append(exclude.keys()[0]) else: excl.append(exclude) excl = pyinotify.ExcludeFilter(excl) wm.add_watch(path, mask, rec=rec, auto_add=auto_add, exclude_filter=excl) # Return event data return ret
# we get inotify events for *at most* timeout seconds, then handle them all notifier = pyinotify.Notifier(wm, handler, timeout=1) t = time.time() watchers = [] for src in sources: log.info( "adding watches to '%s' (this could take several minutes)..." % src) dot_gluster = os.path.join(src, '.glusterfs') watchers.append( wm.add_watch(src, mask, rec=True, exclude_filter=pyinotify.ExcludeFilter( ['^' + dot_gluster]))) log.info("watch added (%s seconds): listening for file events..." % (time.time() - t)) def check_for_events(): #print "check_for_events" notifier.process_events() while notifier.check_events( ): #loop in case more events appear while we are processing notifier.read_events() notifier.process_events() while True: check_for_events() handle_modified_files() time.sleep(1)
log("synoindex %s %s" % (index_argument, event.pathname)) call(["synoindex", index_argument, event.pathname]) self.modified_files.discard(event.pathname) else: log("%s is not an allowed path" % event.pathname) def is_allowed_path(self, filename, is_dir): # Don't check the extension for directories if not is_dir: ext = os.path.splitext(filename)[1][1:].lower() return ext in allowed_exts return True handler = EventHandler() notifier = pyinotify.Notifier(wm, handler) excl_pattern = pyinotify.ExcludeFilter(config['exclude_dir_patterns']) wdd = wm.add_watch( watched_paths, mask, rec=True, auto_add=True, exclude_filter=excl_pattern) try: notifier.loop(daemonize=True, pid_file=config['pidfile']) except pyinotify.NotifierError, err: print >> sys.stderr, err
def main(): #Create option parser. parser = OptionParser(usage="usage: %prog [options] DIRECTORY", version="%prog 0.1") parser.add_option("-s", "--sync", dest="sync", action="store_true", default=False, help="Sync on start") parser.add_option("-b", "--bind", dest="bind", action="store", default='*', help="Host to bind to") parser.add_option( "-p", "--port", dest="port", action="store", default=7890, help="TCP port to bind to, the port +1 from this port is also used.") parser.add_option( "-e", "--exclude", dest="exclude", action="store", default='', help= "Exclude this regular expression from processing. (Multiple isn't implemented because the python opt parser sucks sort of.)" ) (options, args) = parser.parse_args() if len(args) != 1: print 'Invalid Usage' parser.print_help() return CHROOT_DIR = args[ 0] #I could probably ACTUALLY USE CHROOT here... save a lot of filename munging #os.chroot(CHROOT_DIR) #Needs ROOT # The watch manager stores the watches and provides operations on watches wm = pyinotify.WatchManager() sync = FileSync(CHROOT_DIR, host=options.bind, port=options.port) #Exclude Filter excl = None if options.exclude: excl = pyinotify.ExcludeFilter([options.exclude]) notifier = pyinotify.Notifier(wm, FileModifiedHandler(watch_dir=CHROOT_DIR, sync=sync, manager=wm, exclude=excl), timeout=10) notifier.coalesce_events() # Events that indicate modified files. (MOVED_SELF would be useful, but doesn't seem to work) mask = pyinotify.IN_DELETE | pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO | pyinotify.IN_ATTRIB | pyinotify.IN_MOVE_SELF | pyinotify.IN_MOVED_FROM | pyinotify.IN_CREATE #mask = pyinotify.ALL_EVENTS wdd = wm.add_watch(CHROOT_DIR, mask, rec=True, auto_add=True, exclude_filter=excl) try: while running: sync.check_messages() if notifier.check_events(): notifier.read_events() notifier.process_events() #Not necessary because of the pyinotify timeout attribute. #time.sleep(0.0100) #Prevent busy loop except KeyboardInterrupt: logger.info('Keyboard Interrupt, Exiting...')
def watch_section_job(section_name, section): setproctitle.setproctitle(sys.argv[0] + ':' + section_name) # process to consume items from this producer watch_queue = multiprocessing.Queue() watchjob = multiprocessing.Process(target=watcher, args=( watch_queue, section, )) watchjob.start() deleted_wd = [] max_deleted_queue = 10 wm = pyinotify.WatchManager() notifier = pyinotify.AsyncNotifier(wm, EventHandler(deleted_wd, watch_queue), timeout=10) exclude = pyinotify.ExcludeFilter(section['exclude']) template = {} # [small index date, next triggered seconds, wdd] for include_dir in section['include']: key = include_dir template[key] = [0, 0, None] template[key][0] = small_idx_date(include_dir) if template[key][0] > -1: now = tuple_date() template[key][1] = next_seconds(now, template[key][0]) sys.stdout.flush() include_dir = substitute_date(include_dir, now) if not os.path.exists(include_dir): continue template[key][2] = wm.add_watch(include_dir, watch_mask, rec=True, auto_add=True, exclude_filter=exclude) next_trigger = min(map(lambda x: template[x][1], template)) while not is_exit.value: notifier.process_events() while notifier.check_events(): notifier.read_events() notifier.process_events() if len(deleted_wd) > max_deleted_queue: while len(deleted_wd): sys.stdout.write("%s : Deleting unnecessary object : " % time.ctime()) wd = deleted_wd.pop() wm.del_watch(wd) sys.stdout.write("Done\n") sys.stdout.flush() # check for updated watch if next_trigger and time.time() >= next_trigger: # update watch sys.stdout.write("%s\n" % next_trigger) sys.stdout.write("%s : Updating watched directory\n" % time.ctime()) sys.stdout.flush() for include_dir in section['include']: key = include_dir if template[key][1] != next_trigger: continue now = tuple_date(next_trigger) include_dir = substitute_date(include_dir, now) if not os.path.exists(include_dir): continue # wait until directory exists template[key][1] = next_seconds(now, template[key][0]) next_trigger = min(map(lambda x: template[x][1], template)) wm.rm_watch(template[key][2].values(), rec=True) template[key][2] = wm.add_watch(include_dir, watch_mask, rec=True, auto_add=True, exclude_filter=exclude) notifier.stop() watchjob.join()
def process( configfile='salt://hubblestack_pulsar/hubblestack_pulsar_config.yaml', verbose=False): ''' Watch the configured files Example pillar config .. code-block:: yaml beacons: pulsar: paths: - /var/cache/salt/minion/files/base/hubblestack_pulsar/hubblestack_pulsar_config.yaml refresh_interval: 300 verbose: False Example yaml config on fileserver (targeted by pillar) .. code-block:: yaml /path/to/file/or/dir: mask: - open - create - close_write recurse: True auto_add: True exclude: - /path/to/file/or/dir/exclude1 - /path/to/file/or/dir/exclude2 - /path/to/file/or/dir/regex[\d]*$: regex: True return: splunk: batch: True slack: batch: False # overrides the global setting checksum: sha256 stats: True batch: True Note that if `batch: True`, the configured returner must support receiving a list of events, rather than single one-off events. The mask list can contain the following events (the default mask is create, delete, and modify): * access - File accessed * attrib - File metadata changed * close_nowrite - Unwritable file closed * close_write - Writable file closed * create - File created in watched directory * delete - File deleted from watched directory * delete_self - Watched file or directory deleted * modify - File modified * moved_from - File moved out of watched directory * moved_to - File moved into watched directory * move_self - Watched file moved * open - File opened The mask can also contain the following options: * dont_follow - Don't dereference symbolic links * excl_unlink - Omit events for children after they have been unlinked * oneshot - Remove watch after one event * onlydir - Operate only if name is directory recurse: Recursively watch files in the directory auto_add: Automatically start watching files that are created in the watched directory exclude: Exclude directories or files from triggering events in the watched directory. Can use regex if regex is set to True If pillar/grains/minion config key `hubblestack:pulsar:maintenance` is set to True, then changes will be discarded. ''' if not HAS_PYINOTIFY: log.debug('Not running beacon pulsar. No python-inotify installed.') return [] config = __opts__.get('pulsar', {}) if isinstance(configfile, list): config['paths'] = configfile else: config['paths'] = [configfile] config['verbose'] = verbose global CONFIG_STALENESS global CONFIG if config.get('verbose'): log.debug('Pulsar beacon called.') log.debug('Pulsar beacon config from pillar:\n{0}'.format(config)) ret = [] notifier = _get_notifier() wm = notifier._watch_manager update_watches = False # Get config(s) from salt fileserver if we don't have them already if CONFIG and CONFIG_STALENESS < config.get('refresh_interval', 300): CONFIG_STALENESS += 1 CONFIG.update(config) CONFIG['verbose'] = config.get('verbose') config = CONFIG else: if config.get('verbose'): log.debug( 'No cached config found for pulsar, retrieving fresh from fileserver.' ) new_config = config if isinstance(config.get('paths'), list): for path in config['paths']: if 'salt://' in path: path = __salt__['cp.cache_file'](path) if os.path.isfile(path): with open(path, 'r') as f: new_config = _dict_update(new_config, yaml.safe_load(f), recursive_update=True, merge_lists=True) else: log.error( 'Path {0} does not exist or is not a file'.format( path)) else: log.error( 'Pulsar beacon \'paths\' data improperly formatted. Should be list of paths' ) new_config.update(config) config = new_config CONFIG_STALENESS = 0 CONFIG = config update_watches = True if config.get('verbose'): log.debug( 'Pulsar beacon config (compiled from config list):\n{0}'.format( config)) # Read in existing events if notifier.check_events(1): notifier.read_events() notifier.process_events() queue = __context__['pulsar.queue'] if config.get('verbose'): log.debug('Pulsar found {0} inotify events.'.format(len(queue))) while queue: event = queue.popleft() if event.maskname == 'IN_Q_OVERFLOW': log.warn('Your inotify queue is overflowing.') log.warn( 'Fix by increasing /proc/sys/fs/inotify/max_queued_events') continue _append = True # Find the matching path in config path = event.path while path != '/': if path in config: break path = os.path.dirname(path) # Get pathname try: pathname = event.pathname except NameError: pathname = path excludes = config[path].get('exclude', '') if excludes and isinstance(excludes, list): for exclude in excludes: if isinstance(exclude, dict): if exclude.values()[0].get('regex', False): try: if re.search(exclude.keys()[0], event.pathname): _append = False except: log.warn('Failed to compile regex: {0}'.format( exclude.keys()[0])) pass else: exclude = exclude.keys()[0] elif '*' in exclude: if fnmatch.fnmatch(event.pathname, exclude): _append = False else: if event.pathname.startswith(exclude): _append = False if _append: sub = { 'tag': event.path, 'path': event.pathname, 'change': event.maskname, 'name': event.name } if config.get('checksum', False) and os.path.isfile(pathname): sum_type = config['checksum'] if not isinstance(sum_type, salt.ext.six.string_types): sum_type = 'sha256' sub['checksum'] = __salt__['file.get_hash'](pathname, sum_type) sub['checksum_type'] = sum_type if config.get('stats', False): sub['stats'] = __salt__['file.stats'](pathname) ret.append(sub) else: log.info('Excluding {0} from event for {1}'.format( event.pathname, path)) if update_watches: # Get paths currently being watched current = set() for wd in wm.watches: current.add(wm.watches[wd].path) # Update existing watches and add new ones # TODO: make the config handle more options for path in config: if path == 'return' or path == 'checksum' or path == 'stats' \ or path == 'batch' or path == 'verbose' or path == 'paths' \ or path == 'refresh_interval': continue if isinstance(config[path], dict): mask = config[path].get('mask', DEFAULT_MASK) excludes = config[path].get('exclude', None) if isinstance(mask, list): r_mask = 0 for sub in mask: r_mask |= _get_mask(sub) elif isinstance(mask, salt.ext.six.binary_type): r_mask = _get_mask(mask) else: r_mask = mask mask = r_mask rec = config[path].get('recurse', False) auto_add = config[path].get('auto_add', False) else: mask = DEFAULT_MASK rec = False auto_add = False if path in current: for wd in wm.watches: if path == wm.watches[wd].path: update = False if wm.watches[wd].mask != mask: update = True if wm.watches[wd].auto_add != auto_add: update = True if update: wm.update_watch(wd, mask=mask, rec=rec, auto_add=auto_add) elif os.path.exists(path): excl = None if isinstance(excludes, list): excl = [] for exclude in excludes: if isinstance(exclude, dict): excl.append(exclude.keys()[0]) else: excl.append(exclude) excl = pyinotify.ExcludeFilter(excl) wm.add_watch(path, mask, rec=rec, auto_add=auto_add, exclude_filter=excl) # Process watch removals to_delete = [] for wd in wm.watches: found = False for path in config: if path in wm.watches[wd].path: found = True if not found: to_delete.append(wd) for wd in to_delete: wm.del_watch(wd) if __salt__['config.get']('hubblestack:pulsar:maintenance', False): # We're in maintenance mode, throw away findings ret = [] return ret
# @Author : guangzu ([email protected]) # @Link : blog.zhuguangzu.xyz # @Version : $Id$ # Example: exclude items from being monitored. # import os import pyinotify wm = pyinotify.WatchManager() notifier = pyinotify.Notifier(wm) # Method 1: # Exclude patterns from file excl_file = os.path.join(os.getcwd(), 'exclude.lst') excl = pyinotify.ExcludeFilter(excl_file) # Add watches res = wm.add_watch(['/etc/hostname', '/etc/cups', '/etc/rc0.d'], pyinotify.ALL_EVENTS, rec=True, exclude_filter=excl) # Method 2 (Equivalent) # Exclude patterns from list excl_lst = [ '^/etc/apache[2]?/', '^/etc/rc.*', '^/etc/hostname', '^/etc/hosts', '^/etc/(fs|m)tab', '^/etc/cron\..*' ] excl = pyinotify.ExcludeFilter(excl_lst) # Add watches res = wm.add_watch(['/etc/hostname', '/etc/cups', '/etc/rc0.d'],
def process_IN_CLOSE_WRITE(self, event): print "Close write:", event.pathname add(event.pathname) handler = EventHandler() # we get inotify events for *at most* timeout seconds, then handle them all notifier = pyinotify.Notifier(wm, handler, timeout=1) t = time.time() watchers = [] for src in sources: log.info("adding watches to '%s' (this could take several minutes)..."%src) dot_gluster = os.path.join(src, '.glusterfs') watchers.append(wm.add_watch(src, mask, rec=True, exclude_filter=pyinotify.ExcludeFilter(['^'+dot_gluster]))) log.info("watch added (%s seconds): listening for file events..."%(time.time() - t)) def check_for_events(): #print "check_for_events" notifier.process_events() while notifier.check_events(): #loop in case more events appear while we are processing notifier.read_events() notifier.process_events() while True: check_for_events() handle_modified_files() time.sleep(1) def volume_info():