class DirectoryWatcher(WatchManager): def __init__(self): WatchManager.__init__(self) def start(self): self.inotify = ThreadedNotifier(self) self.inotify.start() self.inotify.join() def stop(self): self.inotify.stop() def add_monitor_path(self, path): if path is None: Logger.error("FS: unable to monitor None directory") return False exclude1 = "^%s/conf.Windows*"%(path) exclude2 = "^%s/conf.Linux*"%(path) exc_filter = ExcludeFilter([exclude1, exclude2]) try: self.add_watch(path=path, mask=Rec.mask, proc_fun=Rec(), rec=True, auto_add=True, exclude_filter=exc_filter) except WatchManagerError, e: Logger.error("FS: unable to monitor directory %s, %s"%(path, str(e))) return False return False
class Notify(): def __init__(self): self.wm = WatchManager() self.pe = ProcessNotifyEvents() self.notifier = ThreadedNotifier(self.wm, self.pe) self.notifier.start() self.path = None #thread.start_new_thread(self.jobTask, (self,)) def setNotify(self, path, cbfun): #print 'setnotify ' + path if self.path: self.wm.rm_watch(list(self.wdd.values())) self.path = path self.pe.cbfun = cbfun # ugly... #print sys.getfilesystemencoding() self.wdd = self.wm.add_watch(self.path, pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MOVED_TO | pyinotify.IN_MOVED_FROM | pyinotify.IN_MODIFY) def stop(self): if self.path: self.wm.rm_watch(list(self.wdd.values())) self.notifier.stop() def notifyThread(self): while 1: notifier.process_events() if notifier.check_events(): notifier.read_events()
def test_gutils_netcdf_to_erddap_watch(self): wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE # Convert ASCII data to NetCDF processor = Netcdf2ErddapProcessor( deployments_path=resource('slocum'), erddap_content_path=erddap_content_path, erddap_flag_path=erddap_flag_path ) notifier = ThreadedNotifier(wm, processor, read_freq=5) notifier.coalesce_events() notifier.start() wdd = wm.add_watch( netcdf_path, mask, rec=True, auto_add=True ) # Wait 5 seconds for the watch to start time.sleep(5) orig_netcdf = resource('profile.nc') dummy_netcdf = os.path.join(netcdf_path, 'profile.nc') shutil.copy(orig_netcdf, dummy_netcdf) wait_for_files(erddap_content_path, 1) wait_for_files(erddap_flag_path, 1) wm.rm_watch(wdd.values(), rec=True) notifier.stop()
class DirectoryWatcher(WatchManager): def __init__(self): WatchManager.__init__(self) def start(self): self.inotify = ThreadedNotifier(self) self.inotify.start() self.inotify.join() def stop(self): self.inotify.stop() def add_monitor_path(self, path): if path is None: Logger.error("FS: unable to monitor None directory") return False exclude1 = "^%s/conf.Windows*" % (path) exclude2 = "^%s/conf.Linux*" % (path) exc_filter = ExcludeFilter([exclude1, exclude2]) try: self.add_watch(path=path, mask=Rec.mask, proc_fun=Rec(), rec=True, auto_add=True, exclude_filter=exc_filter) except WatchManagerError, e: Logger.error("FS: unable to monitor directory %s, %s" % (path, str(e))) return False return False
def test_gutils_binary_to_ascii_watch(self): wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE # Convert binary data to ASCII processor = Slocum2AsciiProcessor( deployments_path=resource('slocum'), ) notifier = ThreadedNotifier(wm, processor) notifier.coalesce_events() notifier.start() wdd = wm.add_watch( binary_path, mask, rec=True, auto_add=True ) # Wait 5 seconds for the watch to start time.sleep(5) gpath = os.path.join(original_binary, '*.*bd') # Sort the files so the .cac files are generated in the right order for g in sorted(glob(gpath)): shutil.copy2(g, binary_path) wait_for_files(ascii_path, 32) wm.rm_watch(wdd.values(), rec=True) notifier.stop()
def test_gutils_ascii_to_netcdf_watch(self): wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE # Convert ASCII data to NetCDF processor = Slocum2NetcdfProcessor(deployments_path=resource('slocum'), subset=False, template='trajectory', profile_id_type=2, tsint=10, filter_distance=1, filter_points=5, filter_time=10, filter_z=1) notifier = ThreadedNotifier(wm, processor) notifier.coalesce_events() notifier.start() wdd = wm.add_watch(ascii_path, mask, rec=True, auto_add=True) # Wait 5 seconds for the watch to start time.sleep(5) # Make the ASCII we are watching for merger = SlocumMerger(original_binary, ascii_path, globs=['*.tbd', '*.sbd']) merger.convert() wait_for_files(netcdf_path, 230) wm.rm_watch(wdd.values(), rec=True) notifier.stop()
class PhotoWatcher(ProcessEvent): MASK = (EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CLOSE_WRITE'] | EventsCodes.ALL_FLAGS['IN_MOVED_FROM'] | EventsCodes.ALL_FLAGS['IN_MOVED_TO']) def __init__(self, db, walker, root): self.root = root self.db = db self.walker = walker self.wm = WatchManager() self.wdds = [] def Watch(self): self.notifier = ThreadedNotifier(self.wm, self) self.notifier.start() self.wdds.append(self.wm.add_watch(self.root, self.MASK, rec=True)) # add soft link sub-folders for dirname, dirnames, _filenames in os.walk(self.root, followlinks=True): for d in dirnames: path = os.path.join(dirname, d) if os.path.islink(path): self.wdds.append( self.wm.add_watch(os.path.realpath(path), self.MASK, rec=True)) def Stop(self): self.notifier.stop() def process_IN_DELETE(self, event): self.db.DeletePhoto(os.path.join(event.path, event.name)) def process_IN_MOVED_FROM(self, event): self.process_IN_DELETE(event) def process_IN_MOVED_TO(self, event): full_path = os.path.join(event.path, event.name) try: meta = self.walker.ReadMetadata(full_path) except Exception: return self.db.StorePhoto(full_path, meta) def process_IN_CLOSE_WRITE(self, event): full_path = os.path.join(event.path, event.name) try: meta = self.walker.ReadMetadata(full_path) except Exception: return if self.db.HasPhoto(full_path): self.db.UpdatePhoto(full_path, meta) else: self.db.StorePhoto(full_path, meta)
class FileEvent: def __init__(self, eventHandler): self.logger = logging.getLogger('FileEvent') self.wm = WatchManager() self.watches = dict() # Set the flags of the events that are to be listened to FLAGS = EventsCodes.ALL_FLAGS self.mask = FLAGS['IN_CREATE'] | FLAGS['IN_DELETE'] | FLAGS['IN_MODIFY'] | FLAGS['IN_DELETE_SELF'] # Set-up notifier self.notifier = ThreadedNotifier(self.wm, EventProcessor(eventHandler)) def startNotifyLoop(self): self.notifier.start() def stopNotifyLoop(self): self.notifier.stop() def addWatches(self, paths, mask=None): added_watches = dict() for path in paths: added_watches.update(self.addWatch(path, mask)) return added_watches # Also monitors all sub-directories of the given directory and automatically adds newly # created directories to watch. # TODO should be able to add files as well, but doesn't work atm def addWatch(self, path, mask=None): if mask is None: mask = self.mask added_watches = self.wm.add_watch(path, mask, rec=True, auto_add=True) self.watches.update(added_watches) return added_watches def removeWatch(self, path): watch_descriptor = self.wm.get_wd(path) if watch_descriptor is not None: result = self.wm.rm_watch(watch_descriptor, rec=True) # Remove the no longer active watches from the current watches dictionary for key, value in self.watches.items(): if value in result: del self.watches[key] else: result = None return result def getWatches(self): return self.watches
def main(): # Add a dummy message to the queue: add_msg( MsgTypes.Initialise ) # Setup the HTTP server: SocketServer.TCPServer.allow_reuse_address = True HOST, PORT = "localhost", 9000 server = ThreadedHTTPServer((HOST, PORT), MyTCPHandler) server_thread = threading.Thread(target=server.serve_forever) server_thread.daemon = True server_thread.start() # Setup the heartbeat: heartbeat_thread = threading.Thread(target=heartbeat_messages) heartbeat_thread.daemon = True heartbeat_thread.start() # Setup pyinotify so files are being watched: wm = WatchManager() notifier = ThreadedNotifier(wm, PTmp()) notifier.start() mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_ATTRIB'] |EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events #mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events wdd = wm.add_watch('../sentry.py', mask, rec=True) wdd = wm.add_watch('../', mask, rec=True) try: while True: time.sleep(5) except: # Turn off pyinotify: notifier.stop() print 'Exception caught: shutting down connections' add_msg(MsgTypes.Shutdown) time.sleep(0.5) print 'Terminating...' raise
class FileWatcher(Thread): def __init__(self, holder, uri, schedule_reader): Thread.__init__(self) self._loop = True self._error_event = Event() self._wm = WatchManager() self._holder = holder self._uri = uri self._schedule_reader = schedule_reader self._notifier = ThreadedNotifier(self._wm, _EventHandler(self._holder, self._uri, self._schedule_reader, self._error_event)) self._path, self._pattern = os.path.split(urlparse(uri).path) def start(self): """Start the file watcher """ self._notifier.start() self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY) Thread.start(self) def run(self): while self._loop: if self._error_event.wait(1): self._error_event.clear() self._notifier.stop() del self._notifier self._notifier = ThreadedNotifier( self._wm, _EventHandler(self._holder, self._uri, self._schedule_reader, self._error_event)) self._notifier.start() self._wm.add_watch( self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY) def stop(self): """Stop the file watcher """ self._notifier.stop() self._loop = False
class _PathWatcher(Queue): # iNotify watcher for directory ''' iNotify watcher object for monitor of changes in directory. ''' FLAGS = IN_MODIFY | IN_DELETE | IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO | IN_ATTRIB def __init__(self, path, exclude=None): class _EH(ProcessEvent): def process_default(self, event): _handleEvent(event) Queue.__init__(self) self._path = path self.exclude = exclude or [] _handleEvent = self.put self._wm = WatchManager() self._iNotifier = ThreadedNotifier(self._wm, _EH(), timeout=10) self._iNotifier.start() self.started = False self._watch = [] def start(self, exclude=None): if not self.started: # Add watch and start watching # Update exclude filter if it provided in call of start method self.exclude = exclude or self.exclude self._watch = self._wm.add_watch(self._path, self.FLAGS, exclude_filter=ExcludeFilter( self.exclude), auto_add=True, rec=True, do_glob=False) self.started = True def stop(self): if self.started: # Remove watch and stop watching self._wm.rm_watch(self._watch[self._path], rec=True) self.started = False def exit(self): self.stop() self._iNotifier.stop()
def lastwatch(paths, settings, dry_run=False): flags = EventsCodes.FLAG_COLLECTIONS.get('OP_FLAGS', None) if flags: mask = flags.get('IN_OPEN') | flags.get('IN_CLOSE_NOWRITE') mask |= flags.get('IN_CREATE') | flags.get('IN_MOVED_TO') else: mask = EventsCodes.IN_OPEN | EventsCodes.IN_CLOSE_NOWRITE mask |= EventsCodes.IN_CREATE | EventsCodes.IN_MOVED_TO wm = WatchManager() handler = Handler(settings, dry_run=dry_run) watcher = ThreadedNotifier(wm, handler) watcher.start() try: for path in paths: path = os.path.realpath(path) sys.stdout.write(_("Indexing %s for watching...") % path) sys.stdout.flush() wm.add_watch(path, mask, rec=True, auto_add=True) sys.stdout.write(_(" done.") + "\n") print _("You have successfully launched Lastwatch.") print "\n".join(wrap(_("The directories you have specified will be " "monitored as long as this process is running, " "the flowers are blooming and the earth " "revolves around the sun..."), 80)) # flowers to devhell ;-) handler.set_active() while True: time.sleep(1) except KeyboardInterrupt: watcher.stop() print _("LastWatch stopped.") return except Exception, err: print err
class FilesystemWatcher: # Mask for FS events we are interested in - currently only care about # writable file streams that have been closed (i.e. a pdf has finished # uploading) mask = EventsCodes.ALL_FLAGS['IN_CLOSE_WRITE'] # This is the main watch manager provided by pyinotify # we use this to manage subscriptions _wm = WatchManager() # List of watched directories wdd = {} # create a thread-safe queue for papers that must be processed. paper_queue = Queue() def __init__(self, logger): self.logger = logger self.notifier = ThreadedNotifier( self._wm, PaperProcesser(self.logger, self.paper_queue)) def watch_directory(self, path): """set up threaded directory watcher at given path""" self._wm.add_watch(path, self.mask, rec=True) #add all files in the given directory to queue for root, dirs, files in os.walk(path): for file in files: if file.endswith("pdf") or file.endswith("xml"): self.logger.info("Adding %s to queue", file) self.paper_queue.put(("QUEUE", os.path.join(root, file))) def start(self): self.notifier.start() def stop(self): self.notifier.stop()
class Watch(object): def __init__(self, path, callback, ignore_modifications=False, latency=None): self.event_mask = EVENT_MASK if ignore_modifications else EVENT_MASK_WITH_MODIFICATIONS self._dir_queue = queue.Queue() self._root = os.path.realpath(path) self._watch_manager = WatchManager() self._processor = FileProcessEvent(directory_queue=self._dir_queue, root=self._root, callback=callback, latency=latency) self._notifier = ThreadedNotifier(self._watch_manager, self._processor) self._notifier.name = "[inotify] notifier" self._notifier.daemon = True self._notifier.start() self._watch_manager.add_watch(path, self.event_mask, rec=True, auto_add=True) def stop(self): self._notifier.stop() self._processor.stop()
class FileWatcher(Thread): def __init__(self, holder, uri, schedule_reader): Thread.__init__(self) self._loop = True self._error_event = Event() self._wm = WatchManager() self._holder = holder self._uri = uri self._schedule_reader = schedule_reader self._notifier = ThreadedNotifier( self._wm, _EventHandler(self._holder, self._uri, self._schedule_reader, self._error_event)) self._path, self._pattern = os.path.split(urlparse(uri).path) def start(self): """Start the file watcher """ self._notifier.start() self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY) Thread.start(self) def run(self): while self._loop: if self._error_event.wait(1): self._error_event.clear() self._notifier.stop() del self._notifier self._notifier = ThreadedNotifier( self._wm, _EventHandler(self._holder, self._uri, self._schedule_reader, self._error_event)) self._notifier.start() self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY) def stop(self): """Stop the file watcher """ self._notifier.stop() self._loop = False
class Watcher(object): """ Watching on the fly """ def __init__(self, gdr, workq, src, dst): self.gdr = gdr self.workq = workq self.src = src self.dst = dst def loop(self): wm = WatchManager() handler = EventHandler(self.workq, self.src, self.dst) self.notifier = ThreadedNotifier(wm, handler) self.notifier.start() mask = IN_CREATE | IN_MODIFY wm.add_watch(self.src, mask, rec=self.gdr.rec) def stop(self): self.notifier.stop()
class Watcher(object): """ Watching on the fly """ def __init__(self, gdr, workq, src, dst): self.gdr = gdr self.workq = workq self.src = src self.dst = dst def loop(self): wm = WatchManager() handler = EventHandler(self.workq, self.src, self.dst) self.notifier = ThreadedNotifier(wm, handler) self.notifier.start() mask = IN_CREATE | IN_MODIFY wm.add_watch(self.src, mask, rec=self.gdr.rec) def stop(self): self.notifier.stop()
def main_loop(): # Setup pyinotify so files are being watched: wm = WatchManager() notifier = ThreadedNotifier(wm, PTmp()) notifier.start() mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_ATTRIB'] |EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events #mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events #wdd = wm.add_watch('../sentry.py', mask, rec=True) #wdd = wm.add_watch('../', mask, rec=True) try: while True: print '\rConnections to %d clients. (%s)' % (len(open_handles), time_string()), sys.stdout.flush() update_subscribers(msg_type=MsgTypes.Heartbeat) time.sleep(5) except: notifier.stop() raise
class Observer: """Monitor files and notify the main program when changes happen""" def __init__(self, makeRunView): self.wm = WatchManager() self.eh = EventHandler(makeRunView) self.notifier = ThreadedNotifier(self.wm, self.eh) self.notifier.start() # Watched events self.mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE def kill(self): status = self.notifier.stop() logging.debug("Observer shut down") return status def addFile(self, fname): wdd = self.wm.add_watch(fname, self.mask, rec=True)
class FSMonitorInotify(FSMonitor): """inotify support for FSMonitor""" EVENTMAPPING = { FSMonitor.CREATED: pyinotify.IN_CREATE, FSMonitor.MODIFIED: pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB, FSMonitor.DELETED: pyinotify.IN_DELETE, FSMonitor.MONITORED_DIR_MOVED: pyinotify.IN_MOVE_SELF, FSMonitor.DROPPED_EVENTS: pyinotify.IN_Q_OVERFLOW, } def __init__(self, callback, persistent=False, trigger_events_for_initial_scan=False, ignored_dirs=[], dbfile="fsmonitor.db"): FSMonitor.__init__(self, callback, persistent, trigger_events_for_initial_scan, ignored_dirs, dbfile) self.wm = None self.notifier = None def __fsmonitor_event_to_inotify_event(self, event_mask): """map an FSMonitor event to an inotify event""" inotify_event_mask = 0 for fsmonitor_event_mask in self.__class__.EVENTMAPPING.keys(): if event_mask & fsmonitor_event_mask: inotify_event_mask = inotify_event_mask | self.__class__.EVENTMAPPING[ fsmonitor_event_mask] return inotify_event_mask def __add_dir(self, path, event_mask): """override of FSMonitor.__add_dir()""" # Perform an initial scan of the directory structure. If this has # already been done, then it will return immediately. if self.persistent: if self.trigger_events_for_initial_scan: FSMonitor.generate_missed_events(self, path, event_mask) else: self.pathscanner.initial_scan(path) event_mask_inotify = self.__fsmonitor_event_to_inotify_event( event_mask) # Use the inotify API to monitor a directory. wdd = self.wm.add_watch(path, event_mask_inotify, proc_fun=self.process_event, rec=True, auto_add=True) if wdd is None: raise MonitorError, "Could not monitor %s" % path return None else: self.monitored_paths[path] = MonitoredPath(path, event_mask, wdd) self.monitored_paths[path].monitoring = True # Generate the missed events. if self.persistent: FSMonitor.generate_missed_events(self, path) return self.monitored_paths[path] def __remove_dir(self, path): """override of FSMonitor.__remove_dir()""" if path in self.monitored_paths.keys(): wd = self.monitored_paths[path].fsmonitor_ref # TODO: figure out why this won't work, it seems this fails due to # a bug in pyinotify? #self.wm.rm_watch(wd, rec=True, quiet=True) del self.monitored_paths[path] def run(self): # Setup. Ensure that this isn't interleaved with any other thread, so # that the DB setup continues as expected. self.lock.acquire() FSMonitor.setup(self) self.process_event = FSMonitorInotifyProcessEvent(self) self.lock.release() # Set up inotify. self.wm = WatchManager() self.notifier = ThreadedNotifier(self.wm, self.process_event) self.notifier.start() while not self.die: self.__process_queues() time.sleep(0.5) self.notifier.stop() def stop(self): """override of FSMonitor.stop()""" # Let the thread know it should die. self.lock.acquire() self.die = True self.lock.release() # Stop monitoring each monitored path. for path in self.monitored_paths.keys(): self.__remove_dir(path) def __process_queues(self): # Process add queue. self.lock.acquire() if not self.add_queue.empty(): (path, event_mask) = self.add_queue.get() self.lock.release() self.__add_dir(path, event_mask) else: self.lock.release() # Process remove queue. self.lock.acquire() if not self.remove_queue.empty(): path = self.add_queue.get() self.lock.release() self.__remove_dir(path) else: self.lock.release()
self._dispatch_download(file_path, file_name) else: print "Skipping %s" % file_name def _process_directory(self, dir_path): candidates = os.listdir(dir_path) print candidates for candidate in candidates: if candidate.split('.')[-1].lower() in allowed_extensions: self._dispatch_download(dir_path, candidate) else: print "Skipping %s" % candidate print "SDownloader is now starting..." wm = WatchManager() wm.add_watch(target_dir, mask, rec=True, auto_add=True) print "Sdownloader is now watching %s" % target_dir notifier = ThreadedNotifier(wm, MyWatcher()) notifier.start() while True: try: time.sleep(0.5) except (Exception, KeyboardInterrupt): notifier.stop() break
class FilesystemMonitor(object): """ FileMonitor Class keeps track of all files down a tree starting at the root """ def __init__(self, searcher): self.searcher = searcher self._thread_pool = ThreadPool(THREAD_POOL_WORKS) # Add a watch to the root of the dir self.watch_manager = WatchManager() self.notifier = ThreadedNotifier(self.watch_manager, FileProcessEvent(self)) self.notifier.start() self._build_exclude_list() def _build_exclude_list(self): log.info("[FileMonitor] Set Regexs for Ignore List") self._exclude_regexs = [] # Complie Ignore list in to a list of regexs for ignore in self.searcher.configuration.exclude_list: ignore = ignore.strip() ignore = ignore.replace(".", "\.") ignore = ignore.replace("*", ".*") ignore = "^" + ignore + "$" log.debug("[FileMonitor] Ignore Regex = %s" % ignore) self._exclude_regexs.append(re.compile(ignore)) def change_root(self, previous_root): self._thread_pool.clearTasks() wd = self.watch_manager.get_wd(previous_root) if wd: self.watch_manager.rm_watch(wd, rec=True) self.searcher.clear_database() self.add_directory(self.searcher.current_root) def add_directory(self, path): """ Starts a WalkDirectoryThread to add the directory """ basename = os.path.basename(path) if self.validate(basename): self.watch_manager.add_watch(path, EVENT_MASK) self._thread_pool.queueTask(self.walk_directory, path) def add_file(self, path, name): """ Add a single file to the databse """ if self.validate(name): self.searcher.add_file(path, name) def remove_file(self, path, name): self.searcher.remove_file(path, name) def remove_directory(self, path): self.searcher.remove_directory(path) def walk_directory(self, root): """ From a give root of a tree this method will walk through ever branch and return a generator. """ if os.path.isdir(root): names = os.listdir(root) for name in names: try: file_stat = os.lstat(os.path.join(root, name)) except os.error: continue if stat.S_ISDIR(file_stat.st_mode): self.add_directory(os.path.join(root, name)) else: if not stat.S_ISLNK(file_stat.st_mode): self.add_file(root, name) def finish(self): wd = self.watch_manager.get_wd(self.searcher.current_root) self.watch_manager.rm_watch(wd, rec=True) self.notifier.stop() self._thread_pool.joinAll(waitForTasks=False) def validate(self, name): # Check to make sure the file not in the ignore list for ignore_re in self._exclude_regexs: if ignore_re.match(name): log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name) return False log.debug("[WalkDirectoryThread] # Passed %s", name) return True
def stop(self, *args, **kwargs): self._default_proc_fun.stop() ThreadedNotifier.stop(self, *args, **kwargs)
class TargetWatcher(object): def __init__(self, configuration, builder, watch_index): self._builder = builder self._root = configuration.GetExpandedDir("projects", "root_dir") self._batch_timeout = float( configuration.Get("file_watcher", "event_batch_timeout_ms")) / 1000 self._moddef_filename = configuration.Get( "general", "module_definition_filename") self.wm = WatchManager() self.watch_index = watch_index self.watched_module_definitions = collections.defaultdict(dict) mask = (EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']) handler = functools.partial(TargetWatcher.ProcessEvent, self) self.events_queue = queue.Queue() self.acc_thread = threading.Thread(target=functools.partial( TargetWatcher.AccumulationThreadProc, self), daemon=True) self.acc_thread.start() self.notifier = ThreadedNotifier(self.wm, handler) self.notifier.start() self.watch = self.wm.add_watch(self._root, mask, rec=True, auto_add=True) self.modification_handlers = [] def _GetAllModuleDefinitionsForTarget(self, target_name): prefix = "" module_definition_dirs = [""] for path_element in target_name.split("/"): if prefix: prefix = prefix + "/" + path_element else: prefix = path_element module_definition_dirs.append(prefix) return module_definition_dirs def AddModificationHandler(self, handler): self.modification_handlers.append(handler) def ProcessEventsBatch(self, batch): modified_targets = set() modified_module_definitions = set() root_prefix_len = len(self._root) for event in batch: rel_path = event.pathname[root_prefix_len + 1:] if rel_path.endswith(self._moddef_filename): conf_dir = rel_path[:-len(self._moddef_filename) - 1] modified_module_definitions.update( set(self.watched_module_definitions[conf_dir].values())) else: found_targets = self.watch_index.GetMatchingTargets(rel_path) modified_targets.update(found_targets.values()) if modified_module_definitions or modified_targets: self.ModificationsFound(modified_module_definitions, modified_targets) def ModificationsFound(self, modified_module_definitions, modified_targets): logging.info("Files modified: module definitions %s, other %s", modified_module_definitions, modified_targets) for handler in self.modification_handlers: handler(modified_module_definitions, modified_targets) def AccumulationThreadProc(self): event_buffer = [] while True: try: if event_buffer: item = self.events_queue.get(block=True, timeout=self._batch_timeout) else: item = self.events_queue.get(block=True) event_buffer.append(item) except queue.Empty as e: try: self.ProcessEventsBatch(event_buffer[:]) except e: logging.exception("Uncaught change event processing error") event_buffer = [] def Join(self): self.notifier.stop() def ProcessEvent(self, event): self.events_queue.put(event) def _RefreshGlobs(self, target): watched_globs = self._builder.GetWatchableSources(target.GetName()) target_dir = os.path.dirname(target.GetName()) rel_globs = [ os.path.join(target_dir, glob_p) for glob_p in watched_globs ] self.watch_index.LoadGlobsForTarget(target, rel_globs) def ReloadTarget(self, target): self._RefreshGlobs(target) def AddTarget(self, target): self._RefreshGlobs(target) for prefix in self._GetAllModuleDefinitionsForTarget(target.GetName()): self.watched_module_definitions[prefix][target.GetName()] = target def RemoveTarget(self, target): del self.watched[target.GetName()] for prefix in self._GetAllModuleDefinitionsForTarget(target.GetName()): del self.watched_module_definitions[prefix][target.GetName()]
def main(**options): """ Main function. It will create instances of Odoo Server, EventHandler, ThreadedNotifier and ProcessPool. """ logger = logging.getLogger(PROGNAME) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s") logfile = options["logfile"] if logfile is None: streamH = logging.StreamHandler(sys.stdout) else: streamH = logging.FileHandler(logfile) streamH.setFormatter(formatter) logger.addHandler(streamH) logger.info("Started") odoo = OdooInstance( logger, db=options["db"], user=options["user"], password=options["password"], host=options["host"], port=options["port"], ) odoo.connect_to_odoo() exit_code = 0 if odoo.is_connected: count = 0 while True: count += 1 try: event_handler = EventHandler( odoo=odoo, logger=logger, dir_to_watch=options["dir_to_watch"], regex=options["regex"], delimiter=options["delimiter"], ) # start loop handler masks = IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF wm = WatchManager() wm.add_watch(event_handler.dir_to_watch, masks) notifier = ThreadedNotifier(wm, default_proc_fun=event_handler) notifier.start() pool = ProcessPool(logger, odoo) pool.loop() except (KeyboardInterrupt, EOFError): logger.info("\nReceived signal to exit! Goodbye") break except: logger.error("ERROR! \n%s" % traceback.print_exception(*sys.exc_info())) finally: if notifier: notifier.stop() if count >= 50: logger.error("ERROR! Too many tries, aborting!") break else: logger.critical("Can't connect to odoo") exit_code = 1 exit(exit_code)
class FilesystemMonitor(object): """ FileMonitor Class keeps track of all files down a tree starting at the root """ def __init__(self, searcher): self.searcher = searcher self._thread_pool = ThreadPool(THREAD_POOL_WORKS) # Add a watch to the root of the dir self.watch_manager = WatchManager() self.notifier = ThreadedNotifier(self.watch_manager, FileProcessEvent(self)) self.notifier.start() self._build_exclude_list() def _build_exclude_list(self): log.info("[FileMonitor] Set Regexs for Ignore List") self._exclude_regexs = [] # Complie Ignore list in to a list of regexs for ignore in self.searcher.configuration.get_value("EXCLUDE_LIST"): ignore = ignore.strip() ignore = ignore.replace(".", "\.") ignore = ignore.replace("*", ".*") ignore = "^"+ignore+"$" log.debug("[FileMonitor] Ignore Regex = %s" % ignore) self._exclude_regexs.append(re.compile(ignore)) def change_root(self, previous_root): self._thread_pool.clearTasks() wd = self.watch_manager.get_wd(previous_root) if wd: self.watch_manager.rm_watch(wd, rec=True) self.searcher.clear_database() self.add_directory(self.searcher.current_root) def add_directory(self, path): """ Starts a WalkDirectoryThread to add the directory """ basename = os.path.basename(path) if self.validate(basename): self.watch_manager.add_watch(path, EVENT_MASK) self._thread_pool.queueTask(self.walk_directory, path) def add_file(self, path, name): """ Add a single file to the databse """ if self.validate(name): self.searcher.add_file(path, name) def remove_file(self, path, name): self.searcher.remove_file(path, name) def remove_directory(self, path): self.searcher.remove_directory(path) def walk_directory(self, root): """ From a give root of a tree this method will walk through ever branch and return a generator. """ if os.path.isdir(root): names = os.listdir(root) for name in names: try: file_stat = os.lstat(os.path.join(root, name)) except os.error: continue if stat.S_ISDIR(file_stat.st_mode): self.add_directory(os.path.join(root, name)) else: if not stat.S_ISLNK(file_stat.st_mode): self.add_file(root, name) def finish(self): wd = self.watch_manager.get_wd(self.searcher.current_root) self.watch_manager.rm_watch(wd, rec=True) self.notifier.stop() self._thread_pool.joinAll(waitForTasks=False) def validate(self, name): # Check to make sure the file not in the ignore list for ignore_re in self._exclude_regexs: if ignore_re.match(name): log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name) return False log.debug("[WalkDirectoryThread] # Passed %s", name) return True
class FileInputWatcher: '''controlling class for file monitoring''' def __init__(self, dir_to_watch, queue): print 'FileInputWatcher Initialized' # SBB20090903 Adding debugging capability, not processing multiple file drops into multiple directories. if settings.DEBUG: print '*************Debugging On*************' self.queue = queue self.dir_to_watch = dir_to_watch # make a notifier (nothing) self.notifier1 = None self.notifier2 = None def monitor(self): '''The command to start monitoring a directory or set of them.''' print 'Monitoring Directories: %s' % self.dir_to_watch print "Watching started at %s" % (time.asctime()) if osiswin32: print 'Watching win32 OS' return self.watch_win32(self.dir_to_watch) else: print 'Watching POSIX OS' #if settings.DEBUG: #print 'sending to self.watch_posix_start()' self.watch_posix_start() #if settings.DEBUG: #print "It returned from self.watch_posix_start()!" #print "self.watch_posix_start() returned with value", result return True def stop_monitoring(self): '''os independent method to stop monitoring, but only posix uses it.''' #print "self.notifier1.started", self.notifier1.__getattribute__('started') if isinstance(self.notifier1, ThreadedNotifier): #if settings.DEBUG: #print "self.notifier1 is an instance" if isinstance(self.notifier2, ThreadedNotifier): #if settings.DEBUG: #print "self.notifier2 is an instance" self.watch_posix_stop() else: if settings.DEBUG: print "notifiers were not instantiated, so not calling self.watch_posix_stop() again" print 'Done Monitoring' def watch_win32(self, dir_to_watch): '''os-specific watch command''' # Loop forever, listing any file changes. The WaitFor... will # time out every half a second allowing for keyboard interrupts # to terminate the loop. files_added = [] old_path_contents = [] new_path_contents = [] cnt = 0 try: while 1: cnt += 1 #print 'Watching %s' % cnt #old_path_contents = os.listdir(dirToWatch) for item in dir_to_watch: change_handle = win32file.FindFirstChangeNotification(item, 0, win32con.FILE_NOTIFY_CHANGE_FILE_NAME) old_path_contents.append(os.listdir(item)) result = win32event.WaitForSingleObject(change_handle, 500) # If the WaitFor... returned because of a notification (as # opposed to timing out or some error) then look for the # changes in the directory contents. if result == win32con.WAIT_OBJECT_0: #new_path_contents = os.listdir(dirToWatch) # build the new list with all the files from all dirs for item in dir_to_watch: new_path_contents.append(os.listdir(item)) files_added = [f for f in new_path_contents if not f in old_path_contents] #files_deleted = [f for f in old_path_contents if not f in new_path_contents] if files_added: print print time.asctime () print "Added:", files_added or "Nothing" return files_added #print "Deleted:", files_deleted or "Nothing" win32file.FindNextChangeNotification(change_handle) except KeyboardInterrupt: return [] #finally: # win32file.FindCloseChangeNotification(change_handle) def watch_posix_start(self): '''os-specific command to watch''' # test to see if we already have a notifier object, if not, make it, otherwise we are already watching a set of folders if self.notifier1 == None and self.notifier2 == None: try: pyinotify.compatibility_mode() print 'pyinotify running in compatibility mode' except: print 'pyinotify running in standard mode' try: #mask = EventsCodes.IN_CREATE |EventsCodes.IN_MOVED_TO mask = pyinotify.ALL_EVENTS #ECJ20100831 Reason why we have two threads: it never returns control ever to the main loop if only one thread, so no ctrl+c #The second thread is a dummy, so it performs switching/relinquishing control #Eventually, we want to watch many folders, so that is why we are using the ThreadedNotifier, versus the recommended Notifier. #Even then, Notifier is still probably the way to go, but we'll use this as long as it works, because then you don't have to poll/While loop # Thread #1 watch_manager1 = WatchManager() self.notifier1 = ThreadedNotifier(watch_manager1, EventHandler(self.queue)) self.notifier1.start() print 'Starting the threaded notifier on ', self.dir_to_watch watch_manager1.add_watch(self.dir_to_watch, mask) # Thread #2 watch_manager2 = WatchManager() self.notifier2 = ThreadedNotifier(watch_manager2, EventHandlerDummy(self.queue)) self.notifier2.start() #just watch any place, but don't remove this or Ctrl+C will not work watch_manager2.add_watch(settings.BASE_PATH, mask) if settings.DEBUG: print "both notifiers started" except KeyboardInterrupt: print "Keyboard Interrupt in notifier" self.notifier1.stop() self.notifier2.stop() return except NameError: self.notifier1.stop() self.notifier2.stop() return ['POSIX Watch Error'] except: print "General exception caught within notifier while loop, stopping both notifiers now" self.notifier1.stop() self.notifier2.stop() # SBB20090903 Turn on verbose mode self.notifier1.VERBOSE = settings.DEBUG print "returning to calling function" return True def watch_posix_stop(self): 'os specific call to stop monitoring' print 'Stopping the threaded notifiers.' self.notifier1.stop() print "stopped self.notifier1.stop()" self.notifier2.stop() print "stopped self.notifier2.stop()" return
def stop(self, *args, **kwargs): self._default_proc_fun.stop() ThreadedNotifier.stop(self, *args, **kwargs)
if watch: # set up a directory watch wm = WatchManager() mask = pyinotify.IN_CLOSE_WRITE notifier = ThreadedNotifier(wm, FileCloseCb()) notifier.start() wdd = wm.add_watch(inputDir, mask, rec=False) # process user input run = True print 'Type "QUIT" to exit' while (run): command = raw_input("> ") if command == 'QUIT': run = False notifier.stop() else: # anonymize all files in the inputDir for f in os.listdir(inputDir): if f.endswith(".ds"): while threading.active_count() > threads: time.sleep(5) try: print f #anonymizeFile(inputDir, f) t = FS_Walk_Anonymizer(inputDir, f) t.start() except: logging.debug( "ERROR: [{}] Unable to start the anonymization thread for {}" .format(
class ModuleWatcher(ProcessEvent): """Automatically reload any modules or packages as they change""" def __init__(self, serverhandler): self.wm = WatchManager() self.notifier = ThreadedNotifier(self.wm, self) self.moduleMap = {} self.watchDescriptorMap = {} self.serverhandler = serverhandler self.shuttingDown = False flags = EventsCodes.ALL_FLAGS path, watchDescriptor = list( self.wm.add_watch("commands/", flags["IN_CREATE"] | flags["IN_DELETE"]).items() )[0] self.watchDescriptorMap[path] = watchDescriptor def _watch_file(self, file_name, modulepath, module): """Add a watch for a specific file, and map said file to a module name""" file_name = os.path.realpath(file_name) self.moduleMap[file_name] = modulepath, module flags = EventsCodes.ALL_FLAGS a = self.wm.add_watch(file_name, flags["IN_MODIFY"]) if a == {}: return path, watchDescriptor = list(a.items())[0] self.watchDescriptorMap[path] = watchDescriptor def watch_module(self, name): """Load a module, determine which files it uses, and watch them""" if imp.is_builtin(name) in [-1, 1]: # Pretty pointless to watch built-in modules return f, pathname, description = imp.find_module(name) try: mod = imp.load_module(name, f, pathname, description) if f: self._watch_file(f.name, name, mod) else: for root, dirs, files in os.walk(pathname): for filename in files: fpath = os.path.join(root, filename) if fpath.endswith(".py"): self._watch_file(fpath, name, mod) finally: if f: f.close() def start_watching(self): """Start the pyinotify watch thread""" if self.notifier is not None: self.notifier.start() def stop_watching(self): """Stop the pyinotify watch thread""" self.shuttingDown = True if self.notifier is not None: self.notifier.stop() def process_default(self, event): if event.maskname == "IN_IGNORED": # TODO: Watch IN_IGNORED properly. self.process_IN_IGNORED(event) else: print(event) def process_IN_CREATE(self, event): """A file has been created.""" if event.name.endswith(".py"): self.watch_module("commands/" + event.name[:-3]) modpath, modname = self.moduleMap[event.pathname] f, pathname, description = imp.find_module(modpath) try: module = imp.load_module(modpath, f, pathname, description) print( "Loaded " + event.pathname + " which provides the following command(s): " + ", ".join(module.getCommandNames()) ) for command in module.getCommandNames(): self.serverhandler.commandMap[command] = module finally: if f: f.close() def process_IN_DELETE(self, event): """A file has been deleted.""" if (event.name.endswith(".py") or event.name.endswith(".pyc")) and event.pathname in self.moduleMap: commands = self.moduleMap[event.pathname][1].getCommandNames() print("Unloaded " + event.name + " which provided the following command(s): " + ", ".join(commands)) del self.watchDescriptorMap[event.pathname] for command in commands: del self.serverhandler.commandMap[command] del self.moduleMap[event.pathname] def process_IN_IGNORED(self, event): if not os.path.exists(event.path): return elif not self.shuttingDown: if event.path.endswith(".py"): modname = os.path.relpath(event.path)[:-3] elif event.path.endswith(".pyc"): modname = os.path.relpath(event.path)[:-4] else: return try: imp.find_module(modname) except ImportError as ie: print("While trying to load " + modname + ", received this error: " + ie) return self.process_IN_MODIFY(event) del self.watchDescriptorMap[event.path] del self.moduleMap[event.path] self.watch_module(modname) def process_IN_MODIFY(self, event): """A file has been changed""" modpath, modname = self.moduleMap[event.path] f, pathname, description = imp.find_module(modpath) try: module = imp.load_module(modpath, f, pathname, description) print( "Reloaded " + event.path + " which provides the following command(s): " + ", ".join(module.getCommandNames()) ) for command in module.getCommandNames(): self.serverhandler.commandMap[command] = module finally: if f: f.close()
class Server(Command): description = 'Run the calibre server in development mode conveniently' MONOCLE_PATH = '../monocle' def rebuild_monocole(self): subprocess.check_call(['sprocketize', '-C', self.MONOCLE_PATH, '-I', 'src', 'src/monocle.js'], stdout=open('resources/content_server/read/monocle.js', 'wb')) def launch_server(self): print 'Starting server...\n' with self.lock: self.rebuild_monocole() self.server_proc = p = subprocess.Popen(['calibre-server', '--develop'], stderr=subprocess.STDOUT, stdout=self.server_log) time.sleep(0.2) if p.poll() is not None: print 'Starting server failed' raise SystemExit(1) return p def kill_server(self): print 'Killing server...\n' if self.server_proc is not None: with self.lock: if self.server_proc.poll() is None: self.server_proc.terminate() while self.server_proc.poll() is None: time.sleep(0.1) def watch(self): if wm is not None: self.notifier = ThreadedNotifier(wm, ProcessEvents(self)) self.notifier.start() self.wdd = wm.add_watch(os.path.abspath('src'), mask, rec=True) def reload_browser(self, delay=0.1): time.sleep(delay) try: t = telnetlib.Telnet('localhost', 4242) t.read_until("repl>") t.write('BrowserReload();') t.read_until("repl>") t.close() except: print 'Failed to reload browser' import traceback traceback.print_exc() def run(self, opts): self.lock = RLock() tdir = tempfile.gettempdir() logf = os.path.join(tdir, 'calibre-server.log') self.server_log = open(logf, 'ab') self.prompt = 'Press Enter to kill/restart server. Ctrl+C to quit: ' print 'Server log available at:', logf print self.watch() first = True while True: self.launch_server() if not first: self.reload_browser() first = False try: raw_input(self.prompt) except: print self.kill_server() break else: self.kill_server() print if hasattr(self, 'notifier'): self.notifier.stop()
class Repo(ProcessEvent): def __init__(self, filename=None, config={}, **kwargs): """ Load a repository metadata structure filename: the name of a .chunker file to load, containing either full state, or a useful subset **kwargs: a basic .chunker data structure config: optional dictionary of extra info. Keys: username - for change log hostname - for change log """ self.notifier = None if not filename and not kwargs: raise Exception("Repo has no initialisation data") struct = {} if filename and os.path.exists(filename): try: data = gzip.open(filename).read() except: data = open(filename).read() struct.update(json.loads(data)) struct.update(kwargs) self.config = config self.name = struct.get("name") or os.path.basename(struct.get("root")) or os.path.splitext(os.path.basename(filename or ""))[0] self.root = struct.get("root") or os.path.join(os.path.expanduser("~/Downloads"), self.name) self.type = struct.get("type", "share") # static / share self.uuid = struct.get("uuid", sha256(uuid.uuid4())) self.key = struct.get("key", None) # for encrypting / decrypting chunks self.peers = struct.get("peers", []) self.files = dict([ (filename, File.from_struct(self, filename, data)) for filename, data in struct.get("files", {}).items() ]) # if we're creating a new static chunkfile, then add our local files to the chunkfile # should this be in start()? if (self.type == "static" and not self.files): self.__add_local_files() def to_struct(self, state=False): """ Serialise the repository into a JSON-compatible dictionary """ data = { "name": self.name, "type": self.type, "uuid": self.uuid, "key": self.key, "files": dict([ (filename, file.to_struct(state=state)) for filename, file in self.files.items() ]) } if state: data.update({ "peers": [ peer.to_struct(state=state) for peer in self.peers ], "root": self.root, }) return data def __repr__(self): return "Repo(%r, %r, %r, %r)" % (self.type, self.uuid, self.root, self.name) def save_state(self): """ Save the repository state to the default state location (eg ~/.config/chunker/<uuid>.state on unix) """ self.save(get_config_path(self.uuid + ".state"), state=True, compress=True) def remove_state(self): p = get_config_path(self.uuid + ".state") if os.path.exists(p): os.unlink(p) def save(self, filename=None, state=False, compress=False): """ Export the repository state (ie, write Repo.to_struct() to a JSON file) filename: where to save the state to state: whether to save active state, eg which chunks are currently downloaded True -> useful for an app to exit and re-open on the same PC later False -> useful for exporting the minimal amount of info to get a new node to join the swarm compress: whether or not to run the data through gzip (disabling this can make debugging easier) """ struct = self.to_struct(state=state) if compress: fp = gzip.open(filename, "w") data = json.dumps(struct) else: fp = open(filename, "w") data = json.dumps(struct, indent=4) fp.write(data) fp.close() def log(self, msg): log.info("[%s] %s" % (self.name, msg)) ################################################################### # Metadata ################################################################### def __add_local_files(self): for dirpath, dirnames, filenames in os.walk(self.root): for filename in filenames: path = os.path.join(dirpath, filename) relpath = self.__relpath(path) # look for # - files that we haven't seen before # - files with newer timestamps than our latest known version # # note that if a file has new content, but the timestamp is # unchanged since we last saw it, we won't add a new version, # but rather treat the current version as corrupt if ( relpath not in self.files or ts_round(os.stat(path).st_mtime) > self.files[relpath].timestamp ): self.update(relpath, { "versions": [{ "timestamp": ts_round(os.stat(path).st_mtime), "chunks": None, }] }) for file in self.files.values(): # "not supposed to be deleted, but it is" -> it has been # deleted while we weren't looking. if not file.deleted and not os.path.exists(file.fullpath): # We don't know when it was deleted, so add the deletion # tag as just after the last modification (ie, mark that # version as deleted, but any newer remote version takes # precedence) self.update(file.filename, { "versions": [{ "timestamp": ts_round(file.timestamp + 1), "chunks": [], "deleted": True, }] }) def update(self, filename, filedata): """ Update the repository with new metadata for a named file """ file = File.from_struct(self, filename, filedata) if file.filename not in self.files: self.files[file.filename] = file else: self.files[file.filename].versions.extend(file.versions) self.files[file.filename].versions.sort() if file.deleted: file.log("deleted") if os.path.exists(file.fullpath): os.unlink(file.fullpath) else: if os.path.exists(file.fullpath): file.log("updated") else: with open(file.fullpath, "a"): if file.is_complete(): # mark as finished already os.utime(file.fullpath, (file.timestamp, file.timestamp)) else: # mark as incomplete os.utime(file.fullpath, (0, 0)) file.log("created") self.save_state() ################################################################### # Networking ################################################################### def add_peer(self, peer): if peer not in self.peers: self.log("Found new peer: %r" % (peer, )) self.peers.append(peer) ################################################################### # Chunks ################################################################### def get_missing_chunks(self): """ Get a list of missing chunks """ l = [] for file in self.files.values(): l.extend(file.get_missing_chunks()) return l def get_known_chunks(self): """ Get a list of known chunks """ l = [] for file in self.files.values(): l.extend(file.get_known_chunks()) return l def add_chunk(self, chunk_id, data): """ Notify the repository that a new chunk is available (probably freshly downloaded from the network) """ self.log("Trying to insert chunk %s into files" % chunk_id) for chunk in self.missing_chunks: if chunk.id == chunk_id: chunk.save_data(data) def self_heal(self, known_chunks=None, missing_chunks=None): """ Try to use known chunks to fill in gaps """ if known_chunks is None: known_chunks = self.get_known_chunks() if missing_chunks is None: missing_chunks = self.get_missing_chunks() heal(known_chunks, missing_chunks) ################################################################### # Crypto ################################################################### def encrypt(self, data): if self.key: c = AES.new(self.key) data = c.encrypt(data) return data def decrypt(self, data): if self.key: c = AES.new(self.key) data = c.decrypt(data) return data ################################################################### # File system monitoring ################################################################### def start(self): """ Start monitoring for file changes """ if self.type == "share": self.log("Checking for files updated while we were offline") self.__add_local_files() self.log("Watching %s for file changes" % self.root) watcher = WatchManager() watcher.add_watch(self.root, ALL_EVENTS, rec=True, auto_add=True) self.notifier = ThreadedNotifier(watcher, self) self.notifier.daemon = True self.notifier.start() else: self.log("Not watching %s for file changes" % self.root) # self.self_heal() def netcomms(): while True: # select()'ing three empty lists is an error on windows if not self.peers: sleep(5) continue rs, ws, xs = select(self.peers, self.peers, [], 0) for r in rs: packet = r.recv() r.last_pong = time() for w in ws: if w.last_ping < time() - 30 and w.last_pong < time() - 30: data = json.dumps({"cmd": "get-status", "since": w.last_update}) w.send(data) w.last_ping = time() for peer in self.peers: if peer.last_pong < time() - 300: log.info("Peer no longer reachable - %r" % peer) peer.last_pong = time() + 10000 # if there was nothing to do, sleep for a bit # (if there was something to do, immediately go back for more) if not rs: sleep(1) nc = Thread(target=netcomms, name="NetComms[%s]" % self.name) nc.daemon = True nc.start() def stop(self): """ Stop monitoring for file changes """ if self.notifier: self.log("No longer watching %s for file changes" % self.root) self.notifier.stop() self.notifier = None def __relpath(self, path): base = os.path.abspath(self.root) path = os.path.abspath(path) return path[len(base)+1:] def process_IN_CREATE(self, event): if os.path.isdir(event.pathname): return path = self.__relpath(event.pathname) self.update(path, filedata={ "versions": [{ "timestamp": int(os.stat(event.pathname).st_mtime), "chunks": None, "username": self.config.get("username"), "hostname": self.config.get("hostname"), }] }) # def process_IN_MODIFY(self, event): # if os.path.isdir(event.pathname): # return # path = self.__relpath(event.pathname) # self.update(path, filedata={ # "versions": [{ # "deleted": False, # "timestamp": int(os.stat(event.pathname).st_mtime), # "chunks": None, # "username": self.config.get("username"), # "hostname": self.config.get("hostname"), # }] # }) def process_IN_DELETE(self, event): if os.path.isdir(event.pathname): return path = self.__relpath(event.pathname) self.update(path, filedata={ "versions": [{ "deleted": True, "timestamp": int(time()), "chunks": [], "username": self.config.get("username"), "hostname": self.config.get("hostname"), }] })
class ExcelFileWatcher(object): class Process(ProcessEvent): #def __init__(self, options): # self.regex = re.compile(options.regex) # self.script = options.script def __init__(self, folderpath, event_callback): #self.regex = re.compile(options.regex) #self.script = options.script self.event_callback = event_callback self.folderpath = folderpath def process_IN_CREATE(self, event): target = os.path.join(event.path, event.name) print "create event happened!" if os.path.isdir(target): raise Reload() def process_IN_DELETE(self, event): print "delete event happened" raise Reload() def process_IN_CLOSE_WRITE(self, event): target = os.path.join(event.path, event.name) #print "target = ", target #print 'event.path = ', event.path #print 'event.name = ', event.name self.event_callback(event.path, event.name) #if self.regex.match(target): # args = self.script.replace('$f', target).split() # os.system("clear") # sys.stdout.write("executing script: " + " ".join(args) + "\n") # subprocess.call(args) # sys.stdout.write("------------------------\n") def __init__(self, folderpath, event_callback): self.event_callback = event_callback self.folderpath = folderpath def start_watch_loop(self): wm = WatchManager() process = self.Process(self.folderpath, self.event_callback) #options) self.notifier = ThreadedNotifier(wm, process) #Notifier(wm, process) #notifier = Notifier(wm) mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE #wdd = wm.add_watch(options.directory, mask, rec=True) #wm.add_watch('./excelsrc', mask, rec=True) wm.add_watch('./' + self.folderpath, mask, rec=True) self.notifier.start() """ while True: wm = WatchManager() process = self.Process(self.folderpath, self.event_callback) #options) notifier = ThreadedNotifier(wm, process) #Notifier(wm, process) #notifier = Notifier(wm) mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE #wdd = wm.add_watch(options.directory, mask, rec=True) #wm.add_watch('./excelsrc', mask, rec=True) wm.add_watch('./'+self.folderpath, mask, rec=True) try: while True: print '.' notifier.process_events() print '+' if notifier.check_events(): notifier.read_events() print '-' except Reload: pass except KeyboardInterrupt: notifier.stop() break """ def stop_watch_loop(self): self.notifier.stop()
class FileWatcher(ProcessEvent): """ FileWatcher -> Starts an INotify thread to watch a directory for file changes. """ def __init__(self): """ FileWatcher(directory) -> Watch the directory for changes. """ if not pyinotify: raise Exception("pyinotify is not loaded.") super(FileWatcher, self).__init__() self._file_callback_dict = {} self._watch_manager = WatchManager() self._events_mask = EventsCodes.ALL_FLAGS['IN_MODIFY'] self._notifier = ThreadedNotifier(self._watch_manager, self) self._notifier.setDaemon(True) @classmethod def check(cls): """ Returns true if pyinotify is loaded otherwise false. """ return pyinotify def is_running(self): """ Returns a boolean indecating the state of the notifier. """ return self._notifier.isAlive() def start(self): """ Start the notifier thread. """ self._notifier.start() def stop(self): """ Stop the notifier thread. """ self._notifier.stop() def add_directory(self, directory): """ add_directory(directory) -> Add a directory to watch. """ dir_watch = self._watch_manager.add_watch(directory, self._events_mask, rec=True) def remove_directory(self, directory): """ remove_directory(directory) -> Remove a directory from the watch. """ self._watch_manager.rm_watch(directory, rec=True) def has_file(self, filename): """ Returns a boolean indecating if the file is being watched. """ return filename in self._file_callback_dict def add_file(self, filename, callback, *user_data): """ add_file(filename, callback, *user_data) -> Add a file to watch with its callback and optional user_data. """ self._file_callback_dict[filename] = (callback, user_data) def remove_file(self, filename): """ remove_file(filename) -> Remove the file from the watch list. """ return self._file_callback_dict.pop(filename, None) def process_IN_MODIFY(self, event): """ Process modify events. """ filename = os.path.join(event.path, event.name) callback_data = self._file_callback_dict.get(filename, ()) if callback_data: callback = callback_data[0] glib.idle_add(callback, callback_data[1:])
class INotifier(object): """ Class providing an easy wrapper for listing to INotify kernel events. """ Write, Delete = range(2) def __init__(self, path): """ Initialisation. @param path (str) Full path to watch. """ self.path = path if os.path.isdir(self.path): self.path_dir = self.path self.path_file = None else: self.path_dir = os.path.dirname(self.path) self.path_file = os.path.basename(self.path) self.callbacks = {} wm = WatchManager() wm.add_watch(self.path_dir, pyinotify.ALL_EVENTS) self.inotifier = ThreadedNotifier(wm, self.__processINotify) self.inotifier.start() def __del__(self): """ Destruction. Unload (i.e. stop) the inotifier. """ self.unload() def unload(self): """ Call this to stop listening for events. Should be called on shutdown. """ self.inotifier.stop() def __processINotify(self, event): """ Called when an INotify was received. Call the applicable callback method based on the event type. """ if event.mask == pyinotify.IN_DELETE or event.mask == pyinotify.IN_MOVED_FROM: if self.path_file == None or (self.path_file and event.name == self.path_file): for c in self.__getCallbacks(INotifier.Delete): c(event) elif event.mask == pyinotify.IN_CLOSE_WRITE or event.mask == pyinotify.IN_MOVED_TO: if self.path_file == None or (self.path_file and event.name == self.path_file): for c in self.__getCallbacks(INotifier.Write): c(event) def __getCallbacks(self, type): """ Get the current callback methods for the given type. @param type (INotifier.Write or INotifier.Delete) Type of callback get. """ return self.callbacks.get(type, []) def addCallback(self, type, callback): """ Add a callback function to handle an event. @param type (INotifier.Write or INotifier.Delete) Type of callback to add. Write is called on file writes, Delete on file deletion. @param callback (method) Method to call, should accept one parameter: the pyinotify event. """ if not type in self.callbacks: self.callbacks[type] = set() self.callbacks[type].add(callback)
class FileWatcher(ProcessEvent): """ FileWatcher -> Starts an INotify thread to watch a directory for file changes. """ def __init__(self): """ FileWatcher(directory) -> Watch the directory for changes. """ if not pyinotify: raise Exception("pyinotify is not loaded.") super(FileWatcher, self).__init__() self._file_callback_dict = {} self._watch_manager = WatchManager() self._events_mask = EventsCodes.ALL_FLAGS['IN_MODIFY'] self._notifier = ThreadedNotifier(self._watch_manager, self) self._notifier.setDaemon(True) @classmethod def check(cls): """ Returns true if pyinotify is loaded otherwise false. """ return pyinotify def is_running(self): """ Returns a boolean indecating the state of the notifier. """ return self._notifier.isAlive() def start(self): """ Start the notifier thread. """ self._notifier.start() def stop(self): """ Stop the notifier thread. """ self._notifier.stop() def add_directory(self, directory): """ add_directory(directory) -> Add a directory to watch. """ dir_watch = self._watch_manager.add_watch(directory, self._events_mask, rec=True) def remove_directory(self, directory): """ remove_directory(directory) -> Remove a directory from the watch. """ self._watch_manager.rm_watch(directory, rec=True) def has_file(self, filename): """ Returns a boolean indecating if the file is being watched. """ return filename in self._file_callback_dict def add_file(self, filename, callback, *user_data): """ add_file(filename, callback, *user_data) -> Add a file to watch with its callback and optional user_data. """ self._file_callback_dict[filename] = (callback, user_data) def remove_file(self, filename): """ remove_file(filename) -> Remove the file from the watch list. """ return self._file_callback_dict.pop(filename, None) def process_IN_MODIFY(self, event): """ Process modify events. """ filename = os.path.join(event.path, event.name) callback_data = self._file_callback_dict.get(filename, ()) if callback_data: callback = callback_data[0] glib.idle_add(callback, callback_data[1:])
class AutoLibraryUpdate ( EventPlugin ): # Tell QL about our plugin: PLUGIN_ID = "Automatic library update" PLUGIN_NAME = "Automatic library update" PLUGIN_DESC = "Keep your quodlibet library up-to-date with inotify." PLUGIN_VERSION = "0.1" library = None wm = None tn = None event_handler = None running = False # Set everything up: def __init__( self ): from quodlibet.library import library as library self.library = library # enabled hook, get everything cooking: def enabled( self ): if not self.running : self.wm = WatchManager() # the event handler will do the actual event processing, # see the ALE class below for details self.event_handler = self.ALE( self ) # we'll use a threaded notifier flagged as a daemon thread # so that it will stop when QL does self.tn = ThreadedNotifier( self.wm, self.event_handler ) self.tn.daemon = True self.tn.start() # mask for watched events: FLAGS=EventsCodes.ALL_FLAGS mask = FLAGS['IN_DELETE'] | FLAGS['IN_CLOSE_WRITE']\ | FLAGS['IN_MOVED_FROM'] | FLAGS['IN_MOVED_TO'] # watch paths in scan_list: for path in self.scan_list(): log ( 'Adding watch: for ' + path ) self.wm.add_watch( path, mask, rec=True ) self.running = True # disable hook, stop the notifier: def disabled( self ): if self.running: self.running = False self.tn.stop() # find list of directories to scan def scan_list( self ): return config.get( "settings", "scan").split(":") # auto-library-event class, handles the events class ALE( ProcessEvent ): def __init__( self, alu ): self._alu = alu log("alu : %s" % alu ) # process close-write event ( copy, new file etc ) def process_IN_CLOSE_WRITE(self, event): glib.idle_add( self.add_event, event ) # process moved-to event: def process_IN_MOVED_TO( self, event ): log( 'moved event' ) glib.idle_add( self.add_event, event ) # general we think we added something callback: def add_event ( self, event ): lib = self._alu.library path = os.path.join(event.path, event.name) if event.dir: for path, dnames, fnames in os.walk(path): for filename in fnames: fullfilename = os.path.join(path, filename) lib.add_filename(fullfilename) else: item = lib.add_filename ( path ) log( '%s added to library' % item ) return False # process delete event def process_IN_DELETE(self, event): glib.idle_add( self.check_event, event ) # process the moved-from event: def process_IN_MOVED_FROM( self, event ): glib.idle_add( self.check_event, event ) # general "check to see if it's still there" callback: def check_event( self, event ): lib = self._alu.library path = os.path.join(event.path, event.name) log( 'check (%s) event for %s' % (event.path, path) ) if event.dir: log( 'scan? %s ' % path ) path_re = re.compile('^' + path) to_reload = [] for i in lib._contents: if path_re.match( i ): try: item = lib.__getitem__( i ) except: pass if item: to_reload.append(item) for item in to_reload: lib.reload( item ) else: item = lib.__getitem__( path ) log( 'removing %s from library' % item ) if item: lib.reload(item) return False
class Root(ProcessEvent): watchThread = None notifier = None scanThread = None continueScanning = True peers = [] def __init__(self, path): self.path = path; self.setStatus("???") self.peers.append(Peer("dummy", self.path)) self.watchManager = WatchManager() self.watchMask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] self.contents = {} self.load() self.startWatching() self.startRescan() self.save() def setStatus(self, msg): print "root: " + self.path + " status: " + msg self.status = msg; def save(self): print "Saving to: ", self.getContentsFilepath() try: handle = file(self.getContentsFilepath(), 'w+') handle.write(json.dumps(self.contents, sort_keys=True, indent=4)) handle.close() except Exception as e: print e return self.getContentsFilepath() def load(self): try: handle = file(self.getContentsFilepath(), 'r') jsonRaw = handle.read() jsonContents = json.loads(jsonRaw) print "Contents of json load:" + str(len(jsonContents)) + " items" except Exception as e: print "Load failed:" + str(e) return self.contents = jsonContents def getContentsFilepath(self): contentsDir = os.path.join(os.path.join(self.path, ".ovress")) if not os.path.isdir(contentsDir): os.mkdir(contentsDir) return os.path.join(contentsDir, "contents.json") def startWatching(self): if self.watchThread == None or not self.watchThread.is_alive(): self.watchThread = Thread(target = self.doWatch, name = "watchThread") self.watchThread.start() def doWatch(self): print("watching path: " + str(self.path)); self.wdd = self.watchManager.add_watch(self.path, self.watchMask, rec=True) self.notifier = ThreadedNotifier(self.watchManager, default_proc_fun=self) self.notifier.start() def startRescan(self): if self.scanThread == None or not self.scanThread.is_alive(): self.scanThread = Thread(target = self.doScan, name = "scanThread") self.scanThread.start() def doScan(self): dateStarted = datetime.datetime.now() countCached = 0 countScanned = 0 countNew = 0 self.setStatus("scanning") for dirname, subdirs, fileList in os.walk(self.path): for filename in fileList: absolutePath = os.path.join(dirname, filename) relativePath = absolutePath.replace(self.path, "") generateMetadata = True try: fileMetadata = self.contents[relativePath] if (fileMetadata['refreshed'] + 3600) > time(): self.setStatus("Using cached scan of: " + relativePath) generateMetadata = False except KeyError: pass if generateMetadata: fileMetadata = self.getFileMetadata(absolutePath) self.contents[relativePath] = fileMetadata countScanned = countScanned + 1 countNew = countNew + 1 sleep(0.1) if countNew == Config.instance.saveInterval: countNew = 0 self.save() self.setStatus("Scanned: " + str(countScanned) + ". Cached: " + str(countCached)) for peer in self.peers: peer.onScanFile(relativePath, fileMetadata) if not self.continueScanning: return self.setStatus("complete") def getFileMetadata(self, path): exists = False; filesize = 0; filetype = '?'; md5sum = "" if os.path.exists(path): exists = True filesize = os.path.getsize(path) md5sum = md5.md5(path).hexdigest() return { "exists": exists, "filesize": filesize, "type": filetype, "md5": md5sum, "refreshed": time() } def getContents(self): return self.contents def process_IN_CREATE(self, event): fullPath = os.path.join(event.path, event.name) print "Create %s" % fullPath self.contents[fullPath] = self.getFileMetadata(fullPath) def process_IN_DELETE(self, event): fullPath = os.path.join(event.path, event.name) print "Delete %s" % fullPath self.contents[fullPath] = self.getFileMetadata(fullPath) def stop(self): self.continueScanning = False if self.notifier is not None: self.notifier.stop(); self.setStatus("Stopped"); def getPeers(self): return self.peers def to_JSON(self): return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4)
class FSMonitorInotify(FSMonitor): """inotify support for FSMonitor""" EVENTMAPPING = { FSMonitor.CREATED : pyinotify.IN_CREATE, FSMonitor.MODIFIED : pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB, FSMonitor.DELETED : pyinotify.IN_DELETE, FSMonitor.MONITORED_DIR_MOVED : pyinotify.IN_MOVE_SELF, FSMonitor.DROPPED_EVENTS : pyinotify.IN_Q_OVERFLOW, } def __init__(self, callback, persistent=False, trigger_events_for_initial_scan=False, ignored_dirs=[], dbfile="fsmonitor.db"): FSMonitor.__init__(self, callback, persistent, trigger_events_for_initial_scan, ignored_dirs, dbfile) self.wm = None self.notifier = None def __fsmonitor_event_to_inotify_event(self, event_mask): """map an FSMonitor event to an inotify event""" inotify_event_mask = 0 for fsmonitor_event_mask in self.__class__.EVENTMAPPING.keys(): if event_mask & fsmonitor_event_mask: inotify_event_mask = inotify_event_mask | self.__class__.EVENTMAPPING[fsmonitor_event_mask] return inotify_event_mask def __add_dir(self, path, event_mask): """override of FSMonitor.__add_dir()""" # Perform an initial scan of the directory structure. If this has # already been done, then it will return immediately. if self.persistent: if self.trigger_events_for_initial_scan: FSMonitor.generate_missed_events(self, path, event_mask) else: self.pathscanner.initial_scan(path) event_mask_inotify = self.__fsmonitor_event_to_inotify_event(event_mask) # Use the inotify API to monitor a directory. wdd = self.wm.add_watch(path, event_mask_inotify, proc_fun=self.process_event, rec=True, auto_add=True) if wdd is None: raise MonitorError, "Could not monitor %s" % path return None else: self.monitored_paths[path] = MonitoredPath(path, event_mask, wdd) self.monitored_paths[path].monitoring = True # Generate the missed events. if self.persistent: FSMonitor.generate_missed_events(self, path) return self.monitored_paths[path] def __remove_dir(self, path): """override of FSMonitor.__remove_dir()""" if path in self.monitored_paths.keys(): wd = self.monitored_paths[path].fsmonitor_ref # TODO: figure out why this won't work, it seems this fails due to # a bug in pyinotify? #self.wm.rm_watch(wd, rec=True, quiet=True) del self.monitored_paths[path] def run(self): # Setup. Ensure that this isn't interleaved with any other thread, so # that the DB setup continues as expected. self.lock.acquire() FSMonitor.setup(self) self.process_event = FSMonitorInotifyProcessEvent(self) self.lock.release() # Set up inotify. self.wm = WatchManager() self.notifier = ThreadedNotifier(self.wm, self.process_event) self.notifier.start() while not self.die: self.__process_queues() time.sleep(0.5) self.notifier.stop() def stop(self): """override of FSMonitor.stop()""" # Let the thread know it should die. self.lock.acquire() self.die = True self.lock.release() # Stop monitoring each monitored path. for path in self.monitored_paths.keys(): self.__remove_dir(path) def __process_queues(self): # Process add queue. self.lock.acquire() if not self.add_queue.empty(): (path, event_mask) = self.add_queue.get() self.lock.release() self.__add_dir(path, event_mask) else: self.lock.release() # Process remove queue. self.lock.acquire() if not self.remove_queue.empty(): path = self.add_queue.get() self.lock.release() self.__remove_dir(path) else: self.lock.release()
def run_pipeline(conf_info): """ Run the pipeline and monitor status. """ # Fail if the run_path doesn't actually exist if not os.path.exists(conf_info.run_path): LOGGER.error('Run path does not exist: %s' \ % (conf_info.run_path)) return False # Change cwd to run_path stdout_filepath = os.path.join(conf_info.analysis_dir, 'pipeline_run_stdout.txt') stderr_filepath = os.path.join(conf_info.analysis_dir, 'pipeline_run_stderr.txt') # Create status object conf_info.createStatusObject() # Monitor file creation wm = WatchManager() mask = EventsCodes.IN_DELETE | EventsCodes.IN_CREATE event = RunEvent(conf_info) notifier = ThreadedNotifier(wm, event) notifier.start() wdd = wm.add_watch(conf_info.run_path, mask, rec=True) # Log pipeline starting LOGGER.info('STARTING PIPELINE @ %s' % (time.ctime())) # Start the pipeline (and hide!) #pipe = subprocess.Popen(['make', # '-j8', # 'recursive'], # stdout=subprocess.PIPE, # stderr=subprocess.PIPE) fout = open(stdout_filepath, 'w') ferr = open(stderr_filepath, 'w') pipe = subprocess.Popen(['make', '--directory=%s' % (conf_info.run_path), '-j8', 'recursive'], stdout=fout, stderr=ferr) #shell=True) # Wait for run to finish retcode = pipe.wait() # Clean up notifier.stop() fout.close() ferr.close() # Process stderr ferr = open(stderr_filepath, 'r') run_failed_stderr = False for line in ferr: err_status = pipeline_stderr_handler(line, conf_info) if err_status == RUN_FAILED: run_failed_stderr = True ferr.close() # Finished file check! print('RUN SUCCESS CHECK:') for key, value in event.run_status_dict.items(): print(' %s: %s' % (key, value)) dstatus = event.run_status_dict # Success or failure check status = (retcode == 0) and \ run_failed_stderr is False and \ dstatus['firecrest'] is True and \ dstatus['bustard'] is True and \ dstatus['gerald'] is True return status