class DirectoryWatcher(WatchManager): def __init__(self): WatchManager.__init__(self) def start(self): self.inotify = ThreadedNotifier(self) self.inotify.start() self.inotify.join() def stop(self): self.inotify.stop() def add_monitor_path(self, path): if path is None: Logger.error("FS: unable to monitor None directory") return False exclude1 = "^%s/conf.Windows*"%(path) exclude2 = "^%s/conf.Linux*"%(path) exc_filter = ExcludeFilter([exclude1, exclude2]) try: self.add_watch(path=path, mask=Rec.mask, proc_fun=Rec(), rec=True, auto_add=True, exclude_filter=exc_filter) except WatchManagerError, e: Logger.error("FS: unable to monitor directory %s, %s"%(path, str(e))) return False return False
class Notify(): def __init__(self): self.wm = WatchManager() self.pe = ProcessNotifyEvents() self.notifier = ThreadedNotifier(self.wm, self.pe) self.notifier.start() self.path = None #thread.start_new_thread(self.jobTask, (self,)) def setNotify(self, path, cbfun): #print 'setnotify ' + path if self.path: self.wm.rm_watch(list(self.wdd.values())) self.path = path self.pe.cbfun = cbfun # ugly... #print sys.getfilesystemencoding() self.wdd = self.wm.add_watch(self.path, pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MOVED_TO | pyinotify.IN_MOVED_FROM | pyinotify.IN_MODIFY) def stop(self): if self.path: self.wm.rm_watch(list(self.wdd.values())) self.notifier.stop() def notifyThread(self): while 1: notifier.process_events() if notifier.check_events(): notifier.read_events()
def test_gutils_netcdf_to_erddap_watch(self): wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE # Convert ASCII data to NetCDF processor = Netcdf2ErddapProcessor( deployments_path=resource('slocum'), erddap_content_path=erddap_content_path, erddap_flag_path=erddap_flag_path ) notifier = ThreadedNotifier(wm, processor, read_freq=5) notifier.coalesce_events() notifier.start() wdd = wm.add_watch( netcdf_path, mask, rec=True, auto_add=True ) # Wait 5 seconds for the watch to start time.sleep(5) orig_netcdf = resource('profile.nc') dummy_netcdf = os.path.join(netcdf_path, 'profile.nc') shutil.copy(orig_netcdf, dummy_netcdf) wait_for_files(erddap_content_path, 1) wait_for_files(erddap_flag_path, 1) wm.rm_watch(wdd.values(), rec=True) notifier.stop()
def test_gutils_ascii_to_netcdf_watch(self): wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE # Convert ASCII data to NetCDF processor = Slocum2NetcdfProcessor(deployments_path=resource('slocum'), subset=False, template='trajectory', profile_id_type=2, tsint=10, filter_distance=1, filter_points=5, filter_time=10, filter_z=1) notifier = ThreadedNotifier(wm, processor) notifier.coalesce_events() notifier.start() wdd = wm.add_watch(ascii_path, mask, rec=True, auto_add=True) # Wait 5 seconds for the watch to start time.sleep(5) # Make the ASCII we are watching for merger = SlocumMerger(original_binary, ascii_path, globs=['*.tbd', '*.sbd']) merger.convert() wait_for_files(netcdf_path, 230) wm.rm_watch(wdd.values(), rec=True) notifier.stop()
def _watch(self): wm = WatchManager() wm2 = WatchManager() clusterNotifier = ThreadedNotifier(wm, \ ClustersDefinitionsChangeHandler(\ masterCallback=self.callback)) suiteNotifier = ThreadedNotifier(wm2, \ SuiteDefinitionsChangeHandler(\ masterCallback=self.callback)) clusterNotifier.start() suiteNotifier.start() local_path = '' if self.config.has_option(self.repo, 'local_path'): local_path = self.config.get(self.repo, 'local_path') else: LOGGER.error('No local path defined for repository %s' % self.repo) if not self.config.has_option(self.repo, 'cluster_defs_path'): clustdir = local_path + os.sep + 'clusters' else: clustdir = local_path + os.sep + self.config.get(self.repo, 'cluster_defs_path') if not self.config.has_option(self.repo, 'suite_defs_path'): suitedir = local_path + os.sep + 'test-suites' else: suitedir = local_path + os.sep + self.config.get(self.repo, 'suite_defs_path') try: wm.add_watch(clustdir, self.mask, rec=True, quiet=False) wm2.add_watch(suitedir, self.mask, rec=True, quiet=False) except WatchManagerError, e: LOGGER.error(e)
def start_watching(): class OnEvent(ProcessEvent): def __init__(self): ProcessEvent(self) self.timer = None def process_default(self, event): accepted = [ pyinotify.IN_MODIFY, pyinotify.IN_MOVE_SELF, pyinotify.IN_MOVED_FROM, pyinotify.IN_MOVED_TO ] x = [x for x in accepted if event.mask & x == event.mask] if len(x) > 0: print "default: %-20s %s" % (os.path.join( event.path, event.name), event.maskname) if self.timer: self.timer.cancel() self.timer = None self.timer = threading.Timer(0.2, do_post) self.timer.start() wm = WatchManager() mask = pyinotify.ALL_EVENTS proc = OnEvent() notifier = ThreadedNotifier(wm, proc) notifier.start() wdd = wm.add_watch('.git', mask, rec=True, auto_add=True)
class DirectoryWatcher(WatchManager): def __init__(self): WatchManager.__init__(self) def start(self): self.inotify = ThreadedNotifier(self) self.inotify.start() self.inotify.join() def stop(self): self.inotify.stop() def add_monitor_path(self, path): if path is None: Logger.error("FS: unable to monitor None directory") return False exclude1 = "^%s/conf.Windows*" % (path) exclude2 = "^%s/conf.Linux*" % (path) exc_filter = ExcludeFilter([exclude1, exclude2]) try: self.add_watch(path=path, mask=Rec.mask, proc_fun=Rec(), rec=True, auto_add=True, exclude_filter=exc_filter) except WatchManagerError, e: Logger.error("FS: unable to monitor directory %s, %s" % (path, str(e))) return False return False
def test_gutils_binary_to_ascii_watch(self): wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE # Convert binary data to ASCII processor = Slocum2AsciiProcessor( deployments_path=resource('slocum'), ) notifier = ThreadedNotifier(wm, processor) notifier.coalesce_events() notifier.start() wdd = wm.add_watch( binary_path, mask, rec=True, auto_add=True ) # Wait 5 seconds for the watch to start time.sleep(5) gpath = os.path.join(original_binary, '*.*bd') # Sort the files so the .cac files are generated in the right order for g in sorted(glob(gpath)): shutil.copy2(g, binary_path) wait_for_files(ascii_path, 32) wm.rm_watch(wdd.values(), rec=True) notifier.stop()
def watch(self): mask = pyinotify.IN_MODIFY | pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO self.__wm = WatchManager() self.__wm.add_watch(os.path.join(self.directory, *self.branch_dir), mask, rec = True) notifier = ThreadedNotifier(self.__wm, self.process_event) notifier.start()
def start_watching_disk_file(filename, handler): logger.info("start watching %s" % filename) wm = WatchManager() notifier = ThreadedNotifier(wm, PTmp(wm, filename, handler)) notifier.start() notifiers[filename] = notifier if os.path.exists(filename): handler.on_create(filename)
class PhotoWatcher(ProcessEvent): MASK = (EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CLOSE_WRITE'] | EventsCodes.ALL_FLAGS['IN_MOVED_FROM'] | EventsCodes.ALL_FLAGS['IN_MOVED_TO']) def __init__(self, db, walker, root): self.root = root self.db = db self.walker = walker self.wm = WatchManager() self.wdds = [] def Watch(self): self.notifier = ThreadedNotifier(self.wm, self) self.notifier.start() self.wdds.append(self.wm.add_watch(self.root, self.MASK, rec=True)) # add soft link sub-folders for dirname, dirnames, _filenames in os.walk(self.root, followlinks=True): for d in dirnames: path = os.path.join(dirname, d) if os.path.islink(path): self.wdds.append( self.wm.add_watch(os.path.realpath(path), self.MASK, rec=True)) def Stop(self): self.notifier.stop() def process_IN_DELETE(self, event): self.db.DeletePhoto(os.path.join(event.path, event.name)) def process_IN_MOVED_FROM(self, event): self.process_IN_DELETE(event) def process_IN_MOVED_TO(self, event): full_path = os.path.join(event.path, event.name) try: meta = self.walker.ReadMetadata(full_path) except Exception: return self.db.StorePhoto(full_path, meta) def process_IN_CLOSE_WRITE(self, event): full_path = os.path.join(event.path, event.name) try: meta = self.walker.ReadMetadata(full_path) except Exception: return if self.db.HasPhoto(full_path): self.db.UpdatePhoto(full_path, meta) else: self.db.StorePhoto(full_path, meta)
class FileEvent: def __init__(self, eventHandler): self.logger = logging.getLogger('FileEvent') self.wm = WatchManager() self.watches = dict() # Set the flags of the events that are to be listened to FLAGS = EventsCodes.ALL_FLAGS self.mask = FLAGS['IN_CREATE'] | FLAGS['IN_DELETE'] | FLAGS['IN_MODIFY'] | FLAGS['IN_DELETE_SELF'] # Set-up notifier self.notifier = ThreadedNotifier(self.wm, EventProcessor(eventHandler)) def startNotifyLoop(self): self.notifier.start() def stopNotifyLoop(self): self.notifier.stop() def addWatches(self, paths, mask=None): added_watches = dict() for path in paths: added_watches.update(self.addWatch(path, mask)) return added_watches # Also monitors all sub-directories of the given directory and automatically adds newly # created directories to watch. # TODO should be able to add files as well, but doesn't work atm def addWatch(self, path, mask=None): if mask is None: mask = self.mask added_watches = self.wm.add_watch(path, mask, rec=True, auto_add=True) self.watches.update(added_watches) return added_watches def removeWatch(self, path): watch_descriptor = self.wm.get_wd(path) if watch_descriptor is not None: result = self.wm.rm_watch(watch_descriptor, rec=True) # Remove the no longer active watches from the current watches dictionary for key, value in self.watches.items(): if value in result: del self.watches[key] else: result = None return result def getWatches(self): return self.watches
def schedule(self, name, event_handler, *paths): """Schedules monitoring.""" #from pyinotify import PrintAllEvents #dispatcher = PrintAllEvents() dispatcher = _ProcessEventDispatcher(event_handler=event_handler) notifier = ThreadedNotifier(self.wm, dispatcher) self.notifiers.add(notifier) for path in paths: if not isinstance(path, str): raise TypeError( "Path must be string, not '%s'." % type(path).__name__) descriptors = self.wm.add_watch(path, ALL_EVENTS, rec=True, auto_add=True) self.name_to_rule[name] = _Rule(name, notifier, descriptors) notifier.start()
def live(path, tv): chromedriver = os.path.join(os.path.split(os.path.realpath(__file__))[0],\ 'chromedriver') os.environ['webdriver.chrome.driver'] = chromedriver browsers = [ getattr(webdriver, browser.title())() \ for browser in [s.lower() for s in tv]] wm = WatchManager() notifier = ThreadedNotifier(wm, ProcessDir(browsers)) notifier.start() print('watching %s' % os.path.abspath(path)) mask = IN_MODIFY wdd = wm.add_watch(path, mask, rec=True)
def live(path, tv): chromedriver = os.path.join(os.path.split(os.path.realpath(__file__))[0],\ 'chromedriver') os.environ['webdriver.chrome.driver'] = chromedriver browsers = [ getattr(webdriver, browser.title())() \ for browser in [s.lower() for s in tv]] wm = WatchManager() notifier = ThreadedNotifier(wm, ProcessDir(browsers)) notifier.start() print('watching %s' % os.path.abspath(path)) mask = IN_MODIFY wdd = wm.add_watch(path, mask, rec=True)
def setup_inotify(self): if pyinotify is None: return False watch_manager = WatchManager() result = watch_manager.add_watch(self.__status_file, IN_MODIFY)[self.__status_file] > 0 if result: global notifier def notify_cb(event): glib.idle_add(self.check_status_cb) notifier = ThreadedNotifier(watch_manager, notify_cb) notifier.start() return result
def get_threaded_notifier(paths,db_file,flush_interval,verbose,fs_encodings=[]): wm = WatchManager() eventhandler = FileStatEventHandler(wm,db_file,flush_interval,verbose,fs_encodings) notifier = ThreadedNotifier(wm, eventhandler) notifier.start() for p in paths: if verbose: print ' Monitoring "%s" ... ' % p, wdd = wm.add_watch(p, mask, rec=True,auto_add=False) if verbose: print '[done]' return eventhandler,notifier
def run(self): logging.info( 'Thead Watcher' ) wm = WatchManager() notifier = ThreadedNotifier(wm, PTmp()) notifier.start() wdd = wm.add_watch( account_dir, 4095, rec=True, auto_add=True) #ALL_EVENTS=>4095 logging.info('Add watch on folder ' + account_dir); while True: try: notifier.process_events() if notifier.check_events(): notifier.read_events() except KeyboardInterrupt: break
class Observer: """Monitor files and notify the main program when changes happen""" def __init__(self, makeRunView): self.wm = WatchManager() self.eh = EventHandler(makeRunView) self.notifier = ThreadedNotifier(self.wm, self.eh) self.notifier.start() # Watched events self.mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE def kill(self): status = self.notifier.stop() logging.debug("Observer shut down") return status def addFile(self, fname): wdd = self.wm.add_watch(fname, self.mask, rec=True)
def setup_inotify(self): if pyinotify is None: return False watch_manager = WatchManager() result = watch_manager.add_watch(self.__status_file, IN_MODIFY)[self.__status_file] > 0 if result: global notifier def notify_cb(event): glib.idle_add(self.check_status_cb) notifier = ThreadedNotifier(watch_manager, notify_cb) notifier.start() return result
def main(): # Add a dummy message to the queue: add_msg( MsgTypes.Initialise ) # Setup the HTTP server: SocketServer.TCPServer.allow_reuse_address = True HOST, PORT = "localhost", 9000 server = ThreadedHTTPServer((HOST, PORT), MyTCPHandler) server_thread = threading.Thread(target=server.serve_forever) server_thread.daemon = True server_thread.start() # Setup the heartbeat: heartbeat_thread = threading.Thread(target=heartbeat_messages) heartbeat_thread.daemon = True heartbeat_thread.start() # Setup pyinotify so files are being watched: wm = WatchManager() notifier = ThreadedNotifier(wm, PTmp()) notifier.start() mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_ATTRIB'] |EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events #mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events wdd = wm.add_watch('../sentry.py', mask, rec=True) wdd = wm.add_watch('../', mask, rec=True) try: while True: time.sleep(5) except: # Turn off pyinotify: notifier.stop() print 'Exception caught: shutting down connections' add_msg(MsgTypes.Shutdown) time.sleep(0.5) print 'Terminating...' raise
class FileWatcher(Thread): def __init__(self, holder, uri, schedule_reader): Thread.__init__(self) self._loop = True self._error_event = Event() self._wm = WatchManager() self._holder = holder self._uri = uri self._schedule_reader = schedule_reader self._notifier = ThreadedNotifier(self._wm, _EventHandler(self._holder, self._uri, self._schedule_reader, self._error_event)) self._path, self._pattern = os.path.split(urlparse(uri).path) def start(self): """Start the file watcher """ self._notifier.start() self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY) Thread.start(self) def run(self): while self._loop: if self._error_event.wait(1): self._error_event.clear() self._notifier.stop() del self._notifier self._notifier = ThreadedNotifier( self._wm, _EventHandler(self._holder, self._uri, self._schedule_reader, self._error_event)) self._notifier.start() self._wm.add_watch( self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY) def stop(self): """Stop the file watcher """ self._notifier.stop() self._loop = False
class _PathWatcher(Queue): # iNotify watcher for directory ''' iNotify watcher object for monitor of changes in directory. ''' FLAGS = IN_MODIFY | IN_DELETE | IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO | IN_ATTRIB def __init__(self, path, exclude=None): class _EH(ProcessEvent): def process_default(self, event): _handleEvent(event) Queue.__init__(self) self._path = path self.exclude = exclude or [] _handleEvent = self.put self._wm = WatchManager() self._iNotifier = ThreadedNotifier(self._wm, _EH(), timeout=10) self._iNotifier.start() self.started = False self._watch = [] def start(self, exclude=None): if not self.started: # Add watch and start watching # Update exclude filter if it provided in call of start method self.exclude = exclude or self.exclude self._watch = self._wm.add_watch(self._path, self.FLAGS, exclude_filter=ExcludeFilter( self.exclude), auto_add=True, rec=True, do_glob=False) self.started = True def stop(self): if self.started: # Remove watch and stop watching self._wm.rm_watch(self._watch[self._path], rec=True) self.started = False def exit(self): self.stop() self._iNotifier.stop()
def lastwatch(paths, settings, dry_run=False): flags = EventsCodes.FLAG_COLLECTIONS.get('OP_FLAGS', None) if flags: mask = flags.get('IN_OPEN') | flags.get('IN_CLOSE_NOWRITE') mask |= flags.get('IN_CREATE') | flags.get('IN_MOVED_TO') else: mask = EventsCodes.IN_OPEN | EventsCodes.IN_CLOSE_NOWRITE mask |= EventsCodes.IN_CREATE | EventsCodes.IN_MOVED_TO wm = WatchManager() handler = Handler(settings, dry_run=dry_run) watcher = ThreadedNotifier(wm, handler) watcher.start() try: for path in paths: path = os.path.realpath(path) sys.stdout.write(_("Indexing %s for watching...") % path) sys.stdout.flush() wm.add_watch(path, mask, rec=True, auto_add=True) sys.stdout.write(_(" done.") + "\n") print _("You have successfully launched Lastwatch.") print "\n".join(wrap(_("The directories you have specified will be " "monitored as long as this process is running, " "the flowers are blooming and the earth " "revolves around the sun..."), 80)) # flowers to devhell ;-) handler.set_active() while True: time.sleep(1) except KeyboardInterrupt: watcher.stop() print _("LastWatch stopped.") return except Exception, err: print err
def start_daemon(path, dbpath, md_queue, fd_queue, condition): """ installs a subtree listener and wait for events """ os.nice(19) wm_auto = WatchManager() subtreemask = IN_CLOSE_WRITE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_ISDIR excludefilter = ExcludeFilter(["(.*)sqlite"]) notifier_sb = ThreadedNotifier(wm_auto, SubtreeListener(dbpath, md_queue, fd_queue, condition)) notifier_sb.start() THREADS.append(notifier_sb) wdd_sb = wm_auto.add_watch(path, subtreemask, auto_add=True, rec=True, exclude_filter=excludefilter) THREADS.append(PollAnalyzer(condition, dbpath)) THREADS[-1].start() # THREADS.append(CatalogThreadingTCPServer(("localhost", 8080), CatalogHTTPRequestHandler, dbpath)) # THREADS[-1].serve_forever() THREADS[-1].join()
def create_watcher(): from pyinotify import WatchManager, Notifier, ThreadedNotifier, \ EventsCodes, ProcessEvent, IN_CLOSE_WRITE wm = WatchManager() mask = IN_CLOSE_WRITE #| EventsCodes.IN_CREATE # watched events class PTmp(ProcessEvent): def process_IN_CLOSE_WRITE(self, event): def inner(): on_reload_event(event) gdb.post_event(inner) notifier = ThreadedNotifier(wm, PTmp()) wdd = wm.add_watch(WORKING_DIR, mask, rec=True) notifier.daemon = True # Then our atexit function will work notifier.start() def on_exited(*e): notifier.stop() import atexit atexit.register(on_exited) return (notifier, wdd)
class FilesystemWatcher: # Mask for FS events we are interested in - currently only care about # writable file streams that have been closed (i.e. a pdf has finished # uploading) mask = EventsCodes.ALL_FLAGS['IN_CLOSE_WRITE'] # This is the main watch manager provided by pyinotify # we use this to manage subscriptions _wm = WatchManager() # List of watched directories wdd = {} # create a thread-safe queue for papers that must be processed. paper_queue = Queue() def __init__(self, logger): self.logger = logger self.notifier = ThreadedNotifier( self._wm, PaperProcesser(self.logger, self.paper_queue)) def watch_directory(self, path): """set up threaded directory watcher at given path""" self._wm.add_watch(path, self.mask, rec=True) #add all files in the given directory to queue for root, dirs, files in os.walk(path): for file in files: if file.endswith("pdf") or file.endswith("xml"): self.logger.info("Adding %s to queue", file) self.paper_queue.put(("QUEUE", os.path.join(root, file))) def start(self): self.notifier.start() def stop(self): self.notifier.stop()
class Watch(object): def __init__(self, path, callback, ignore_modifications=False, latency=None): self.event_mask = EVENT_MASK if ignore_modifications else EVENT_MASK_WITH_MODIFICATIONS self._dir_queue = queue.Queue() self._root = os.path.realpath(path) self._watch_manager = WatchManager() self._processor = FileProcessEvent(directory_queue=self._dir_queue, root=self._root, callback=callback, latency=latency) self._notifier = ThreadedNotifier(self._watch_manager, self._processor) self._notifier.name = "[inotify] notifier" self._notifier.daemon = True self._notifier.start() self._watch_manager.add_watch(path, self.event_mask, rec=True, auto_add=True) def stop(self): self._notifier.stop() self._processor.stop()
class FileWatcher(Thread): def __init__(self, holder, uri, schedule_reader): Thread.__init__(self) self._loop = True self._error_event = Event() self._wm = WatchManager() self._holder = holder self._uri = uri self._schedule_reader = schedule_reader self._notifier = ThreadedNotifier( self._wm, _EventHandler(self._holder, self._uri, self._schedule_reader, self._error_event)) self._path, self._pattern = os.path.split(urlparse(uri).path) def start(self): """Start the file watcher """ self._notifier.start() self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY) Thread.start(self) def run(self): while self._loop: if self._error_event.wait(1): self._error_event.clear() self._notifier.stop() del self._notifier self._notifier = ThreadedNotifier( self._wm, _EventHandler(self._holder, self._uri, self._schedule_reader, self._error_event)) self._notifier.start() self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY) def stop(self): """Stop the file watcher """ self._notifier.stop() self._loop = False
def monitor(filename, server): """Monitor the config file for changes and update the server's values when they do. Using pyinotify, it checks the config file for changes and tells the server to make changes when they occur. Starts and returns the ThreadedNotifier that is created. Keyword arguments: filename -- the name of the file you want to monitor. Note that it won't actually monitor this file, but the directory it resides in. server -- the EmailServer currently running, to pass the new values to when they're changed. """ logging.info('Monitoring {0} for changes'.format(filename)) wm = WatchManager() notifier = ThreadedNotifier(wm, PClose(server, filename)) notifier.name = 'MonitorThread' # we actually watch the folder to the file, otherwise the handle is lost every time the file is modified. wm.add_watch(os.path.split(filename)[0], IN_CLOSE_WRITE, rec=True) notifier.start() return notifier
class Watcher(object): """ Watching on the fly """ def __init__(self, gdr, workq, src, dst): self.gdr = gdr self.workq = workq self.src = src self.dst = dst def loop(self): wm = WatchManager() handler = EventHandler(self.workq, self.src, self.dst) self.notifier = ThreadedNotifier(wm, handler) self.notifier.start() mask = IN_CREATE | IN_MODIFY wm.add_watch(self.src, mask, rec=self.gdr.rec) def stop(self): self.notifier.stop()
class Watcher(object): """ Watching on the fly """ def __init__(self, gdr, workq, src, dst): self.gdr = gdr self.workq = workq self.src = src self.dst = dst def loop(self): wm = WatchManager() handler = EventHandler(self.workq, self.src, self.dst) self.notifier = ThreadedNotifier(wm, handler) self.notifier.start() mask = IN_CREATE | IN_MODIFY wm.add_watch(self.src, mask, rec=self.gdr.rec) def stop(self): self.notifier.stop()
def start_watching(): class OnEvent(ProcessEvent): def __init__(self): ProcessEvent(self) self.timer = None def process_default(self, event): accepted = [pyinotify.IN_MODIFY, pyinotify.IN_MOVE_SELF, pyinotify.IN_MOVED_FROM, pyinotify.IN_MOVED_TO] x = [x for x in accepted if event.mask & x == event.mask] if len(x) > 0: print "default: %-20s %s" % (os.path.join(event.path, event.name), event.maskname) if self.timer: self.timer.cancel() self.timer = None self.timer = threading.Timer(0.2, do_post) self.timer.start() wm = WatchManager() mask = pyinotify.ALL_EVENTS proc = OnEvent() notifier = ThreadedNotifier(wm, proc) notifier.start() wdd = wm.add_watch('.git', mask, rec=True, auto_add=True)
def main_loop(): # Setup pyinotify so files are being watched: wm = WatchManager() notifier = ThreadedNotifier(wm, PTmp()) notifier.start() mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_ATTRIB'] |EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events #mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events #wdd = wm.add_watch('../sentry.py', mask, rec=True) #wdd = wm.add_watch('../', mask, rec=True) try: while True: print '\rConnections to %d clients. (%s)' % (len(open_handles), time_string()), sys.stdout.flush() update_subscribers(msg_type=MsgTypes.Heartbeat) time.sleep(5) except: notifier.stop() raise
class ExcelFileWatcher(object): class Process(ProcessEvent): #def __init__(self, options): # self.regex = re.compile(options.regex) # self.script = options.script def __init__(self, folderpath, event_callback): #self.regex = re.compile(options.regex) #self.script = options.script self.event_callback = event_callback self.folderpath = folderpath def process_IN_CREATE(self, event): target = os.path.join(event.path, event.name) print "create event happened!" if os.path.isdir(target): raise Reload() def process_IN_DELETE(self, event): print "delete event happened" raise Reload() def process_IN_CLOSE_WRITE(self, event): target = os.path.join(event.path, event.name) #print "target = ", target #print 'event.path = ', event.path #print 'event.name = ', event.name self.event_callback(event.path, event.name) #if self.regex.match(target): # args = self.script.replace('$f', target).split() # os.system("clear") # sys.stdout.write("executing script: " + " ".join(args) + "\n") # subprocess.call(args) # sys.stdout.write("------------------------\n") def __init__(self, folderpath, event_callback): self.event_callback = event_callback self.folderpath = folderpath def start_watch_loop(self): wm = WatchManager() process = self.Process(self.folderpath, self.event_callback) #options) self.notifier = ThreadedNotifier(wm, process) #Notifier(wm, process) #notifier = Notifier(wm) mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE #wdd = wm.add_watch(options.directory, mask, rec=True) #wm.add_watch('./excelsrc', mask, rec=True) wm.add_watch('./' + self.folderpath, mask, rec=True) self.notifier.start() """ while True: wm = WatchManager() process = self.Process(self.folderpath, self.event_callback) #options) notifier = ThreadedNotifier(wm, process) #Notifier(wm, process) #notifier = Notifier(wm) mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE #wdd = wm.add_watch(options.directory, mask, rec=True) #wm.add_watch('./excelsrc', mask, rec=True) wm.add_watch('./'+self.folderpath, mask, rec=True) try: while True: print '.' notifier.process_events() print '+' if notifier.check_events(): notifier.read_events() print '-' except Reload: pass except KeyboardInterrupt: notifier.stop() break """ def stop_watch_loop(self): self.notifier.stop()
self._dispatch_download(file_path, file_name) else: print "Skipping %s" % file_name def _process_directory(self, dir_path): candidates = os.listdir(dir_path) print candidates for candidate in candidates: if candidate.split('.')[-1].lower() in allowed_extensions: self._dispatch_download(dir_path, candidate) else: print "Skipping %s" % candidate print "SDownloader is now starting..." wm = WatchManager() wm.add_watch(target_dir, mask, rec=True, auto_add=True) print "Sdownloader is now watching %s" % target_dir notifier = ThreadedNotifier(wm, MyWatcher()) notifier.start() while True: try: time.sleep(0.5) except (Exception, KeyboardInterrupt): notifier.stop() break
class _Core(): """ the core functionaliy for persy """ def init(self, config, log): """ initializes the git binding """ self.config = config self.log = log self.worker = None self.notifier = None self.running = False #initialzing the git binding #=========================== #if persy is interrupted while git was running, a git.lockfile may be present. we have to remove it! if os.path.exists(self.config.getAttribute('GIT_LOCKFILE')): try: os.remove(self.config.getAttribute('GIT_LOCKFILE')) except Exception as err: log.warn(str(err)) else: log.warn(_("removed git lock file")) #init pug os.popen("touch %s"%self.config.getAttribute('LOGFILE_GIT')) std = open(self.config.getAttribute('LOGFILE_GIT'), 'a') stdin = None #default stdin stdout = std #default stdout stderr = std #default stderr cwd = os.environ["HOME"] self.vcs = pug.PuG(cwd=cwd, GIT_WORK_TREE=config['general']['gitworktree'], GIT_DIR=config['general']['gitdir'], stdin=stdin, stdout=subprocess2.PIPE, stderr=stderr) def init_local(self): """ initialises the local repository """ if not self.config.has_key('general') or not self.config['general'].has_key('name') or not self.config['general']['name'] : self.log.critical(_('username not set, cannot create git repository. use "persy --config --name=NAME" to set one'), verbose=True) sys.exit(-1) if not self.config.has_key('general') or not self.config['general'].has_key('mail') or not self.config['general']['mail']: self.log.critical(_('mail not set, cannot create git repository. use "persy --config --mail=MAIL" to set one'), verbose=True) sys.exit(-1) self.log.info(_("initialising local repository..."), verbose=True) try: self.vcs.init() self.vcs.config('user.name',self.config['general']['name']) self.vcs.config('user.email',self.config['general']['mail']) self.vcsignore() except Exception as err: self.log.critical(str(err), verbose=True) else: self.log.info(_("done"), verbose=True) def vcslog(self): """ runs the vcs(git)-log command and writs the output to stdout """ self.vcs.log(stdout=sys.stdout, stdin=sys.stdin, stderr=sys.stderr) def vcsstatus(self): """ runs the vcs(git)-status command """ self.vcs.status(stdout=sys.stdout, stdin=sys.stdin, stderr=sys.stderr) def initRemote(self): """ initialises the remote repository and adds it to the synchronization pack tries to connect to the remote server and creates the destination folder and initializes the remote git repository """ self.log.info(_("initialising and adding remote repository..."), verbose=True) try: client = paramiko.SSHClient() client.load_system_host_keys() client.connect(self.config['remote']['hostname'], username=self.config['remote']['username'], port=int(self.config['remote']['port'])) # the follow commands are executet on a remote host. we can not know the path to git, # mkdir and cd so we will not replace them with a absolute path stdin1, stdout1, stderr1 = client.exec_command("mkdir -m 700 %s"%self.config['remote']['path']) stdin2, stdout2, stderr2 = client.exec_command("cd %s && git --bare init"%self.config['remote']['path']) client.close() if stderr1: self.log.warn(_("error creating dir, maybe it exists already?"), verbose=True) elif stderr2: self.log.critical(_("error on remote git init"), verbose=True) elif not self.config['remote']['use_remote']: #no errors:so we are save to use the remote self.config['remote']['use_remote'] = True self.config.write() self.vcs.remoteAdd(self.config.getAttribute('SERVER_NICK'),"ssh://%s@%s/%s"%(self.config['remote']['username'],self.config['remote']['hostname'],self.config['remote']['path'])) except Exception as e: self.log.critical(str(e), verbose=True) else: self.log.info(_("done"), verbose=True) def syncWithRemote(self): """ Syncs with a remote server adds a new remote repository with the information from config to the local git repository and performs a pull """ # i dont use clone because of massive errors when using it # the best way iś to add the remote server and pull from it self.log.debug('persy-core-sync called') #check to only do this if all of the remote watched directoris do NOT exist for f in self.config['local']['watched']: if os.path.exists(f): self.log.critical(_("%f does already exist but it should not! Please remove it and try it again.")) return False try: self.log.debug('persy-core-sync remote add') self.vcs.remoteAdd(self.config.getAttribute('SERVER_NICK'),"ssh://%s@%s/%s"%(self.config['remote']['username'],self.config['remote']['hostname'],self.config['remote']['path'])) self.log.debug('persy-core-sync pull') self.vcs.pull(self.config.getAttribute('SERVER_NICK'),self.config.getAttribute('BRANCH')) except Exception as e: self.log.critical(str(e)) self.log.debug('persy-core-sync ende') self.config['remote']['use_remote'] = True self.config.write() def isInSyncWithRemote(self): """ returns true if it is already in sync "in sync" means: git remote returns an entry for an "origin" repository. """ try: self.vcs.command('git', params=['remote']) except Exception as e: self.log.critical(str(e)) std = self.vcs.getLastOutput().strip("\n ") if std == 'origin': return True return False def vcsignore(self): """ creates a file for ignoring unwatched directories so they dont appear in the status (and cant be removed exidently with "git clean") list every file in /home/USER. add every file and folder (if not already done) to .vcsignore if they are not part WATCHED """ current = os.listdir(self.config.getAttribute('USERHOME')) for f in self.config['local']['watched']: #if not f.startswith(USERHOME): if f.startswith(self.config.getAttribute('USERHOME')): #if absolute path #strip dir stuff, the +1 is for the file seperator f = f[len(self.config.getAttribute('USERHOME'))+1:] elif f.startswith('~/'): f = f[2:] #strip the ~/ elif f.startswith('./'): f = f[2:] #strip the ./ elif f.startswith('/'): #continue #savetycheck #i assume if it still starts with /, its outside of /home continue if os.sep in f: f = f[:f.index(os.sep)] if f in current: current.remove(f) if self.config['local']['maxfilesize'] and self.config['local']['maxfilesize'] > 0: #add files to the ignorelist if they are larger than maxfilesize callcmd = [] callcmd.append('find') callcmd.append(os.path.join(self.config.getAttribute('USERHOME'),f)) callcmd.append('-type') callcmd.append('f') callcmd.append('-size') callcmd.append("+" + str(self.config['local']['maxfilesize']) + "k") (stdoutdata, stderrdata) = subprocess.Popen(callcmd, stdout=subprocess.PIPE).communicate() for entry in stdoutdata.split("\n"): current.append(entry) for entry in self.config['local']['exclude']: current.append(entry) with open(os.path.join(self.config.getAttribute('PERSY_DIR'),self.config.getAttribute('GITIGNOREFILE')), "w+") as f: for c in current: f.write(c+"\n") def optimize(self): """ tries to optimize the local repository. executes the git-gc command. """ self.log.info('starting optimization', verbose=True) class Starter(Thread): def __init__(self,vcs, log): Thread.__init__(self) self.vcs = vcs self.log = log def run(self): try: self.log.debug('git gc') self.vcs.gc() except Exception as e: self.log.warn(str(e), verbose=True) try: Starter(self.vcs, self.log).start() except Exception as e: self.log.warn(str(e), verbose=True) def browse(self): """ starts the default git browser """ class Starter(Thread): def __init__(self,vcs, config): Thread.__init__(self) self.vcs = vcs self.config = config def run(self): self.vcs.command(self.config['general']['prefgitbrowser'], stdout=sys.stdout, stdin=sys.stdin, stderr=sys.stderr) Starter(self.vcs, self.config).start() def persy_stop(self): """ Stops persys working thread and the notifier. """ self.log.info("stop working") self.log.setStop() self.running = False if self.worker: try: self.worker.stop() self.worker.join() except RuntimeError: pass if self.notifier: try: self.notifier.stop() except RuntimeError: pass except OSError: pass except KeyError: pass def persy_start(self): """ initializes the worker thread and notifier """ self.log.info("start working") self.log.setStart() self.running = True FLAGS=EventsCodes.ALL_FLAGS mask = FLAGS['IN_MODIFY'] | FLAGS['IN_DELETE_SELF']|FLAGS['IN_DELETE'] | FLAGS['IN_CREATE'] | FLAGS['IN_CLOSE_WRITE'] | FLAGS['IN_MOVE_SELF'] | FLAGS['IN_MOVED_TO'] | FLAGS['IN_MOVED_FROM'] # watched events wm = WatchManager() #addin the watched directories for watch in self.config['local']['watched']: wdd = wm.add_watch("%s"%(watch), mask, rec=True, auto_add=True) #watch for changes of the configurationfile if self.config['general']['autoshare']: wdd = wm.add_watch(self.config.getAttribute('CONFIGFILE'), mask, rec=True, auto_add=True) self.log.debug("init the syncer") self.worker = TheSyncer(self, self.config, self.log, self.config['remote']['sleep'], self.config['local']['sleep']) self.log.debug("init the filesystem notifier") self.notifier = ThreadedNotifier(wm, FileChangeHandler(self.log, self.worker.newEvent)) self.log.resetError() self.log.debug("starting syncer") self.worker.start() self.notifier.start() #if self.statusIcon: # statusIcon.set_from_file(config.getAttribute('ICON_OK'))#from_stock(gtk.STOCK_HOME) def setonetimesync(self): """ if the worker is running, execute the setonetimesync function of the worker """ if self.worker: try: self.worker.setonetimesync() except RuntimeError: pass def isLocalInitialized(self): """ returns true if the local GIT_DIR exists """ return os.path.exists(self.config.getAttribute('GIT_DIR')) def git_add(self, item): """ adds an item to git """ self.vcs.add(item) def git_commit(self, message): """ executes a commit with "message" as the commit message """ self.vcs.commit(message) def git_pull(self, nickname, branch): """ executes a pull from "branch" on "nickname" """ self.vcs.pull(nickname, branch) def git_push(self, nickname, branch): """ executes a push to "branch" on "nickname" """ self.vcs.push(nickname,branch) def git_svn_pull(self): self.vcs.svn_pull() def git_svn_push(self): self.vcs.svn_push() def git_get_submodules(self): """ returns all submodules in watched directories """ return self.vcs.get_submodules(include_dir = self.config['local']['watched'])
class FSMonitorInotify(FSMonitor): """inotify support for FSMonitor""" EVENTMAPPING = { FSMonitor.CREATED: pyinotify.IN_CREATE, FSMonitor.MODIFIED: pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB, FSMonitor.DELETED: pyinotify.IN_DELETE, FSMonitor.MONITORED_DIR_MOVED: pyinotify.IN_MOVE_SELF, FSMonitor.DROPPED_EVENTS: pyinotify.IN_Q_OVERFLOW, } def __init__(self, callback, persistent=False, trigger_events_for_initial_scan=False, ignored_dirs=[], dbfile="fsmonitor.db"): FSMonitor.__init__(self, callback, persistent, trigger_events_for_initial_scan, ignored_dirs, dbfile) self.wm = None self.notifier = None def __fsmonitor_event_to_inotify_event(self, event_mask): """map an FSMonitor event to an inotify event""" inotify_event_mask = 0 for fsmonitor_event_mask in self.__class__.EVENTMAPPING.keys(): if event_mask & fsmonitor_event_mask: inotify_event_mask = inotify_event_mask | self.__class__.EVENTMAPPING[ fsmonitor_event_mask] return inotify_event_mask def __add_dir(self, path, event_mask): """override of FSMonitor.__add_dir()""" # Perform an initial scan of the directory structure. If this has # already been done, then it will return immediately. if self.persistent: if self.trigger_events_for_initial_scan: FSMonitor.generate_missed_events(self, path, event_mask) else: self.pathscanner.initial_scan(path) event_mask_inotify = self.__fsmonitor_event_to_inotify_event( event_mask) # Use the inotify API to monitor a directory. wdd = self.wm.add_watch(path, event_mask_inotify, proc_fun=self.process_event, rec=True, auto_add=True) if wdd is None: raise MonitorError, "Could not monitor %s" % path return None else: self.monitored_paths[path] = MonitoredPath(path, event_mask, wdd) self.monitored_paths[path].monitoring = True # Generate the missed events. if self.persistent: FSMonitor.generate_missed_events(self, path) return self.monitored_paths[path] def __remove_dir(self, path): """override of FSMonitor.__remove_dir()""" if path in self.monitored_paths.keys(): wd = self.monitored_paths[path].fsmonitor_ref # TODO: figure out why this won't work, it seems this fails due to # a bug in pyinotify? #self.wm.rm_watch(wd, rec=True, quiet=True) del self.monitored_paths[path] def run(self): # Setup. Ensure that this isn't interleaved with any other thread, so # that the DB setup continues as expected. self.lock.acquire() FSMonitor.setup(self) self.process_event = FSMonitorInotifyProcessEvent(self) self.lock.release() # Set up inotify. self.wm = WatchManager() self.notifier = ThreadedNotifier(self.wm, self.process_event) self.notifier.start() while not self.die: self.__process_queues() time.sleep(0.5) self.notifier.stop() def stop(self): """override of FSMonitor.stop()""" # Let the thread know it should die. self.lock.acquire() self.die = True self.lock.release() # Stop monitoring each monitored path. for path in self.monitored_paths.keys(): self.__remove_dir(path) def __process_queues(self): # Process add queue. self.lock.acquire() if not self.add_queue.empty(): (path, event_mask) = self.add_queue.get() self.lock.release() self.__add_dir(path, event_mask) else: self.lock.release() # Process remove queue. self.lock.acquire() if not self.remove_queue.empty(): path = self.add_queue.get() self.lock.release() self.__remove_dir(path) else: self.lock.release()
class TargetWatcher(object): def __init__(self, configuration, builder, watch_index): self._builder = builder self._root = configuration.GetExpandedDir("projects", "root_dir") self._batch_timeout = float( configuration.Get("file_watcher", "event_batch_timeout_ms")) / 1000 self._moddef_filename = configuration.Get( "general", "module_definition_filename") self.wm = WatchManager() self.watch_index = watch_index self.watched_module_definitions = collections.defaultdict(dict) mask = (EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']) handler = functools.partial(TargetWatcher.ProcessEvent, self) self.events_queue = queue.Queue() self.acc_thread = threading.Thread(target=functools.partial( TargetWatcher.AccumulationThreadProc, self), daemon=True) self.acc_thread.start() self.notifier = ThreadedNotifier(self.wm, handler) self.notifier.start() self.watch = self.wm.add_watch(self._root, mask, rec=True, auto_add=True) self.modification_handlers = [] def _GetAllModuleDefinitionsForTarget(self, target_name): prefix = "" module_definition_dirs = [""] for path_element in target_name.split("/"): if prefix: prefix = prefix + "/" + path_element else: prefix = path_element module_definition_dirs.append(prefix) return module_definition_dirs def AddModificationHandler(self, handler): self.modification_handlers.append(handler) def ProcessEventsBatch(self, batch): modified_targets = set() modified_module_definitions = set() root_prefix_len = len(self._root) for event in batch: rel_path = event.pathname[root_prefix_len + 1:] if rel_path.endswith(self._moddef_filename): conf_dir = rel_path[:-len(self._moddef_filename) - 1] modified_module_definitions.update( set(self.watched_module_definitions[conf_dir].values())) else: found_targets = self.watch_index.GetMatchingTargets(rel_path) modified_targets.update(found_targets.values()) if modified_module_definitions or modified_targets: self.ModificationsFound(modified_module_definitions, modified_targets) def ModificationsFound(self, modified_module_definitions, modified_targets): logging.info("Files modified: module definitions %s, other %s", modified_module_definitions, modified_targets) for handler in self.modification_handlers: handler(modified_module_definitions, modified_targets) def AccumulationThreadProc(self): event_buffer = [] while True: try: if event_buffer: item = self.events_queue.get(block=True, timeout=self._batch_timeout) else: item = self.events_queue.get(block=True) event_buffer.append(item) except queue.Empty as e: try: self.ProcessEventsBatch(event_buffer[:]) except e: logging.exception("Uncaught change event processing error") event_buffer = [] def Join(self): self.notifier.stop() def ProcessEvent(self, event): self.events_queue.put(event) def _RefreshGlobs(self, target): watched_globs = self._builder.GetWatchableSources(target.GetName()) target_dir = os.path.dirname(target.GetName()) rel_globs = [ os.path.join(target_dir, glob_p) for glob_p in watched_globs ] self.watch_index.LoadGlobsForTarget(target, rel_globs) def ReloadTarget(self, target): self._RefreshGlobs(target) def AddTarget(self, target): self._RefreshGlobs(target) for prefix in self._GetAllModuleDefinitionsForTarget(target.GetName()): self.watched_module_definitions[prefix][target.GetName()] = target def RemoveTarget(self, target): del self.watched[target.GetName()] for prefix in self._GetAllModuleDefinitionsForTarget(target.GetName()): del self.watched_module_definitions[prefix][target.GetName()]
class Repo(ProcessEvent): def __init__(self, filename=None, config={}, **kwargs): """ Load a repository metadata structure filename: the name of a .chunker file to load, containing either full state, or a useful subset **kwargs: a basic .chunker data structure config: optional dictionary of extra info. Keys: username - for change log hostname - for change log """ self.notifier = None if not filename and not kwargs: raise Exception("Repo has no initialisation data") struct = {} if filename and os.path.exists(filename): try: data = gzip.open(filename).read() except: data = open(filename).read() struct.update(json.loads(data)) struct.update(kwargs) self.config = config self.name = struct.get("name") or os.path.basename(struct.get("root")) or os.path.splitext(os.path.basename(filename or ""))[0] self.root = struct.get("root") or os.path.join(os.path.expanduser("~/Downloads"), self.name) self.type = struct.get("type", "share") # static / share self.uuid = struct.get("uuid", sha256(uuid.uuid4())) self.key = struct.get("key", None) # for encrypting / decrypting chunks self.peers = struct.get("peers", []) self.files = dict([ (filename, File.from_struct(self, filename, data)) for filename, data in struct.get("files", {}).items() ]) # if we're creating a new static chunkfile, then add our local files to the chunkfile # should this be in start()? if (self.type == "static" and not self.files): self.__add_local_files() def to_struct(self, state=False): """ Serialise the repository into a JSON-compatible dictionary """ data = { "name": self.name, "type": self.type, "uuid": self.uuid, "key": self.key, "files": dict([ (filename, file.to_struct(state=state)) for filename, file in self.files.items() ]) } if state: data.update({ "peers": [ peer.to_struct(state=state) for peer in self.peers ], "root": self.root, }) return data def __repr__(self): return "Repo(%r, %r, %r, %r)" % (self.type, self.uuid, self.root, self.name) def save_state(self): """ Save the repository state to the default state location (eg ~/.config/chunker/<uuid>.state on unix) """ self.save(get_config_path(self.uuid + ".state"), state=True, compress=True) def remove_state(self): p = get_config_path(self.uuid + ".state") if os.path.exists(p): os.unlink(p) def save(self, filename=None, state=False, compress=False): """ Export the repository state (ie, write Repo.to_struct() to a JSON file) filename: where to save the state to state: whether to save active state, eg which chunks are currently downloaded True -> useful for an app to exit and re-open on the same PC later False -> useful for exporting the minimal amount of info to get a new node to join the swarm compress: whether or not to run the data through gzip (disabling this can make debugging easier) """ struct = self.to_struct(state=state) if compress: fp = gzip.open(filename, "w") data = json.dumps(struct) else: fp = open(filename, "w") data = json.dumps(struct, indent=4) fp.write(data) fp.close() def log(self, msg): log.info("[%s] %s" % (self.name, msg)) ################################################################### # Metadata ################################################################### def __add_local_files(self): for dirpath, dirnames, filenames in os.walk(self.root): for filename in filenames: path = os.path.join(dirpath, filename) relpath = self.__relpath(path) # look for # - files that we haven't seen before # - files with newer timestamps than our latest known version # # note that if a file has new content, but the timestamp is # unchanged since we last saw it, we won't add a new version, # but rather treat the current version as corrupt if ( relpath not in self.files or ts_round(os.stat(path).st_mtime) > self.files[relpath].timestamp ): self.update(relpath, { "versions": [{ "timestamp": ts_round(os.stat(path).st_mtime), "chunks": None, }] }) for file in self.files.values(): # "not supposed to be deleted, but it is" -> it has been # deleted while we weren't looking. if not file.deleted and not os.path.exists(file.fullpath): # We don't know when it was deleted, so add the deletion # tag as just after the last modification (ie, mark that # version as deleted, but any newer remote version takes # precedence) self.update(file.filename, { "versions": [{ "timestamp": ts_round(file.timestamp + 1), "chunks": [], "deleted": True, }] }) def update(self, filename, filedata): """ Update the repository with new metadata for a named file """ file = File.from_struct(self, filename, filedata) if file.filename not in self.files: self.files[file.filename] = file else: self.files[file.filename].versions.extend(file.versions) self.files[file.filename].versions.sort() if file.deleted: file.log("deleted") if os.path.exists(file.fullpath): os.unlink(file.fullpath) else: if os.path.exists(file.fullpath): file.log("updated") else: with open(file.fullpath, "a"): if file.is_complete(): # mark as finished already os.utime(file.fullpath, (file.timestamp, file.timestamp)) else: # mark as incomplete os.utime(file.fullpath, (0, 0)) file.log("created") self.save_state() ################################################################### # Networking ################################################################### def add_peer(self, peer): if peer not in self.peers: self.log("Found new peer: %r" % (peer, )) self.peers.append(peer) ################################################################### # Chunks ################################################################### def get_missing_chunks(self): """ Get a list of missing chunks """ l = [] for file in self.files.values(): l.extend(file.get_missing_chunks()) return l def get_known_chunks(self): """ Get a list of known chunks """ l = [] for file in self.files.values(): l.extend(file.get_known_chunks()) return l def add_chunk(self, chunk_id, data): """ Notify the repository that a new chunk is available (probably freshly downloaded from the network) """ self.log("Trying to insert chunk %s into files" % chunk_id) for chunk in self.missing_chunks: if chunk.id == chunk_id: chunk.save_data(data) def self_heal(self, known_chunks=None, missing_chunks=None): """ Try to use known chunks to fill in gaps """ if known_chunks is None: known_chunks = self.get_known_chunks() if missing_chunks is None: missing_chunks = self.get_missing_chunks() heal(known_chunks, missing_chunks) ################################################################### # Crypto ################################################################### def encrypt(self, data): if self.key: c = AES.new(self.key) data = c.encrypt(data) return data def decrypt(self, data): if self.key: c = AES.new(self.key) data = c.decrypt(data) return data ################################################################### # File system monitoring ################################################################### def start(self): """ Start monitoring for file changes """ if self.type == "share": self.log("Checking for files updated while we were offline") self.__add_local_files() self.log("Watching %s for file changes" % self.root) watcher = WatchManager() watcher.add_watch(self.root, ALL_EVENTS, rec=True, auto_add=True) self.notifier = ThreadedNotifier(watcher, self) self.notifier.daemon = True self.notifier.start() else: self.log("Not watching %s for file changes" % self.root) # self.self_heal() def netcomms(): while True: # select()'ing three empty lists is an error on windows if not self.peers: sleep(5) continue rs, ws, xs = select(self.peers, self.peers, [], 0) for r in rs: packet = r.recv() r.last_pong = time() for w in ws: if w.last_ping < time() - 30 and w.last_pong < time() - 30: data = json.dumps({"cmd": "get-status", "since": w.last_update}) w.send(data) w.last_ping = time() for peer in self.peers: if peer.last_pong < time() - 300: log.info("Peer no longer reachable - %r" % peer) peer.last_pong = time() + 10000 # if there was nothing to do, sleep for a bit # (if there was something to do, immediately go back for more) if not rs: sleep(1) nc = Thread(target=netcomms, name="NetComms[%s]" % self.name) nc.daemon = True nc.start() def stop(self): """ Stop monitoring for file changes """ if self.notifier: self.log("No longer watching %s for file changes" % self.root) self.notifier.stop() self.notifier = None def __relpath(self, path): base = os.path.abspath(self.root) path = os.path.abspath(path) return path[len(base)+1:] def process_IN_CREATE(self, event): if os.path.isdir(event.pathname): return path = self.__relpath(event.pathname) self.update(path, filedata={ "versions": [{ "timestamp": int(os.stat(event.pathname).st_mtime), "chunks": None, "username": self.config.get("username"), "hostname": self.config.get("hostname"), }] }) # def process_IN_MODIFY(self, event): # if os.path.isdir(event.pathname): # return # path = self.__relpath(event.pathname) # self.update(path, filedata={ # "versions": [{ # "deleted": False, # "timestamp": int(os.stat(event.pathname).st_mtime), # "chunks": None, # "username": self.config.get("username"), # "hostname": self.config.get("hostname"), # }] # }) def process_IN_DELETE(self, event): if os.path.isdir(event.pathname): return path = self.__relpath(event.pathname) self.update(path, filedata={ "versions": [{ "deleted": True, "timestamp": int(time()), "chunks": [], "username": self.config.get("username"), "hostname": self.config.get("hostname"), }] })
def __init__(self, parent=None, infile=None): QtGui.QWidget.__init__(self, parent) #set up the main UI self.ui = Ui_MainWindow() self.ui.setupUi(self) self.dbglvl = dbglvl # self.libdir='/home/gilbank/Proj/spup/GUI/libs/' # self.libdir='libs/' self.libdir = '/usr/local/lib/qlgui/' self.settingsdir = '/home/ccd/.spup/' self.spup = True # else work with old spectrograph data self.ui.logView.setReadOnly(True) # Install the custom output stream sys.stdout = EmittingStream(textWritten=self.normalOutputWritten) # set some default size values. use to try to set sliders, etc. to sensible ranges: if (self.spup): self.sx = 2148 self.sy = 256 # self.sy = 128 # 1x2 binning else: self.sx = 2098 self.sy = 512 self.xbin = 1 # self.ybin=1 self.ybin = 2 self.sx = self.sx / self.xbin self.xy = self.sy / self.ybin # -- update sliders to double sliders: geom = self.ui.SB1Slider.geometry() self.ui.SB1Slider = QxtSpanSlider(self.ui.centralWidget) self.ui.SB1Slider.setGeometry(geom) #self.ui.SB1Slider.setOrientation(QtCore.Qt.Horizontal) """ self.SB1Slider = QxtSpanSlider(self.centralWidget) self.SB1Slider.setGeometry(QtCore.QRect(1100, 520, 160, 23)) self.SB1Slider.setOrientation(QtCore.Qt.Horizontal) self.SB1Slider.setObjectName(_fromUtf8("SB1Slider")) """ if (self.spup): self.ui.SB1Slider.setRange(0, 132) #*** self.ui.SB1Slider.setSpan(80, 120) #*** else: self.ui.SB1Slider.setRange(0, 132) self.ui.SB1Slider.setSpan(80, 120) # -- geom = self.ui.SB2Slider.geometry() self.ui.SB2Slider = QxtSpanSlider(self.ui.centralWidget) self.ui.SB2Slider.setGeometry(geom) #self.ui.SB1Slider.setOrientation(QtCore.Qt.Horizontal) """ self.SB1Slider = QxtSpanSlider(self.centralWidget) self.SB1Slider.setGeometry(QtCore.QRect(1100, 520, 160, 23)) self.SB1Slider.setOrientation(QtCore.Qt.Horizontal) self.SB1Slider.setObjectName(_fromUtf8("SB1Slider")) """ if (self.spup): self.ui.SB2Slider.setRange(0, 132) #*** self.ui.SB2Slider.setSpan(50, 10) #*** else: self.ui.SB2Slider.setRange(0, 132) self.ui.SB2Slider.setSpan(50, 10) geom = self.ui.extractWinSlider.geometry() self.ui.extractWinSlider = QxtSpanSlider(self.ui.centralWidget) self.ui.extractWinSlider.setGeometry(geom) """ self.extractWin_x0=self.ui.extractWinSlider.lowerValue self.extractWin_x1=self.ui.extractWinSlider.upperValue self.SB1_x0=self.ui.SB1Slider.lowerValue self.SB1_x1=self.ui.SB1Slider.upperValue self.SB2_x0=self.ui.SB2Slider.lowerValue self.SB2_x1=self.ui.SB2Slider.upperValue """ # self.mklog() self.ui.lineEditz0.setText('0.') self.ui.lineEditz1.setText('10000.') self.extractionUnits = '' # header keyword: if (self.spup): self.imtype = 'EXPTYPE' self.sciKW = 'SCIENCE' # keyword for science # self.arcKW='arc' self.arcKW = 'ARC' else: self.imtype = 'IMAGETYP' self.sciKW = 'object' # keyword for science self.arcKW = 'COMPARISON' pars = {} pars['smoothPix'] = 0 pars['SkySub'] = False pars['useSB1'] = True pars['useSB2'] = True pars['LockDisplays'] = False # pars['showExtWin'] = True # pars['showSideBands'] = True # pars['dataDir'] = '/home/gilbank/Proj/CassSpect/data/Deatrick/' # pars['dataDir'] = '/home/ccd/cassspectr/cassspectr/' try: f = open('/home/ccd/.spup/datadir.json', 'r') datadir = json.load(f) pars['dataDir'] = datadir f.close() except ValueError as V: print '%s\ncould not open datadir.json' % V self.showDialogSetDatadir() except: self.showDialogSetDatadir() # pars['outDataDir'] = '/home/ccd/cassspectr/cassspectr/' pars['outDataDir'] = pars['dataDir'] pars['currFrame'] = 'none' pars['obsType'] = 'none' pars['currArc'] = 'none' pars['currFlat'] = 'none' pars['currFluxCal'] = 'none' pars['currBias'] = 'none' pars['scaling'] = 'ZScale' pars['z0'] = np.NAN pars['z1'] = np.NAN if (self.spup): pars['extractWin_x0'] = 120.0 #** pars['extractWin_x1'] = 140.0 #** else: pars['extractWin_x0'] = 50.0 pars['extractWin_x1'] = 70.0 pars['SBoffset'] = 5 pars['SBwidth'] = 15 self.extractWin_x0 = pars['extractWin_x0'] self.extractWin_x1 = pars['extractWin_x1'] pars['autoExtraction'] = True # pars['isScience']=True self.pars = pars # check if qlsettings.json exists and, if so, reload: self.loadPars() # Decide extraction type: self.ui.lineEditDataDir.setText(self.pars['dataDir']) self.ui.lineEditOutDataDir.setText(self.pars['outDataDir']) # self.imagetype='science' self.imagetype = 'none' self.wlc = False # -- Initialise values: self.autoExtraction = True self.lamp = None self.grating = None self.parentDataDir = None #'/home/gilbank/Proj/CassSpect/data/Deatrick/' self.imagetype = "" self.lastWLCFile = None self.wlc = False self.latestFrame = None ##self.parentDataDir+'a3000142.fits' self.scaledImage = None # disable unimplemented options: # self.ui.useSB1Check.setEnabled(False) # self.ui.useSB2Check.setEnabled(False) self.ui.checkBoxLockDisplays.setEnabled(False) self.ui.checkBoxWrite1DFITS.setEnabled(False) self.ui.pushButCurrArc.setEnabled(False) # -- Connect signals and slots QtCore.QObject.connect(self.ui.actionQuit, QtCore.SIGNAL(_fromUtf8("activated()")), self.closeEvent) QtCore.QObject.connect(self.ui.pushButPan, QtCore.SIGNAL(_fromUtf8("pressed()")), self.doPanAll) QtCore.QObject.connect(self.ui.pushButZoom, QtCore.SIGNAL(_fromUtf8("pressed()")), self.doZoomAll) QtCore.QObject.connect(self.ui.pushButResetView, QtCore.SIGNAL(_fromUtf8("pressed()")), self.resetZoomAll) QtCore.QObject.connect(self.ui.pushButSmoothObj, QtCore.SIGNAL(_fromUtf8("pressed()")), self.display1D) QtCore.QObject.connect(self.ui.pushButNoSmooth, QtCore.SIGNAL(_fromUtf8("pressed()")), self.noSmoothing) QtCore.QObject.connect(self.ui.pushButCurrFrame, QtCore.SIGNAL(_fromUtf8("pressed()")), self.loadFrame) # QtCore.QObject.connect(self.ui.pushButCurrFrame, QtCore.SIGNAL(_fromUtf8("pressed()")), self.loadAndUpdate) QtCore.QObject.connect(self.ui.checkBoxShowExtractWin, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.display2D) QtCore.QObject.connect(self.ui.checkBoxShowSB, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.display2D) QtCore.QObject.connect(self.ui.zScaleUpdateBut, QtCore.SIGNAL(_fromUtf8("clicked()")), self.updatezScaling) QtCore.QObject.connect(self.ui.pushButAutoExtract, QtCore.SIGNAL(_fromUtf8("clicked()")), self.updateAutoExtraction) # QtCore.QObject.connect(self.ui.useSB1Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll) QtCore.QObject.connect(self.ui.useSB1Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.sideBandToggled) QtCore.QObject.connect(self.ui.useSB2Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.sideBandToggled) # QtCore.QObject.connect(self.ui.useSB2Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll) QtCore.QObject.connect(self.ui.SB1Slider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders) QtCore.QObject.connect(self.ui.SB2Slider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders) QtCore.QObject.connect(self.ui.extractWinSlider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders) QtCore.QObject.connect( self.ui.comboBoxColorMap, QtCore.SIGNAL(_fromUtf8("currentIndexChanged(int)")), self.display2D) QtCore.QObject.connect(self.ui.pushButChooseDataDir, QtCore.SIGNAL(_fromUtf8("clicked()")), self.showDialogSetDatadir) QtCore.QObject.connect(self.ui.pushButChooseOutDataDir, QtCore.SIGNAL(_fromUtf8("pressed()")), self.showDialogSetOutDatadir) QtCore.QObject.connect(self.ui.pushButCurrFrameBrowse, QtCore.SIGNAL(_fromUtf8("pressed()")), self.showDialogSetCurrFrame) QtCore.QObject.connect(self.ui.pushButIgnoreWLC, QtCore.SIGNAL(_fromUtf8("clicked()")), self.ignoreWLC) # QtCore.QObject.connect(self.ui.pushButIgnoreWLC, QtCore.SIGNAL(_fromUtf8("clicked()")), self.ignoreWLC) QtCore.QObject.connect(self.ui.pushButIgnoreWLClibArc, QtCore.SIGNAL(_fromUtf8("clicked()")), self.useLibArc) QtCore.QObject.connect(self.ui.useSB1Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll) QtCore.QObject.connect(self.ui.useSB2Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll) # QtCore.QObject.connect(self.ui.raw2DGraph,QtCore.SIGNAL("MOUSE_MOVED"),self.display2Dlabel) QtCore.QObject.connect(self.ui.raw2DGraph, QtCore.SIGNAL(_fromUtf8("enterEvent()")), self.display2Dlabel) # ---- Set up logging # see: https://docs.python.org/2/howto/logging-cookbook.html logfile = 'QL%s.log' % (time.strftime("%Y%m%d", time.gmtime())) logging.basicConfig(filename=logfile, level=logging.DEBUG, format='%(asctime)s %(message)s') # logFormatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M:%S') logFormatter = logging.Formatter( '%(asctime)s %(levelname)-8s %(message)s', datefmt='%H:%M:%S') self.logBuffer = logBuffer() self.logBuffer.bufferMessage.connect(self.on_logBuffer_bufferMessage) ### logFormatter = logging.Formatter('%(levelname)s: %(message)s') logHandler = logging.StreamHandler(self.logBuffer) logHandler.setFormatter(logFormatter) # add some colours: # logging.addLevelName( logging.WARNING, "\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARNING)) # logging.addLevelName( logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR)) # logging.addLevelName( logging.INFO, "\x1b[31m%s" % logging.getLevelName(logging.INFO)) #** self.logger = logging.getLogger() self.logger.setLevel(logging.INFO) self.logger.addHandler(logHandler) self.logtext = '' #self.ui.scrollAreaWidgetContents = QtGui.QTextEdit() self.logger.info('GUI started') self.makeDummy() #-- Monitor data directory for new images wm = WatchManager() notifier = ThreadedNotifier(wm, self.process_IN_CREATE) notifier.start() self.notifier = notifier mask = EventsCodes.ALL_FLAGS['IN_CREATE'] wdd = wm.add_watch(self.pars['dataDir'], mask, rec=False) #-- # read values from GUI: self.pars['showSideBands'] = self.ui.checkBoxShowSB.isChecked() self.pars['smoothPix'] = self.ui.spinBoxSmoothWidth.value() self.logger.debug(self.ui.checkBoxShowSB.isChecked()) # self.profileGraph = matplotlibWidgetwBar(self.centralWidget) # self.profPlot = matplotlibWidget(self.ui.centralWidget) ### sc = matplotlibWidgetwBar(self.ui.profileGraph) # **** clunky, but effective! #profWin = matplotlibWidgetwBar(self.ui.centralWidget) profWin = matplotlibWidget(self.ui.centralWidget) #self.profPlot.setGeometry(QtCore.QRect(1100, 140, 330, 271)) self.profWin = profWin # imname = 'a3000134.fits' # self.logger.info(imname) # self.image = self.pars['dataDir']+imname # data2D,hdr = fits.getdata(self.image, header=True) # self.data2D = data2D # self.pars['currFrame'] = imname # self.pars['obsType'] = hdr[self.imtype] # print self.pars['obsType'] # self.ui.lineEditCurrFrame.setText(pars['currFrame']) self.plotProfile() #self.profWin.mpl_toolbar.zoom() #graph2D = matplotlibWidget(self.ui.centralWidget) graph2D = matplotlibWidgetwBar(self.ui.centralWidget) self.graph2D = graph2D self.display2D() specWin = matplotlibWidget(self.ui.centralWidget) self.specWin = specWin # self.logger.debug('**',self.sky1d) self.display1D() """
def main(**options): """ Main function. It will create instances of Odoo Server, EventHandler, ThreadedNotifier and ProcessPool. """ logger = logging.getLogger(PROGNAME) logger.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s") logfile = options["logfile"] if logfile is None: streamH = logging.StreamHandler(sys.stdout) else: streamH = logging.FileHandler(logfile) streamH.setFormatter(formatter) logger.addHandler(streamH) logger.info("Started") odoo = OdooInstance( logger, db=options["db"], user=options["user"], password=options["password"], host=options["host"], port=options["port"], ) odoo.connect_to_odoo() exit_code = 0 if odoo.is_connected: count = 0 while True: count += 1 try: event_handler = EventHandler( odoo=odoo, logger=logger, dir_to_watch=options["dir_to_watch"], regex=options["regex"], delimiter=options["delimiter"], ) # start loop handler masks = IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF wm = WatchManager() wm.add_watch(event_handler.dir_to_watch, masks) notifier = ThreadedNotifier(wm, default_proc_fun=event_handler) notifier.start() pool = ProcessPool(logger, odoo) pool.loop() except (KeyboardInterrupt, EOFError): logger.info("\nReceived signal to exit! Goodbye") break except: logger.error("ERROR! \n%s" % traceback.print_exception(*sys.exc_info())) finally: if notifier: notifier.stop() if count >= 50: logger.error("ERROR! Too many tries, aborting!") break else: logger.critical("Can't connect to odoo") exit_code = 1 exit(exit_code)
class FilesystemMonitor(object): """ FileMonitor Class keeps track of all files down a tree starting at the root """ def __init__(self, searcher): self.searcher = searcher self._thread_pool = ThreadPool(THREAD_POOL_WORKS) # Add a watch to the root of the dir self.watch_manager = WatchManager() self.notifier = ThreadedNotifier(self.watch_manager, FileProcessEvent(self)) self.notifier.start() self._build_exclude_list() def _build_exclude_list(self): log.info("[FileMonitor] Set Regexs for Ignore List") self._exclude_regexs = [] # Complie Ignore list in to a list of regexs for ignore in self.searcher.configuration.get_value("EXCLUDE_LIST"): ignore = ignore.strip() ignore = ignore.replace(".", "\.") ignore = ignore.replace("*", ".*") ignore = "^"+ignore+"$" log.debug("[FileMonitor] Ignore Regex = %s" % ignore) self._exclude_regexs.append(re.compile(ignore)) def change_root(self, previous_root): self._thread_pool.clearTasks() wd = self.watch_manager.get_wd(previous_root) if wd: self.watch_manager.rm_watch(wd, rec=True) self.searcher.clear_database() self.add_directory(self.searcher.current_root) def add_directory(self, path): """ Starts a WalkDirectoryThread to add the directory """ basename = os.path.basename(path) if self.validate(basename): self.watch_manager.add_watch(path, EVENT_MASK) self._thread_pool.queueTask(self.walk_directory, path) def add_file(self, path, name): """ Add a single file to the databse """ if self.validate(name): self.searcher.add_file(path, name) def remove_file(self, path, name): self.searcher.remove_file(path, name) def remove_directory(self, path): self.searcher.remove_directory(path) def walk_directory(self, root): """ From a give root of a tree this method will walk through ever branch and return a generator. """ if os.path.isdir(root): names = os.listdir(root) for name in names: try: file_stat = os.lstat(os.path.join(root, name)) except os.error: continue if stat.S_ISDIR(file_stat.st_mode): self.add_directory(os.path.join(root, name)) else: if not stat.S_ISLNK(file_stat.st_mode): self.add_file(root, name) def finish(self): wd = self.watch_manager.get_wd(self.searcher.current_root) self.watch_manager.rm_watch(wd, rec=True) self.notifier.stop() self._thread_pool.joinAll(waitForTasks=False) def validate(self, name): # Check to make sure the file not in the ignore list for ignore_re in self._exclude_regexs: if ignore_re.match(name): log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name) return False log.debug("[WalkDirectoryThread] # Passed %s", name) return True
class INotifier(object): """ Class providing an easy wrapper for listing to INotify kernel events. """ Write, Delete = range(2) def __init__(self, path): """ Initialisation. @param path (str) Full path to watch. """ self.path = path if os.path.isdir(self.path): self.path_dir = self.path self.path_file = None else: self.path_dir = os.path.dirname(self.path) self.path_file = os.path.basename(self.path) self.callbacks = {} wm = WatchManager() wm.add_watch(self.path_dir, pyinotify.ALL_EVENTS) self.inotifier = ThreadedNotifier(wm, self.__processINotify) self.inotifier.start() def __del__(self): """ Destruction. Unload (i.e. stop) the inotifier. """ self.unload() def unload(self): """ Call this to stop listening for events. Should be called on shutdown. """ self.inotifier.stop() def __processINotify(self, event): """ Called when an INotify was received. Call the applicable callback method based on the event type. """ if event.mask == pyinotify.IN_DELETE or event.mask == pyinotify.IN_MOVED_FROM: if self.path_file == None or (self.path_file and event.name == self.path_file): for c in self.__getCallbacks(INotifier.Delete): c(event) elif event.mask == pyinotify.IN_CLOSE_WRITE or event.mask == pyinotify.IN_MOVED_TO: if self.path_file == None or (self.path_file and event.name == self.path_file): for c in self.__getCallbacks(INotifier.Write): c(event) def __getCallbacks(self, type): """ Get the current callback methods for the given type. @param type (INotifier.Write or INotifier.Delete) Type of callback get. """ return self.callbacks.get(type, []) def addCallback(self, type, callback): """ Add a callback function to handle an event. @param type (INotifier.Write or INotifier.Delete) Type of callback to add. Write is called on file writes, Delete on file deletion. @param callback (method) Method to call, should accept one parameter: the pyinotify event. """ if not type in self.callbacks: self.callbacks[type] = set() self.callbacks[type].add(callback)
class Server(Command): description = 'Run the calibre server in development mode conveniently' MONOCLE_PATH = '../monocle' def rebuild_monocole(self): subprocess.check_call(['sprocketize', '-C', self.MONOCLE_PATH, '-I', 'src', 'src/monocle.js'], stdout=open('resources/content_server/read/monocle.js', 'wb')) def launch_server(self): print 'Starting server...\n' with self.lock: self.rebuild_monocole() self.server_proc = p = subprocess.Popen(['calibre-server', '--develop'], stderr=subprocess.STDOUT, stdout=self.server_log) time.sleep(0.2) if p.poll() is not None: print 'Starting server failed' raise SystemExit(1) return p def kill_server(self): print 'Killing server...\n' if self.server_proc is not None: with self.lock: if self.server_proc.poll() is None: self.server_proc.terminate() while self.server_proc.poll() is None: time.sleep(0.1) def watch(self): if wm is not None: self.notifier = ThreadedNotifier(wm, ProcessEvents(self)) self.notifier.start() self.wdd = wm.add_watch(os.path.abspath('src'), mask, rec=True) def reload_browser(self, delay=0.1): time.sleep(delay) try: t = telnetlib.Telnet('localhost', 4242) t.read_until("repl>") t.write('BrowserReload();') t.read_until("repl>") t.close() except: print 'Failed to reload browser' import traceback traceback.print_exc() def run(self, opts): self.lock = RLock() tdir = tempfile.gettempdir() logf = os.path.join(tdir, 'calibre-server.log') self.server_log = open(logf, 'ab') self.prompt = 'Press Enter to kill/restart server. Ctrl+C to quit: ' print 'Server log available at:', logf print self.watch() first = True while True: self.launch_server() if not first: self.reload_browser() first = False try: raw_input(self.prompt) except: print self.kill_server() break else: self.kill_server() print if hasattr(self, 'notifier'): self.notifier.stop()
def __init__(self, parent = None): QtGui.QWidget.__init__(self, parent) self.canvas = MplCanvas() self.vbl = QtGui.QVBoxLayout() self.vbl.addWidget(self.canvas) #-- self.mpl_toolbar = NavigationToolbar(self.canvas, None) self.vbl.addWidget(self.mpl_toolbar) #-- self.setLayout(self.vbl) self.extractWin_x0=self.extractWinSlider.lowerValue self.extractWin_x1=self.extractWinSlider.upperValue self.SB1_x0=self.SB1Slider.lowerValue self.SB1_x1=self.SB1Slider.upperValue self.SB2_x0=self.SB2Slider.lowerValue self.SB2_x1=self.SB2Slider.upperValue self.autoExtraction=True self.lamp=None self.grating=None # self.raw2DGraph = QtGui.QGraphicsView(self.centralWidget) # self.raw2DGraph = QtGui.QWidget(self.centralWidget) self.raw2DGraph = matplotlibWidget(self.centralWidget) self.raw2DGraph.setGeometry(QtCore.QRect(20, 40, 1061, 81)) self.raw2DGraph.setObjectName(_fromUtf8("raw2DGraph")) self.skysub2DView = matplotlibWidget(self.centralWidget) self.skysub2DView.setGeometry(QtCore.QRect(20, 160, 1061, 81)) self.skysub2DView.setObjectName(_fromUtf8("skysub2DView")) self.useSB1Check = QtGui.QCheckBox(self.centralWidget) self.useSB1Check.setGeometry(QtCore.QRect(1110, 560, 85, 21)) self.useSB1Check.setChecked(True) self.useSB1Check.setObjectName(_fromUtf8("useSB1Check")) self.useSB2Check = QtGui.QCheckBox(self.centralWidget) self.useSB2Check.setGeometry(QtCore.QRect(1260, 560, 85, 21)) self.useSB2Check.setChecked(True) self.useSB2Check.setObjectName(_fromUtf8("useSB2Check")) # ---- # ---- 1D extracted spec self.spec1DView = matplotlibWidgetwBar(self.centralWidget) self.spec1DView.setGeometry(QtCore.QRect(20, 290, 1061, 381)) self.spec1DView.setObjectName(_fromUtf8("spec1DView")) # ---- # ---- 2D spec related stuff self.z0Fill = QtGui.QLineEdit(self.centralWidget) self.z0Fill.setGeometry(QtCore.QRect(1140, 50, 113, 23)) self.z0Fill.setObjectName(_fromUtf8("z0Fill")) self.z1Fill = QtGui.QLineEdit(self.centralWidget) self.z1Fill.setGeometry(QtCore.QRect(1290, 50, 113, 23)) self.z1Fill.setObjectName(_fromUtf8("z1Fill")) self.ScalingCombo = QtGui.QComboBox(self.centralWidget) self.ScalingCombo.setGeometry(QtCore.QRect(1160, 80, 78, 24)) self.ScalingCombo.setObjectName(_fromUtf8("ScalingCombo")) self.ScalingCombo.addItem(_fromUtf8("")) self.ScalingCombo.addItem(_fromUtf8("")) self.ScalingCombo.addItem(_fromUtf8("")) self.zScaleUpdateBut = QtGui.QPushButton(self.centralWidget) self.zScaleUpdateBut.setGeometry(QtCore.QRect(1280, 80, 99, 24)) self.zScaleUpdateBut.setObjectName(_fromUtf8("zScaleUpdateBut")) # ---- # ---- log window and file loaders # self.logView = QtGui.QListView(self.centralWidget) self.logView = QtGui.QTextEdit(self.centralWidget) self.logView.setReadOnly(True) # self.logView.setGeometry(QtCore.QRect(20, 710, 531, 192)) self.logView.setGeometry(QtCore.QRect(20, 710, 731, 192)) self.logView.setObjectName(_fromUtf8("logView")) self.CurrFrameBut = QtGui.QPushButton(self.centralWidget) #self.CurrFrameBut.setGeometry(QtCore.QRect(740, 720, 99, 24)) self.CurrFrameBut.setGeometry(QtCore.QRect(940, 720, 99, 24)) self.CurrFrameBut.setObjectName(_fromUtf8("CurrFrameBut")) self.currFrameFill = QtGui.QLineEdit(self.centralWidget) #self.currFrameFill.setGeometry(QtCore.QRect(600, 720, 131, 23)) self.currFrameFill.setGeometry(QtCore.QRect(800, 720, 131, 23)) self.currFrameFill.setObjectName(_fromUtf8("currFrameFill")) self.curArcBut = QtGui.QPushButton(self.centralWidget) self.curArcBut.setEnabled(False) self.curArcBut.setGeometry(QtCore.QRect(940, 770, 99, 24)) self.curArcBut.setObjectName(_fromUtf8("curArcBut")) self.currArcFill = QtGui.QLineEdit(self.centralWidget) self.currArcFill.setGeometry(QtCore.QRect(800, 770, 131, 23)) self.currArcFill.setObjectName(_fromUtf8("currArcFill")) self.curFlatBut = QtGui.QPushButton(self.centralWidget) self.curFlatBut.setEnabled(False) self.curFlatBut.setGeometry(QtCore.QRect(940, 820, 99, 24)) self.curFlatBut.setObjectName(_fromUtf8("curFlatBut")) self.currFlatFill = QtGui.QLineEdit(self.centralWidget) self.currFlatFill.setGeometry(QtCore.QRect(800, 820, 131, 23)) self.currFlatFill.setObjectName(_fromUtf8("currFlatFill")) self.curFluxCalBut = QtGui.QPushButton(self.centralWidget) self.curFluxCalBut.setEnabled(False) self.curFluxCalBut.setGeometry(QtCore.QRect(940, 870, 99, 24)) self.curFluxCalBut.setObjectName(_fromUtf8("curFluxCalBut")) self.currFluxCalFill = QtGui.QLineEdit(self.centralWidget) self.currFluxCalFill.setGeometry(QtCore.QRect(800, 870, 131, 23)) self.currFluxCalFill.setObjectName(_fromUtf8("currFluxCalFill")) # -- temp stuff fold old spectrograph only (no grating keyword in header) self.gratingCombo = QtGui.QComboBox(self.centralWidget) self.gratingCombo.setGeometry(QtCore.QRect(1150, 790, 171, 24)) self.gratingCombo.setObjectName(_fromUtf8("gratingCombo")) self.gratingCombo.addItem(_fromUtf8("")) self.gratingCombo.addItem(_fromUtf8("")) self.gratingCombo.addItem(_fromUtf8("")) self.label_15 = QtGui.QLabel(self.centralWidget) self.label_15.setGeometry(QtCore.QRect(1130, 760, 141, 16)) self.label_15.setObjectName(_fromUtf8("label_15")) # -- self.parentDataDir=None #'/home/gilbank/Proj/CassSpect/data/Deatrick/' self.imagetype="" self.lastWLCFile=None self.wlc=False self.latestFrame=None ##self.parentDataDir+'a3000142.fits' #self.ReadOutDelay = 2.0 # seconds # not used now # ---- self.widget = QtGui.QWidget(self.centralWidget) """ self.widget.setGeometry(QtCore.QRect(740, 130, 333, 23)) self.widget.setObjectName(_fromUtf8("widget")) """ self.horizontalLayout = QtGui.QHBoxLayout(self.widget) self.horizontalLayout.setMargin(0) self.horizontalLayout.setObjectName(_fromUtf8("horizontalLayout")) self.showSBCheck = QtGui.QCheckBox(self.widget) self.showSBCheck.setChecked(True) self.showSBCheck.setObjectName(_fromUtf8("showSBCheck")) self.horizontalLayout.addWidget(self.showSBCheck) self.showEWCheck = QtGui.QCheckBox(self.widget) self.showEWCheck.setChecked(True) self.showEWCheck.setObjectName(_fromUtf8("showEWCheck")) self.horizontalLayout.addWidget(self.showEWCheck) MainWindow.setCentralWidget(self.centralWidget) self.menuBar = QtGui.QMenuBar(MainWindow) self.menuBar.setGeometry(QtCore.QRect(0, 0, 1437, 21)) self.menuBar.setObjectName(_fromUtf8("menuBar")) self.menuFile = QtGui.QMenu(self.menuBar) self.menuFile.setObjectName(_fromUtf8("menuFile")) MainWindow.setMenuBar(self.menuBar) self.mainToolBar = QtGui.QToolBar(MainWindow) self.mainToolBar.setObjectName(_fromUtf8("mainToolBar")) MainWindow.addToolBar(QtCore.Qt.TopToolBarArea, self.mainToolBar) self.statusBar = QtGui.QStatusBar(MainWindow) self.statusBar.setObjectName(_fromUtf8("statusBar")) MainWindow.setStatusBar(self.statusBar) self.actionSet_data_dir = QtGui.QAction(MainWindow) self.actionSet_data_dir.setObjectName(_fromUtf8("actionSet_data_dir")) self.actionQuit = QtGui.QAction(MainWindow) self.actionQuit.setObjectName(_fromUtf8("actionQuit")) self.actionLoadData = QtGui.QAction(MainWindow) self.actionLoadData.setObjectName(_fromUtf8("actionLoadData")) self.menuFile.addAction(self.actionSet_data_dir) self.menuFile.addAction(self.actionLoadData) self.menuFile.addAction(self.actionQuit) self.menuBar.addAction(self.menuFile.menuAction()) self.scaledImage=None self.retranslateUi(MainWindow) QtCore.QObject.connect(self.showEWCheck, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.displayRaw2D) QtCore.QObject.connect(self.showSBCheck, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.displayRaw2D) QtCore.QObject.connect(self.zScaleUpdateBut, QtCore.SIGNAL(_fromUtf8("clicked()")), self.updatezScaling) QtCore.QObject.connect(self.autoExtractBut, QtCore.SIGNAL(_fromUtf8("clicked()")), self.updateAutoExtraction) #QtCore.QObject.connect(self.showSBCheck, QtCore.SIGNAL(_fromUtf8("clicked()")), MainWindow.show) QtCore.QObject.connect(self.SB1Slider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders) QtCore.QObject.connect(self.SB2Slider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders) QtCore.QObject.connect(self.extractWinSlider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders) QtCore.QObject.connect(self.useSB1Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll) QtCore.QObject.connect(self.useSB2Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll) QtCore.QObject.connect(self.actionLoadData, QtCore.SIGNAL(_fromUtf8("activated()")), self.showDialog) # QtCore.QObject.connect(self.CurrFrameBut, QtCore.SIGNAL(_fromUtf8("clicked()")), self.showDialog) QtCore.QObject.connect(self.actionSet_data_dir, QtCore.SIGNAL(_fromUtf8("activated()")), self.showDialogSetDatadir) QtCore.QObject.connect(self.actionQuit, QtCore.SIGNAL(_fromUtf8("activated()")), self.closeEvent) QtCore.QMetaObject.connectSlotsByName(MainWindow) if(self.parentDataDir==None): self.showDialogSetDatadir() # -- pyinotify code for monitoring dir for new files: wm = WatchManager() # notifier = ThreadedNotifier(wm, PTmp()) notifier = ThreadedNotifier(wm, self.process_IN_CREATE) # notifier = ThreadedNotifier(wm, self.processMonitorData) # notifier = Notifier(wm, PTmp()) notifier.start() mask = EventsCodes.ALL_FLAGS['IN_CREATE'] wdd = wm.add_watch(self.parentDataDir, mask, rec=False)
_redis = redis.Redis(host='localhost', port=6379, db=0) #TODO: put into config file _queue_name = "rsync_files_changed" def process_IN_MOVED_TO(self, event): if os.path.basename(event.name)[0] != ".": self._redis.lpush(self._queue_name, os.path.join(event.path, event.name)) path = "." wm = WatchManager() mask = IN_MOVED_TO notifier = ThreadedNotifier(wm, FileRsyncWatcher()) wm.add_watch(path, mask, auto_add=True, rec=True) notifier.start() def fun_quit(a, b): global flag_run flag_run = False notifier.stop() signal.signal(signal.SIGINT, fun_quit) signal.signal(signal.SIGTERM, fun_quit) while flag_run: time.sleep(1) # print len(list_file_newer)#, #list_file_newer
class FilesystemMonitor(object): """ FileMonitor Class keeps track of all files down a tree starting at the root """ def __init__(self, searcher): self.searcher = searcher self._thread_pool = ThreadPool(THREAD_POOL_WORKS) # Add a watch to the root of the dir self.watch_manager = WatchManager() self.notifier = ThreadedNotifier(self.watch_manager, FileProcessEvent(self)) self.notifier.start() self._build_exclude_list() def _build_exclude_list(self): log.info("[FileMonitor] Set Regexs for Ignore List") self._exclude_regexs = [] # Complie Ignore list in to a list of regexs for ignore in self.searcher.configuration.exclude_list: ignore = ignore.strip() ignore = ignore.replace(".", "\.") ignore = ignore.replace("*", ".*") ignore = "^" + ignore + "$" log.debug("[FileMonitor] Ignore Regex = %s" % ignore) self._exclude_regexs.append(re.compile(ignore)) def change_root(self, previous_root): self._thread_pool.clearTasks() wd = self.watch_manager.get_wd(previous_root) if wd: self.watch_manager.rm_watch(wd, rec=True) self.searcher.clear_database() self.add_directory(self.searcher.current_root) def add_directory(self, path): """ Starts a WalkDirectoryThread to add the directory """ basename = os.path.basename(path) if self.validate(basename): self.watch_manager.add_watch(path, EVENT_MASK) self._thread_pool.queueTask(self.walk_directory, path) def add_file(self, path, name): """ Add a single file to the databse """ if self.validate(name): self.searcher.add_file(path, name) def remove_file(self, path, name): self.searcher.remove_file(path, name) def remove_directory(self, path): self.searcher.remove_directory(path) def walk_directory(self, root): """ From a give root of a tree this method will walk through ever branch and return a generator. """ if os.path.isdir(root): names = os.listdir(root) for name in names: try: file_stat = os.lstat(os.path.join(root, name)) except os.error: continue if stat.S_ISDIR(file_stat.st_mode): self.add_directory(os.path.join(root, name)) else: if not stat.S_ISLNK(file_stat.st_mode): self.add_file(root, name) def finish(self): wd = self.watch_manager.get_wd(self.searcher.current_root) self.watch_manager.rm_watch(wd, rec=True) self.notifier.stop() self._thread_pool.joinAll(waitForTasks=False) def validate(self, name): # Check to make sure the file not in the ignore list for ignore_re in self._exclude_regexs: if ignore_re.match(name): log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name) return False log.debug("[WalkDirectoryThread] # Passed %s", name) return True
class FileInputWatcher: '''controlling class for file monitoring''' def __init__(self, dir_to_watch, queue): print 'FileInputWatcher Initialized' # SBB20090903 Adding debugging capability, not processing multiple file drops into multiple directories. if settings.DEBUG: print '*************Debugging On*************' self.queue = queue self.dir_to_watch = dir_to_watch # make a notifier (nothing) self.notifier1 = None self.notifier2 = None def monitor(self): '''The command to start monitoring a directory or set of them.''' print 'Monitoring Directories: %s' % self.dir_to_watch print "Watching started at %s" % (time.asctime()) if osiswin32: print 'Watching win32 OS' return self.watch_win32(self.dir_to_watch) else: print 'Watching POSIX OS' #if settings.DEBUG: #print 'sending to self.watch_posix_start()' self.watch_posix_start() #if settings.DEBUG: #print "It returned from self.watch_posix_start()!" #print "self.watch_posix_start() returned with value", result return True def stop_monitoring(self): '''os independent method to stop monitoring, but only posix uses it.''' #print "self.notifier1.started", self.notifier1.__getattribute__('started') if isinstance(self.notifier1, ThreadedNotifier): #if settings.DEBUG: #print "self.notifier1 is an instance" if isinstance(self.notifier2, ThreadedNotifier): #if settings.DEBUG: #print "self.notifier2 is an instance" self.watch_posix_stop() else: if settings.DEBUG: print "notifiers were not instantiated, so not calling self.watch_posix_stop() again" print 'Done Monitoring' def watch_win32(self, dir_to_watch): '''os-specific watch command''' # Loop forever, listing any file changes. The WaitFor... will # time out every half a second allowing for keyboard interrupts # to terminate the loop. files_added = [] old_path_contents = [] new_path_contents = [] cnt = 0 try: while 1: cnt += 1 #print 'Watching %s' % cnt #old_path_contents = os.listdir(dirToWatch) for item in dir_to_watch: change_handle = win32file.FindFirstChangeNotification(item, 0, win32con.FILE_NOTIFY_CHANGE_FILE_NAME) old_path_contents.append(os.listdir(item)) result = win32event.WaitForSingleObject(change_handle, 500) # If the WaitFor... returned because of a notification (as # opposed to timing out or some error) then look for the # changes in the directory contents. if result == win32con.WAIT_OBJECT_0: #new_path_contents = os.listdir(dirToWatch) # build the new list with all the files from all dirs for item in dir_to_watch: new_path_contents.append(os.listdir(item)) files_added = [f for f in new_path_contents if not f in old_path_contents] #files_deleted = [f for f in old_path_contents if not f in new_path_contents] if files_added: print print time.asctime () print "Added:", files_added or "Nothing" return files_added #print "Deleted:", files_deleted or "Nothing" win32file.FindNextChangeNotification(change_handle) except KeyboardInterrupt: return [] #finally: # win32file.FindCloseChangeNotification(change_handle) def watch_posix_start(self): '''os-specific command to watch''' # test to see if we already have a notifier object, if not, make it, otherwise we are already watching a set of folders if self.notifier1 == None and self.notifier2 == None: try: pyinotify.compatibility_mode() print 'pyinotify running in compatibility mode' except: print 'pyinotify running in standard mode' try: #mask = EventsCodes.IN_CREATE |EventsCodes.IN_MOVED_TO mask = pyinotify.ALL_EVENTS #ECJ20100831 Reason why we have two threads: it never returns control ever to the main loop if only one thread, so no ctrl+c #The second thread is a dummy, so it performs switching/relinquishing control #Eventually, we want to watch many folders, so that is why we are using the ThreadedNotifier, versus the recommended Notifier. #Even then, Notifier is still probably the way to go, but we'll use this as long as it works, because then you don't have to poll/While loop # Thread #1 watch_manager1 = WatchManager() self.notifier1 = ThreadedNotifier(watch_manager1, EventHandler(self.queue)) self.notifier1.start() print 'Starting the threaded notifier on ', self.dir_to_watch watch_manager1.add_watch(self.dir_to_watch, mask) # Thread #2 watch_manager2 = WatchManager() self.notifier2 = ThreadedNotifier(watch_manager2, EventHandlerDummy(self.queue)) self.notifier2.start() #just watch any place, but don't remove this or Ctrl+C will not work watch_manager2.add_watch(settings.BASE_PATH, mask) if settings.DEBUG: print "both notifiers started" except KeyboardInterrupt: print "Keyboard Interrupt in notifier" self.notifier1.stop() self.notifier2.stop() return except NameError: self.notifier1.stop() self.notifier2.stop() return ['POSIX Watch Error'] except: print "General exception caught within notifier while loop, stopping both notifiers now" self.notifier1.stop() self.notifier2.stop() # SBB20090903 Turn on verbose mode self.notifier1.VERBOSE = settings.DEBUG print "returning to calling function" return True def watch_posix_stop(self): 'os specific call to stop monitoring' print 'Stopping the threaded notifiers.' self.notifier1.stop() print "stopped self.notifier1.stop()" self.notifier2.stop() print "stopped self.notifier2.stop()" return
try: if pattern.match(i) and datetime.datetime.strptime( i[:16], "%Y-%m-%d_%H-%M") >= now: timedict[i[:16]] = "%s/%s" % (AUTODIR, i) except Exception, e: logging.error(e) # 3. crea e avvia il thread Pusher pusher = Pusher("localhost") pusher.start() # crea e avvia il ThreadedNotifier wm_drop = WatchManager() dropboxmask = IN_MOVED_TO notifier_drop = ThreadedNotifier(wm_drop, Dropboxer()) notifier_drop.start() wm_auto = WatchManager() autodirmask = IN_CLOSE_WRITE | IN_DELETE notifier_auto = ThreadedNotifier(wm_auto, Autodir()) notifier_auto.start() wdd_dropbox = wm_drop.add_watch(AUTOSHAREDIR, dropboxmask, rec=True) wdd_autodir = wm_auto.add_watch(AUTODIR, autodirmask, rec=True) if __name__ == "__main__": daemonize('/dev/null', '/tmp/auto.log', '/tmp/auto.log') main()
def run(): if not client_status.is_registered: print 'Sending info about new client...' status, content = api.hi(platform.uname()) print content if status == 200: client_status.is_registered = True client_status.save() else: exit('Aborted') if os.path.exists(CRASH_PATH): crash = os.stat(CRASH_PATH) if crash.st_size: with open(CRASH_PATH) as f: crash_info = f.read() status = api.report_crash(crash_info, crash.st_mtime) if status == 200: log.info('Crash reported') os.remove(CRASH_PATH) context = DaemonContext(pidfile=PIDLockFile(PIDFILE_PATH), signal_map={signal.SIGTERM: on_stop}, stderr=open(CRASH_PATH, 'w')) context.files_preserve = map( lambda h: h.stream, filter(lambda h: isinstance(h, FileHandler), log.logger.handlers)) print 'Starting daemon' with context: log.info('Daemon started') log.info('Build filesystem image') basepath = '/' root = FSNode(basepath, ignore=IGNORE_PATHS) root_d = root.as_dict() root_str = json.dumps(root_d) h = sha(root_str).hexdigest() if not client_status.fshash or client_status.fshash != h: status, content = api.set_fs(root_str) if status == 200: client_status.fshash = h client_status.save() log.info('Filesystem image updated') else: log.error('Filesystem image update failed') log.info('Create watch manager') wm = WatchManager() changelog = [] global notifier notifier = ThreadedNotifier(wm, FSEvent(changelog=changelog)) notifier.start() mask = IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO log.info('Start watching filesystem changes') for item in os.listdir(basepath): path = os.path.join(basepath, item) if item in IGNORE_PATHS or os.path.islink(path): continue wm.add_watch(path, mask, rec=True) actions = [ Action(FS_UPLOAD_PERIOD, upload_fs, changelog), Action(LOG_UPLOAD_PERIOD, upload_log), Action(RESTORE_CHECK_PERIOD, restore) ] backup_action = lambda: Action(backup.get_next(), make_backup) def on_schedule_update(actions=actions): actions[-1] = backup_action() update_schedule_action = lambda: Action(SCHEDULE_UPDATE_PERIOD, update_schedule, on_update=on_schedule_update) if update_schedule() or client_status.schedule: actions.append(update_schedule_action()) actions.append(backup_action()) else: def on_schedule_download(actions=actions): actions[-1] = update_schedule_action() actions.append(backup_action()) actions.append( Action(SCHEDULE_UPDATE_PERIOD, update_schedule, on_update=on_schedule_download, on_404=True)) log.info('Start main loop') while True: action = min(actions) log.info('Next action is %s' % action) time.sleep(action.time_left()) action()
handler = ScoreHandler() handlerloc = LocHandler() notifier = ThreadedNotifier(wm, handler) notifierloc = ThreadedNotifier(wmloc, handlerloc) wm.add_watch(FILE100, IN_CLOSE_WRITE) wm.add_watch(FILE111, IN_CLOSE_WRITE) wm.add_watch(FILE120p3, IN_CLOSE_WRITE) wm.add_watch(FILE120p4, IN_CLOSE_WRITE) wm.add_watch(FILE120p6, IN_CLOSE_WRITE) wm.add_watch(FILE120p14, IN_CLOSE_WRITE) wm.add_watch(FILE120p16, IN_CLOSE_WRITE) wm.add_watch(FILE120p21, IN_CLOSE_WRITE) wm.add_watch(FILEr48, IN_CLOSE_WRITE) wm.add_watch(FILEr59, IN_CLOSE_WRITE) wmloc.add_watch(LOCDIR, IN_CLOSE_WRITE) notifier.start() notifierloc.start() if ANCRSS == True: c.execute_delayed(60, check_rss) if READTWIT == True: c.execute_delayed(30, check_tweets) irc.process_forever()
class FSMonitorInotify(FSMonitor): """inotify support for FSMonitor""" EVENTMAPPING = { FSMonitor.CREATED : pyinotify.IN_CREATE, FSMonitor.MODIFIED : pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB, FSMonitor.DELETED : pyinotify.IN_DELETE, FSMonitor.MONITORED_DIR_MOVED : pyinotify.IN_MOVE_SELF, FSMonitor.DROPPED_EVENTS : pyinotify.IN_Q_OVERFLOW, } def __init__(self, callback, persistent=False, trigger_events_for_initial_scan=False, ignored_dirs=[], dbfile="fsmonitor.db"): FSMonitor.__init__(self, callback, persistent, trigger_events_for_initial_scan, ignored_dirs, dbfile) self.wm = None self.notifier = None def __fsmonitor_event_to_inotify_event(self, event_mask): """map an FSMonitor event to an inotify event""" inotify_event_mask = 0 for fsmonitor_event_mask in self.__class__.EVENTMAPPING.keys(): if event_mask & fsmonitor_event_mask: inotify_event_mask = inotify_event_mask | self.__class__.EVENTMAPPING[fsmonitor_event_mask] return inotify_event_mask def __add_dir(self, path, event_mask): """override of FSMonitor.__add_dir()""" # Perform an initial scan of the directory structure. If this has # already been done, then it will return immediately. if self.persistent: if self.trigger_events_for_initial_scan: FSMonitor.generate_missed_events(self, path, event_mask) else: self.pathscanner.initial_scan(path) event_mask_inotify = self.__fsmonitor_event_to_inotify_event(event_mask) # Use the inotify API to monitor a directory. wdd = self.wm.add_watch(path, event_mask_inotify, proc_fun=self.process_event, rec=True, auto_add=True) if wdd is None: raise MonitorError, "Could not monitor %s" % path return None else: self.monitored_paths[path] = MonitoredPath(path, event_mask, wdd) self.monitored_paths[path].monitoring = True # Generate the missed events. if self.persistent: FSMonitor.generate_missed_events(self, path) return self.monitored_paths[path] def __remove_dir(self, path): """override of FSMonitor.__remove_dir()""" if path in self.monitored_paths.keys(): wd = self.monitored_paths[path].fsmonitor_ref # TODO: figure out why this won't work, it seems this fails due to # a bug in pyinotify? #self.wm.rm_watch(wd, rec=True, quiet=True) del self.monitored_paths[path] def run(self): # Setup. Ensure that this isn't interleaved with any other thread, so # that the DB setup continues as expected. self.lock.acquire() FSMonitor.setup(self) self.process_event = FSMonitorInotifyProcessEvent(self) self.lock.release() # Set up inotify. self.wm = WatchManager() self.notifier = ThreadedNotifier(self.wm, self.process_event) self.notifier.start() while not self.die: self.__process_queues() time.sleep(0.5) self.notifier.stop() def stop(self): """override of FSMonitor.stop()""" # Let the thread know it should die. self.lock.acquire() self.die = True self.lock.release() # Stop monitoring each monitored path. for path in self.monitored_paths.keys(): self.__remove_dir(path) def __process_queues(self): # Process add queue. self.lock.acquire() if not self.add_queue.empty(): (path, event_mask) = self.add_queue.get() self.lock.release() self.__add_dir(path, event_mask) else: self.lock.release() # Process remove queue. self.lock.acquire() if not self.remove_queue.empty(): path = self.add_queue.get() self.lock.release() self.__remove_dir(path) else: self.lock.release()
class FileWatcher(ProcessEvent): """ FileWatcher -> Starts an INotify thread to watch a directory for file changes. """ def __init__(self): """ FileWatcher(directory) -> Watch the directory for changes. """ if not pyinotify: raise Exception("pyinotify is not loaded.") super(FileWatcher, self).__init__() self._file_callback_dict = {} self._watch_manager = WatchManager() self._events_mask = EventsCodes.ALL_FLAGS['IN_MODIFY'] self._notifier = ThreadedNotifier(self._watch_manager, self) self._notifier.setDaemon(True) @classmethod def check(cls): """ Returns true if pyinotify is loaded otherwise false. """ return pyinotify def is_running(self): """ Returns a boolean indecating the state of the notifier. """ return self._notifier.isAlive() def start(self): """ Start the notifier thread. """ self._notifier.start() def stop(self): """ Stop the notifier thread. """ self._notifier.stop() def add_directory(self, directory): """ add_directory(directory) -> Add a directory to watch. """ dir_watch = self._watch_manager.add_watch(directory, self._events_mask, rec=True) def remove_directory(self, directory): """ remove_directory(directory) -> Remove a directory from the watch. """ self._watch_manager.rm_watch(directory, rec=True) def has_file(self, filename): """ Returns a boolean indecating if the file is being watched. """ return filename in self._file_callback_dict def add_file(self, filename, callback, *user_data): """ add_file(filename, callback, *user_data) -> Add a file to watch with its callback and optional user_data. """ self._file_callback_dict[filename] = (callback, user_data) def remove_file(self, filename): """ remove_file(filename) -> Remove the file from the watch list. """ return self._file_callback_dict.pop(filename, None) def process_IN_MODIFY(self, event): """ Process modify events. """ filename = os.path.join(event.path, event.name) callback_data = self._file_callback_dict.get(filename, ()) if callback_data: callback = callback_data[0] glib.idle_add(callback, callback_data[1:])