def main_to_ascii(): setup_cli_logger(logging.INFO) parser = create_ascii_arg_parser() args = parser.parse_args() if not args.deployments_path: L.error("Please provide a --deployments_path agrument or set the " "GUTILS_DEPLOYMENTS_DIRECTORY environmental variable") sys.exit(parser.print_usage()) wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE wm.add_watch(args.deployments_path, mask, rec=True, auto_add=True) # Convert binary data to ASCII if args.type == 'slocum': processor = Slocum2AsciiProcessor( deployments_path=args.deployments_path) notifier = Notifier(wm, processor, read_freq=10) # Read every 10 seconds # Enable coalescing of events. This merges event types of the same type on the same file # together over the `read_freq` specified in the Notifier. notifier.coalesce_events() try: L.info(f"Watching {args.deployments_path} for new binary files") notifier.loop(daemonize=args.daemonize) except NotifierError: L.exception('Unable to start notifier loop') return 1 L.info("GUTILS binary_to_ascii Exited Successfully") return 0
class QNotifier(QThread): def __init__(self, wm, processor): self.event_queue = list() self._processor = processor self.notifier = Notifier(wm, NinjaProcessEvent(self.event_queue.append)) self.notifier.coalesce_events(True) self.keep_running = True QThread.__init__(self) def run(self): while self.keep_running: try: self.notifier.process_events() except OSError: pass # OSError: [Errno 2] No such file or directory happens e_dict = {} while len(self.event_queue): e_type, e_path = self.event_queue.pop(0) e_dict.setdefault(e_path, []).append(e_type) keys = list(e_dict.keys()) while len(keys): key = keys.pop(0) event = e_dict.pop(key) if (ADDED in event) and (DELETED in event): event = [e for e in event if e not in (ADDED, DELETED)] for each_event in event: self._processor(each_event, key) if self.notifier.check_events(): self.notifier.read_events() self.notifier.stop()
class QNotifier(QThread): def __init__(self, wm, processor): self.event_queue = list() self._processor = processor self.notifier = Notifier(wm, NinjaProcessEvent(self.event_queue.append)) self.notifier.coalesce_events(True) self.keep_running = True QThread.__init__(self) def run(self): while self.keep_running: try: self.notifier.process_events() except OSError: pass # OSError: [Errno 2] No such file or directory happens e_dict = {} while len(self.event_queue): e_type, e_path = self.event_queue.pop(0) e_dict.setdefault(e_path, []).append(e_type) keys = e_dict.keys() while len(keys): key = keys.pop(0) event = e_dict.pop(key) if (ADDED in event) and (DELETED in event): event = [e for e in event if e not in (ADDED, DELETED)] for each_event in event: self._processor(each_event, key) if self.notifier.check_events(): self.notifier.read_events() self.notifier.stop()
def fsMonitor(path="/data"): wm = WatchManager() mask = IN_DELETE | IN_MODIFY | IN_CREATE notifier = Notifier(wm, EventHandler(), read_freq=10) notifier.coalesce_events() wm.add_watch(path, mask, rec=True, auto_add=True) notifier.loop()
def FSMonitor(path='/root/wpf'): wm = WatchManager() mask = IN_DELETE | IN_MODIFY | IN_CREATE notifier = Notifier(wm, EventHandler(), read_freq=10) notifier.coalesce_events() # 设置受监视的事件,这里只监视文件创建事件,(rec=True, auto_add=True)为递归处理 wm.add_watch(path, mask, rec=True, auto_add=True) notifier.loop()
def FSMonitor(path='/root/wpf'): wm = WatchManager() mask = IN_DELETE | IN_MODIFY | IN_CREATE notifier = Notifier(wm, EventHandler(),read_freq=10) notifier.coalesce_events() # 设置受监视的事件,这里只监视文件创建事件,(rec=True, auto_add=True)为递归处理 wm.add_watch(path,mask,rec=True, auto_add=True) notifier.loop()
def main_to_netcdf(): setup_cli_logger(logging.INFO) parser = create_netcdf_arg_parser() args = parser.parse_args() filter_args = vars(args) # Remove non-filter args into positional arguments deployments_path = filter_args.pop('deployments_path') subset = filter_args.pop('subset') daemonize = filter_args.pop('daemonize') template = filter_args.pop('template') profile_id_type = int(filter_args.pop('profile_id_type')) # Move reader_class to a class reader_class = filter_args.pop('reader_class') if reader_class == 'slocum': reader_class = SlocumReader if not deployments_path: L.error("Please provide a --deployments_path argument or set the " "GUTILS_DEPLOYMENTS_DIRECTORY environmental variable") sys.exit(parser.print_usage()) # Add inotify watch wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE wm.add_watch( deployments_path, mask, rec=True, auto_add=True ) # Convert ASCII data to NetCDF using a specific reader class if reader_class == SlocumReader: processor = Slocum2NetcdfProcessor( deployments_path=deployments_path, subset=subset, template=template, profile_id_type=profile_id_type, prefer_file_filters=True, **filter_args ) notifier = Notifier(wm, processor, read_freq=10) # Enable coalescing of events. This merges event types of the same type on the same file # together over the `read_freq` specified in the Notifier. notifier.coalesce_events() try: L.info(f"Watching {deployments_path} for new ascii files") notifier.loop(daemonize=daemonize) except NotifierError: L.exception('Unable to start notifier loop') return 1 L.info("GUTILS ascii_to_netcdf Exited Successfully") return 0
def main_to_erddap(): setup_cli_logger(logging.WARNING) parser = create_erddap_arg_parser() args = parser.parse_args() if not args.data_path: L.error("Please provide an --data_path agrument or set the " "GUTILS_NETCDF_DIRECTORY environmental variable") sys.exit(parser.print_usage()) if not args.erddap_content_path: L.error("Please provide an --erddap_content_path agrument or set the " "GUTILS_ERDDAP_CONTENT_PATH environmental variable") sys.exit(parser.print_usage()) wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE wm.add_watch( args.data_path, mask, rec=True, auto_add=True ) processor = Netcdf2ErddapProcessor( outputs_path=args.data_path, erddap_content_path=args.erddap_content_path, erddap_flag_path=args.erddap_flag_path ) notifier = Notifier(wm, processor, read_freq=30) # Read every 30 seconds # Enable coalescing of events. This merges event types of the same type on the same file # together over the `read_freq` specified in the Notifier. notifier.coalesce_events() try: L.info("Watching {}, updating content at {} and flags at {}".format( args.data_path, args.erddap_content_path, args.erddap_flag_path )) notifier.loop(daemonize=args.daemonize) except NotifierError: L.exception('Unable to start notifier loop') return 1 except BaseException as e: L.exception(e) return 1 L.info("GUTILS netcdf_to_erddap Exited Successfully") return 0
def main_to_ftp(): setup_cli_logger(logging.INFO) parser = create_ftp_arg_parser() args = parser.parse_args() if not args.data_path: L.error("Please provide an --data_path agrument or set the " "GUTILS_NETCDF_DIRECTORY environmental variable") sys.exit(parser.print_usage()) if not args.ftp_url: L.error("Please provide an --ftp_url agrument or set the " "GUTILS_FTP_URL environmental variable") sys.exit(parser.print_usage()) wm = WatchManager() mask = IN_MOVED_TO | IN_CLOSE_WRITE wm.add_watch( args.data_path, mask, rec=True, auto_add=True ) processor = Netcdf2FtpProcessor( ftp_url=args.ftp_url, ftp_user=args.ftp_user, ftp_pass=args.ftp_pass, ) notifier = Notifier(wm, processor, read_freq=10) # Read every 10 seconds # Enable coalescing of events. This merges event types of the same type on the same file # together over the `read_freq` specified in the Notifier. notifier.coalesce_events() try: L.info("Watching {} and Uploading to {}".format( args.data_path, args.ftp_url) ) notifier.loop(daemonize=args.daemonize) except NotifierError: L.exception('Unable to start notifier loop') return 1 except BaseException as e: L.exception(e) return 1 L.info("GUTILS netcdf_to_ftp Exited Successfully") return 0
class DirtyTracker(object): """Track the changes to (part of) a working tree.""" def __init__(self, tree, subpath='.'): self._tree = tree self._wm = WatchManager() self._process = _Process() self._notifier = Notifier(self._wm, self._process) self._notifier.coalesce_events(True) def check_excluded(p): return tree.is_control_filename(tree.relpath(p)) self._wdd = self._wm.add_watch(tree.abspath(subpath), MASK, rec=True, auto_add=True, exclude_filter=check_excluded) def _process_pending(self): if self._notifier.check_events(timeout=0): self._notifier.read_events() self._notifier.process_events() def __del__(self): self._notifier.stop() def mark_clean(self): """Mark the subtree as not having any changes.""" self._process_pending() self._process.paths.clear() self._process.created.clear() def is_dirty(self): """Check whether there are any changes.""" self._process_pending() return bool(self._process.paths) def paths(self): """Return the paths that have changed.""" self._process_pending() return self._process.paths def relpaths(self): """Return the paths relative to the tree root that changed.""" return set(self._tree.relpath(p) for p in self.paths())