def main_to_ascii():
setup_cli_logger(logging.INFO)
parser = create_ascii_arg_parser()
args = parser.parse_args()
if not args.deployments_path:
L.error("Please provide a --deployments_path agrument or set the "
"GUTILS_DEPLOYMENTS_DIRECTORY environmental variable")
sys.exit(parser.print_usage())
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
wm.add_watch(args.deployments_path, mask, rec=True, auto_add=True)
# Convert binary data to ASCII
if args.type == 'slocum':
processor = Slocum2AsciiProcessor(
deployments_path=args.deployments_path)
notifier = Notifier(wm, processor, read_freq=10) # Read every 10 seconds
# Enable coalescing of events. This merges event types of the same type on the same file
# together over the `read_freq` specified in the Notifier.
notifier.coalesce_events()
try:
L.info(f"Watching {args.deployments_path} for new binary files")
notifier.loop(daemonize=args.daemonize)
except NotifierError:
L.exception('Unable to start notifier loop')
return 1
L.info("GUTILS binary_to_ascii Exited Successfully")
return 0
class QNotifier(QThread):
def __init__(self, wm, processor):
self.event_queue = list()
self._processor = processor
self.notifier = Notifier(wm,
NinjaProcessEvent(self.event_queue.append))
self.notifier.coalesce_events(True)
self.keep_running = True
QThread.__init__(self)
def run(self):
while self.keep_running:
try:
self.notifier.process_events()
except OSError:
pass # OSError: [Errno 2] No such file or directory happens
e_dict = {}
while len(self.event_queue):
e_type, e_path = self.event_queue.pop(0)
e_dict.setdefault(e_path, []).append(e_type)
keys = list(e_dict.keys())
while len(keys):
key = keys.pop(0)
event = e_dict.pop(key)
if (ADDED in event) and (DELETED in event):
event = [e for e in event if e not in (ADDED, DELETED)]
for each_event in event:
self._processor(each_event, key)
if self.notifier.check_events():
self.notifier.read_events()
self.notifier.stop()
class QNotifier(QThread):
def __init__(self, wm, processor):
self.event_queue = list()
self._processor = processor
self.notifier = Notifier(wm,
NinjaProcessEvent(self.event_queue.append))
self.notifier.coalesce_events(True)
self.keep_running = True
QThread.__init__(self)
def run(self):
while self.keep_running:
try:
self.notifier.process_events()
except OSError:
pass # OSError: [Errno 2] No such file or directory happens
e_dict = {}
while len(self.event_queue):
e_type, e_path = self.event_queue.pop(0)
e_dict.setdefault(e_path, []).append(e_type)
keys = e_dict.keys()
while len(keys):
key = keys.pop(0)
event = e_dict.pop(key)
if (ADDED in event) and (DELETED in event):
event = [e for e in event if e not in (ADDED, DELETED)]
for each_event in event:
self._processor(each_event, key)
if self.notifier.check_events():
self.notifier.read_events()
self.notifier.stop()
def fsMonitor(path="/data"):
wm = WatchManager()
mask = IN_DELETE | IN_MODIFY | IN_CREATE
notifier = Notifier(wm, EventHandler(), read_freq=10)
notifier.coalesce_events()
wm.add_watch(path, mask, rec=True, auto_add=True)
notifier.loop()
def FSMonitor(path='/root/wpf'):
wm = WatchManager()
mask = IN_DELETE | IN_MODIFY | IN_CREATE
notifier = Notifier(wm, EventHandler(), read_freq=10)
notifier.coalesce_events()
# 设置受监视的事件，这里只监视文件创建事件，（rec=True, auto_add=True）为递归处理
wm.add_watch(path, mask, rec=True, auto_add=True)
notifier.loop()
def FSMonitor(path='/root/wpf'):
wm = WatchManager()
mask = IN_DELETE | IN_MODIFY | IN_CREATE
notifier = Notifier(wm, EventHandler(),read_freq=10)
notifier.coalesce_events()
# 设置受监视的事件，这里只监视文件创建事件，（rec=True, auto_add=True）为递归处理
wm.add_watch(path,mask,rec=True, auto_add=True)
notifier.loop()
def main_to_netcdf():
setup_cli_logger(logging.INFO)
parser = create_netcdf_arg_parser()
args = parser.parse_args()
filter_args = vars(args)
# Remove non-filter args into positional arguments
deployments_path = filter_args.pop('deployments_path')
subset = filter_args.pop('subset')
daemonize = filter_args.pop('daemonize')
template = filter_args.pop('template')
profile_id_type = int(filter_args.pop('profile_id_type'))
# Move reader_class to a class
reader_class = filter_args.pop('reader_class')
if reader_class == 'slocum':
reader_class = SlocumReader
if not deployments_path:
L.error("Please provide a --deployments_path argument or set the "
"GUTILS_DEPLOYMENTS_DIRECTORY environmental variable")
sys.exit(parser.print_usage())
# Add inotify watch
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
wm.add_watch(
deployments_path,
mask,
rec=True,
auto_add=True
)
# Convert ASCII data to NetCDF using a specific reader class
if reader_class == SlocumReader:
processor = Slocum2NetcdfProcessor(
deployments_path=deployments_path,
subset=subset,
template=template,
profile_id_type=profile_id_type,
prefer_file_filters=True,
**filter_args
)
notifier = Notifier(wm, processor, read_freq=10)
# Enable coalescing of events. This merges event types of the same type on the same file
# together over the `read_freq` specified in the Notifier.
notifier.coalesce_events()
try:
L.info(f"Watching {deployments_path} for new ascii files")
notifier.loop(daemonize=daemonize)
except NotifierError:
L.exception('Unable to start notifier loop')
return 1
L.info("GUTILS ascii_to_netcdf Exited Successfully")
return 0
def main_to_erddap():
setup_cli_logger(logging.WARNING)
parser = create_erddap_arg_parser()
args = parser.parse_args()
if not args.data_path:
L.error("Please provide an --data_path agrument or set the "
"GUTILS_NETCDF_DIRECTORY environmental variable")
sys.exit(parser.print_usage())
if not args.erddap_content_path:
L.error("Please provide an --erddap_content_path agrument or set the "
"GUTILS_ERDDAP_CONTENT_PATH environmental variable")
sys.exit(parser.print_usage())
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
wm.add_watch(
args.data_path,
mask,
rec=True,
auto_add=True
)
processor = Netcdf2ErddapProcessor(
outputs_path=args.data_path,
erddap_content_path=args.erddap_content_path,
erddap_flag_path=args.erddap_flag_path
)
notifier = Notifier(wm, processor, read_freq=30) # Read every 30 seconds
# Enable coalescing of events. This merges event types of the same type on the same file
# together over the `read_freq` specified in the Notifier.
notifier.coalesce_events()
try:
L.info("Watching {}, updating content at {} and flags at {}".format(
args.data_path,
args.erddap_content_path,
args.erddap_flag_path
))
notifier.loop(daemonize=args.daemonize)
except NotifierError:
L.exception('Unable to start notifier loop')
return 1
except BaseException as e:
L.exception(e)
return 1
L.info("GUTILS netcdf_to_erddap Exited Successfully")
return 0
def main_to_ftp():
setup_cli_logger(logging.INFO)
parser = create_ftp_arg_parser()
args = parser.parse_args()
if not args.data_path:
L.error("Please provide an --data_path agrument or set the "
"GUTILS_NETCDF_DIRECTORY environmental variable")
sys.exit(parser.print_usage())
if not args.ftp_url:
L.error("Please provide an --ftp_url agrument or set the "
"GUTILS_FTP_URL environmental variable")
sys.exit(parser.print_usage())
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
wm.add_watch(
args.data_path,
mask,
rec=True,
auto_add=True
)
processor = Netcdf2FtpProcessor(
ftp_url=args.ftp_url,
ftp_user=args.ftp_user,
ftp_pass=args.ftp_pass,
)
notifier = Notifier(wm, processor, read_freq=10) # Read every 10 seconds
# Enable coalescing of events. This merges event types of the same type on the same file
# together over the `read_freq` specified in the Notifier.
notifier.coalesce_events()
try:
L.info("Watching {} and Uploading to {}".format(
args.data_path,
args.ftp_url)
)
notifier.loop(daemonize=args.daemonize)
except NotifierError:
L.exception('Unable to start notifier loop')
return 1
except BaseException as e:
L.exception(e)
return 1
L.info("GUTILS netcdf_to_ftp Exited Successfully")
return 0
class DirtyTracker(object):
"""Track the changes to (part of) a working tree."""
def __init__(self, tree, subpath='.'):
self._tree = tree
self._wm = WatchManager()
self._process = _Process()
self._notifier = Notifier(self._wm, self._process)
self._notifier.coalesce_events(True)
def check_excluded(p):
return tree.is_control_filename(tree.relpath(p))
self._wdd = self._wm.add_watch(tree.abspath(subpath),
MASK,
rec=True,
auto_add=True,
exclude_filter=check_excluded)
def _process_pending(self):
if self._notifier.check_events(timeout=0):
self._notifier.read_events()
self._notifier.process_events()
def __del__(self):
self._notifier.stop()
def mark_clean(self):
"""Mark the subtree as not having any changes."""
self._process_pending()
self._process.paths.clear()
self._process.created.clear()
def is_dirty(self):
"""Check whether there are any changes."""
self._process_pending()
return bool(self._process.paths)
def paths(self):
"""Return the paths that have changed."""
self._process_pending()
return self._process.paths
def relpaths(self):
"""Return the paths relative to the tree root that changed."""
return set(self._tree.relpath(p) for p in self.paths())
