def __init__(self, port, attrs=None): Thread.__init__(self) self._loop = True self.out_socket = get_context().socket(ROUTER) self.out_socket.bind("tcp://*:" + str(port)) self.port = port self.in_socket = get_context().socket(PULL) self.in_socket.bind("inproc://replies" + str(port)) self._poller = Poller() self._poller.register(self.out_socket, POLLIN) self._poller.register(self.in_socket, POLLIN) self._attrs = attrs try: # Checking the validity of the file pattern globify(attrs["origin"]) except ValueError as err: raise ConfigError('Invalid file pattern: ' + str(err)) except KeyError: if 'listen' not in attrs: raise self._deleter = Deleter(attrs) try: self._station = self._attrs["station"] except (KeyError, TypeError): LOGGER.warning("Station is not defined in config file") self._station = "unknown" LOGGER.debug("Station is '%s'", self._station)
def _update_chains(chains, new_chains, manager, use_watchdog, publisher, notifier_builder): old_glob = [] for chain_name, chain in new_chains.items(): chain_updated = False if chain_name in chains: if _chains_are_identical(chains, new_chains, chain_name): continue chain_updated = True _stop_chain(chains[chain_name]) if not _add_chain(chains, chain_name, chain, manager): continue fun = _create_notifier_and_get_function(notifier_builder, chains[chain_name], use_watchdog, chain, publisher) if 'origin' in chain: old_glob.append((globify(chain["origin"]), fun, chain)) if chain_updated: LOGGER.debug("Updated %s", chain_name) else: LOGGER.debug("Added %s", chain_name) return old_glob
def test_create_watchdog_notifier(process_notify): """Test creating a watchdog notifier.""" import time from trollmoves.server import create_watchdog_notifier from trollsift import globify fname = "20200428_1000_foo.tif" fname_pattern = "{start_time:%Y%m%d_%H%M}_{product}.tif" publisher = "publisher" with TemporaryDirectory() as tmpdir: pattern_path = os.path.join(tmpdir, fname_pattern) file_path = os.path.join(tmpdir, fname) attrs = {"origin": pattern_path} observer, fun = create_watchdog_notifier(attrs, publisher) observer.start() with open(os.path.join(file_path), "w") as fid: fid.write('') # Wait for a while for the watchdog to register the event time.sleep(2.0) observer.stop() observer.join() fun.assert_called_with(file_path, publisher, globify(pattern_path), attrs)
def push(self, message): """Reply to push request """ uri = urlparse(message.data["uri"]) pathname = uri.path if not fnmatch.fnmatch( os.path.basename(pathname), os.path.basename(globify(self._attrs["origin"]))): LOGGER.warning('Client trying to get invalid file: %s', pathname) return Message(message.subject, "err", data="{0:s} not reachable".format(pathname)) try: move_it(message, self._attrs) except Exception as err: return Message(message.subject, "err", data=str(err)) else: if (self._attrs.get('compression') or self._attrs.get( 'delete', 'False').lower() in ["1", "yes", "true", "on"]): self._deleter.add(pathname) new_msg = Message(message.subject, "file", data=message.data.copy()) new_msg.data['destination'] = clean_url( new_msg.data['destination']) return new_msg
def create_notifier(attrs): """Create a notifier from the specified configuration attributes *attrs*. """ tmask = (pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO | pyinotify.IN_CREATE) wm_ = pyinotify.WatchManager() opath, ofile = os.path.split(globify(attrs["origin"])) def fun(pathname): """Execute unpacking and copying/moving of *pathname* """ efile = os.path.basename(pathname) if fnmatch.fnmatch(efile, ofile): LOGGER.info("We have a match: " + str(pathname)) if attrs["compression"]: try: unpack_fun = eval(attrs["compression"]) if "prog" in attrs: new_path = unpack_fun(pathname, attrs["working_directory"], attrs["prog"]) else: new_path = unpack_fun(pathname, attrs["working_directory"]) except: LOGGER.exception("Could not decompress " + pathname) return else: new_path = pathname try: move_it(new_path, attrs["destinations"], attrs.get("copy_hook", None)) except Exception, e: LOGGER.error("Something went wrong during copy of " + pathname) else: if attrs["delete"]: try: os.remove(pathname) if attrs["delete_hook"]: attrs["delete_hook"](pathname) LOGGER.debug("Removed " + pathname) except OSError, e__: if e__.errno == 2: LOGGER.debug("Already deleted: " + pathname) else: raise # delete temporary file if pathname != new_path: try: os.remove(new_path) except OSError, e__: if e__.errno == 2: pass else: raise
def ack(self, message): """Reply with ack to a publication """ for url in gen_dict_extract(message.data, 'uri'): uri = urlparse(url) pathname = uri.path if 'origin' in self._attrs and not fnmatch.fnmatch( os.path.basename(pathname), os.path.basename(globify(self._attrs["origin"]))): LOGGER.warning('Client trying to get invalid file: %s', pathname) return Message(message.subject, "err", data="{0:s} not reacheable".format(pathname)) if (self._attrs.get('compression') or self._attrs.get( 'delete', 'False').lower() in ["1", "yes", "true", "on"]): self._deleter.add(pathname) new_msg = Message(message.subject, "ack", data=message.data.copy()) try: new_msg.data['destination'] = clean_url( new_msg.data['destination']) except KeyError: pass return new_msg
def _validate_file_pattern(self): try: _ = globify(self._attrs["origin"]) except ValueError as err: raise ConfigError('Invalid file pattern: ' + str(err)) except KeyError: if 'listen' not in self._attrs: raise
def _validate_requested_file(self, pathname, message): # FIXME: check against file_cache if 'origin' in self._attrs and not fnmatch.fnmatch( os.path.basename(pathname), os.path.basename(globify(self._attrs["origin"]))): LOGGER.warning('Client trying to get invalid file: %s', pathname) return Message(message.subject, "err", data="{0:s} not reachable".format(pathname)) return None
def create_watchdog_notifier(attrs, publisher): """Create a notifier from the specified configuration attributes *attrs*.""" pattern = globify(attrs["origin"]) opath = os.path.dirname(pattern) timeout = float(attrs.get("watchdog_timeout", 1.)) LOGGER.debug("Watchdog timeout: %.1f", timeout) observer = PollingObserver(timeout=timeout) handler = WatchdogHandler(process_notify, publisher, pattern, attrs) observer.schedule(handler, opath) return observer, process_notify
def test_process_notify(Message): """Test process_notify().""" from trollmoves.server import process_notify from trollmoves.server import file_cache, file_cache_lock from trollsift import globify import datetime as dt fname = "20200428_1000_foo.tif" fname_pattern = "{start_time:%Y%m%d_%H%M}_{product}.tif" not_matching_pattern = "bar" publisher = MagicMock() with TemporaryDirectory() as tmpdir: matching_pattern = os.path.join(tmpdir, fname_pattern) pathname = os.path.join(tmpdir, fname) kwargs = { "origin": matching_pattern, "request_address": "localhost", "request_port": "9001", "topic": "/topic" } process_notify(pathname, publisher, not_matching_pattern, kwargs) publisher.assert_not_called() with open(os.path.join(pathname), "w") as fid: fid.write('foo') process_notify(pathname, publisher, globify(matching_pattern), kwargs) # Check that the message was formed correctly message_info = { 'start_time': dt.datetime(2020, 4, 28, 10, 0), 'product': 'foo', 'uri': pathname, 'uid': fname, 'request_address': 'localhost:9001' } Message.assert_called_with(kwargs['topic'], 'file', message_info) # Check that the publisher send was called publisher.send.assert_called_with(str(Message.return_value)) # Check that the file cache was updated with file_cache_lock: assert "/topic/20200428_1000_foo.tif" in file_cache assert len(file_cache) == 1
def create_file_notifier(attrs, publisher): """Create a notifier from the specified configuration attributes *attrs*. """ tmask = (pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO | pyinotify.IN_CREATE) wm_ = pyinotify.WatchManager() pattern = globify(attrs["origin"]) opath = os.path.dirname(pattern) def fun(orig_pathname): """Publish what we have.""" if not fnmatch.fnmatch(orig_pathname, pattern): return else: LOGGER.debug('We have a match: %s', orig_pathname) pathname = unpack(orig_pathname, **attrs) info = attrs.get("info", {}) if info: info = dict((elt.strip().split('=') for elt in info.split(";"))) for infokey, infoval in info.items(): if "," in infoval: info[infokey] = infoval.split(",") info.update(parse(attrs["origin"], orig_pathname)) info['uri'] = pathname info['uid'] = os.path.basename(pathname) info['request_address'] = attrs.get( "request_address", get_own_ip()) + ":" + attrs["request_port"] msg = Message(attrs["topic"], 'file', info) publisher.send(str(msg)) with file_cache_lock: file_cache.appendleft(attrs["topic"] + '/' + info["uid"]) LOGGER.debug("Message sent: " + str(msg)) tnotifier = pyinotify.ThreadedNotifier(wm_, EventHandler(fun)) wm_.add_watch(opath, tmask) return tnotifier, fun
def push(self, message): """Reply to push request """ for the_dict in gen_dict_contains(message.data, 'uri'): uri = urlparse(the_dict['uri']) rel_path = the_dict.get('path', '') pathname = uri.path # FIXME: check against file_cache if 'origin' in self._attrs and not fnmatch.fnmatch( os.path.basename(pathname), os.path.basename(globify(self._attrs["origin"]))): LOGGER.warning('Client trying to get invalid file: %s', pathname) return Message(message.subject, "err", data="{0:s} not reachable".format(pathname)) try: move_it(pathname, message.data['destination'], self._attrs, rel_path=rel_path) except Exception as err: return Message(message.subject, "err", data=str(err)) else: if (self._attrs.get('compression') or self._attrs.get('delete', 'False').lower() in ["1", "yes", "true", "on"]): self._deleter.add(pathname) if 'dataset' in message.data: mtype = 'dataset' elif 'collection' in message.data: mtype = 'collection' elif 'uid' in message.data: mtype = 'file' else: raise KeyError('No known metadata in message.') new_msg = Message(message.subject, mtype, data=message.data.copy()) new_msg.data['destination'] = clean_url(new_msg.data['destination']) return new_msg
def create_inotify_notifier(attrs, publisher): """Create a notifier from the specified configuration attributes *attrs*.""" tmask = (pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO | pyinotify.IN_CREATE | pyinotify.IN_DELETE) wm_ = pyinotify.WatchManager() pattern = globify(attrs["origin"]) opath = os.path.dirname(pattern) if 'origin_inotify_base_dir_skip_levels' in attrs: """If you need to inotify monitor for new directories within the origin this attribute tells the server how many levels to skip from the origin before staring to inorify monitor a directory Eg. origin=/tmp/{platform_name_dir}_{start_time_dir:%Y%m%d_%H%M}_{orbit_number_dir:05d}/ {sensor}_{platform_name}_{start_time:%Y%m%d_%H%M}_{orbit_number:05d}.{data_processing_level:3s} and origin_inotify_base_dir_skip_levels=-2 this means the inotify monitor will use opath=/tmp""" pattern_list = pattern.split('/') pattern_join = os.path.join( *pattern_list[:int(attrs['origin_inotify_base_dir_skip_levels'])]) opath = os.path.join("/", pattern_join) LOGGER.debug("Using %s as base path for pyinotify add_watch.", opath) def process_notify_publish(pathname): pattern = globify(attrs["origin"]) return process_notify(pathname, publisher, pattern, attrs) tnotifier = pyinotify.ThreadedNotifier( wm_, EventHandler(process_notify_publish, watchManager=wm_, tmask=tmask)) wm_.add_watch(opath, tmask) return tnotifier, process_notify
def __init__(self, configfile, shp_boarders, shp_mask, regional_filtermask=None): """Initialize the active fires post processor class.""" super().__init__() self.shp_boarders = shp_boarders self.shp_filtermask = shp_mask self.regional_filtermask = regional_filtermask self.configfile = configfile self.options = {} config = read_config(self.configfile) self._set_options_from_config(config) self.host = socket.gethostname() self.timezone = self.options.get('timezone', 'GMT') self.input_topic = self.options['subscribe_topics'][0] self.output_topic = self.options['publish_topic'] self.infile_pattern = self.options.get('af_pattern_ibands') self.outfile_pattern_national = self.options.get( 'geojson_file_pattern_national') self.outfile_pattern_regional = self.options.get( 'geojson_file_pattern_regional') self.output_dir = self.options.get('output_dir', '/tmp') frmt = self.options['regional_shapefiles_format'] self.regional_shapefiles_globstr = globify(frmt) self.listener = None self.publisher = None self.loop = False self._setup_and_start_communication()
def reload_config(filename, disable_backlog=False): """Rebuild chains if needed (if the configuration changed) from *filename*. """ LOGGER.debug("New config file detected! %s", filename) new_chains = read_config(filename) old_glob = [] config_changed = False for key, val in new_chains.items(): identical = True if key in chains: for key2, val2 in new_chains[key].items(): if ((key2 not in ["notifier", "publisher"]) and ((key2 not in chains[key]) or (chains[key][key2] != val2))): identical = False config_changed = True break if identical: continue chains[key]["notifier"].stop() if "publisher" in chains[key]: chains[key]["publisher"].stop() chains[key] = val try: chains[key]["publisher"] = NoisyPublisher("move_it_" + key, val["publish_port"]) except (KeyError, NameError): pass chains[key]["notifier"] = create_notifier(val) # create logger too! if "publisher" in chains[key]: pub = chains[key]["publisher"].start() chains[key]["notifier"].start() old_glob.append(globify(val["origin"])) if "publisher" in chains[key]: def copy_hook(pathname, dest, val=val, pub=pub): fname = os.path.basename(pathname) destination = urlunparse((dest.scheme, dest.hostname, os.path.join(dest.path, fname), dest.params, dest.query, dest.fragment)) info = val.get("info", "") if info: info = dict( (elt.strip().split('=') for elt in info.split(";"))) for infokey, infoval in info.items(): if "," in infoval: info[infokey] = infoval.split(",") else: info = {} try: info.update( parse(os.path.basename(val["origin"]), os.path.basename(pathname))) except ValueError: info.update( parse( os.path.basename( os.path.splitext(val["origin"])[0]), os.path.basename(pathname))) info['uri'] = destination info['uid'] = fname msg = Message(val["topic"], 'file', info) pub.send(str(msg)) LOGGER.debug("Message sent: %s", str(msg)) chains[key]["copy_hook"] = copy_hook def delete_hook(pathname, val=val, pub=pub): fname = os.path.basename(pathname) info = val.get("info", "") if info: info = dict( (elt.strip().split('=') for elt in info.split(";"))) info['uri'] = pathname info['uid'] = fname msg = Message(val["topic"], 'del', info) pub.send(str(msg)) LOGGER.debug("Message sent: %s", str(msg)) chains[key]["delete_hook"] = delete_hook if not identical: LOGGER.debug("Updated %s", key) else: LOGGER.debug("Added %s", key) for key in (set(chains.keys()) - set(new_chains.keys())): chains[key]["notifier"].stop() del chains[key] LOGGER.debug("Removed %s", key) if config_changed: LOGGER.debug("Reloaded config from %s", filename) else: LOGGER.debug("No changes to reload in %s", filename) if old_glob and not disable_backlog: fnames = [] for pattern in old_glob: fnames += glob.glob(pattern) if fnames: time.sleep(3) LOGGER.debug("Touching old files") for fname in fnames: if os.path.exists(fname): fp_ = open(fname, "ab") fp_.close() old_glob = [] LOGGER.info("Old files transferred")
def reload_config(filename, chains, notifier_builder=None, manager=RequestManager, publisher=None, disable_backlog=False): """Rebuild chains if needed (if the configuration changed) from *filename*. """ LOGGER.debug("New config file detected! " + filename) new_chains = read_config(filename) old_glob = [] for key, val in new_chains.items(): identical = True if key in chains: for key2, val2 in new_chains[key].items(): if ((key2 not in ["notifier", "publisher"]) and ((key2 not in chains[key]) or (chains[key][key2] != val2))): identical = False break if identical: continue chains[key]["notifier"].stop() if "request_manager" in chains[key]: chains[key]["request_manager"].stop() LOGGER.debug('Stopped reqman') chains[key] = val.copy() try: chains[key]["request_manager"] = manager(int(val["request_port"]), val) LOGGER.debug("Created request manager on port %s", val["request_port"]) except (KeyError, NameError): LOGGER.exception('In reading config') except ConfigError as err: LOGGER.error('Invalid config parameters in %s: %s', key, str(err)) LOGGER.warning('Remove and skip %s', key) del chains[key] continue if notifier_builder is None: if 'origin' in val: notifier_builder = create_file_notifier elif 'listen' in val: notifier_builder = create_posttroll_notifier chains[key]["notifier"], fun = notifier_builder(val, publisher) chains[key]["request_manager"].start() chains[key]["notifier"].start() if 'origin' in val: old_glob.append((globify(val["origin"]), fun)) if not identical: LOGGER.debug("Updated " + key) else: LOGGER.debug("Added " + key) for key in (set(chains.keys()) - set(new_chains.keys())): chains[key]["notifier"].stop() del chains[key] LOGGER.debug("Removed " + key) LOGGER.debug("Reloaded config from " + filename) if old_glob and not disable_backlog: time.sleep(3) for pattern, fun in old_glob: process_old_files(pattern, fun) LOGGER.debug("done reloading config")
def reload_config(filename): """Rebuild chains if needed (if the configuration changed) from *filename*. """ if os.path.abspath(filename) != os.path.abspath(cmd_args.config_file): return LOGGER.debug("New config file detected! " + filename) new_chains = read_config(filename) old_glob = [] for key, val in new_chains.iteritems(): identical = True if key in chains: for key2, val2 in new_chains[key].iteritems(): if ((key2 not in ["notifier", "publisher"]) and ((key2 not in chains[key]) or (chains[key][key2] != val2))): identical = False break if identical: continue chains[key]["notifier"].stop() if "publisher" in chains[key]: chains[key]["publisher"].stop() chains[key] = val try: chains[key]["publisher"] = NoisyPublisher("move_it_" + key, val["publish_port"]) except (KeyError, NameError): pass chains[key]["notifier"] = create_notifier(val) # create logger too! if "publisher" in chains[key]: pub = chains[key]["publisher"].start() chains[key]["notifier"].start() old_glob.append(globify(val["origin"])) if "publisher" in chains[key]: def copy_hook(pathname, dest, val=val, pub=pub): fname = os.path.basename(pathname) destination = urlunparse((dest.scheme, dest.hostname, os.path.join(dest.path, fname), dest.params, dest.query, dest.fragment)) info = val.get("info", "") if info: info = dict((elt.strip().split('=') for elt in info.split(";"))) for infokey, infoval in info.items(): if "," in infoval: info[infokey] = infoval.split(",") else: info = {} info.update(parse(val["origin"], pathname)) info['uri'] = destination info['uid'] = fname msg = Message(val["topic"], 'file', info) pub.send(str(msg)) LOGGER.debug("Message sent: " + str(msg)) chains[key]["copy_hook"] = copy_hook def delete_hook(pathname, val=val, pub=pub): fname = os.path.basename(pathname) info = val.get("info", "") if info: info = dict((elt.strip().split('=') for elt in info.split(";"))) info['uri'] = pathname info['uid'] = fname msg = Message(val["topic"], 'del', info) pub.send(str(msg)) LOGGER.debug("Message sent: " + str(msg)) chains[key]["delete_hook"] = delete_hook if not identical: LOGGER.debug("Updated " + key) else: LOGGER.debug("Added " + key) for key in (set(chains.keys()) - set(new_chains.keys())): chains[key]["notifier"].stop() del chains[key] LOGGER.debug("Removed " + key) LOGGER.debug("Reloaded config from " + filename) if old_glob: fnames = [] for pattern in old_glob: fnames += glob.glob(pattern) if fnames: time.sleep(3) LOGGER.debug("Touching old files") for fname in fnames: if os.path.exists(fname): fp_ = open(fname, "ab") fp_.close() old_glob = [] LOGGER.debug("done reloading config")
def process_notify_publish(pathname): pattern = globify(attrs["origin"]) return process_notify(pathname, publisher, pattern, attrs)
def create_file_notifier(attrs, publisher): """Create a notifier from the specified configuration attributes *attrs*. """ tmask = (pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO | pyinotify.IN_CREATE | pyinotify.IN_DELETE) wm_ = pyinotify.WatchManager() pattern = globify(attrs["origin"]) opath = os.path.dirname(pattern) if 'origin_inotify_base_dir_skip_levels' in attrs: """If you need to inotify monitor for new directories within the origin this attribute tells the server how many levels to skip from the origin before staring to inorify monitor a directory Eg. origin=/tmp/{platform_name_dir}_{start_time_dir:%Y%m%d_%H%M}_{orbit_number_dir:05d}/ {sensor}_{platform_name}_{start_time:%Y%m%d_%H%M}_{orbit_number:05d}.{data_processing_level:3s} and origin_inotify_base_dir_skip_levels=-2 this means the inotify monitor will use opath=/tmp""" pattern_list = pattern.split('/') pattern_join = os.path.join(*pattern_list[:int(attrs['origin_inotify_base_dir_skip_levels'])]) opath = os.path.join("/", pattern_join) LOGGER.debug("Using {} as base path for pyinotify add_watch.".format(opath)) def fun(orig_pathname): """Publish what we have.""" if not fnmatch.fnmatch(orig_pathname, pattern): return elif (os.stat(orig_pathname).st_size == 0): # Want to avoid files with size 0. return else: LOGGER.debug('We have a match: %s', orig_pathname) pathname = unpack(orig_pathname, **attrs) info = attrs.get("info", {}) if info: info = dict((elt.strip().split('=') for elt in info.split(";"))) for infokey, infoval in info.items(): if "," in infoval: info[infokey] = infoval.split(",") info.update(parse(attrs["origin"], orig_pathname)) info['uri'] = pathname info['uid'] = os.path.basename(pathname) info['request_address'] = attrs.get( "request_address", get_own_ip()) + ":" + attrs["request_port"] msg = Message(attrs["topic"], 'file', info) publisher.send(str(msg)) with file_cache_lock: file_cache.appendleft(attrs["topic"] + '/' + info["uid"]) LOGGER.debug("Message sent: %s", str(msg)) tnotifier = pyinotify.ThreadedNotifier(wm_, EventHandler(fun, watchManager=wm_, tmask=tmask)) wm_.add_watch(opath, tmask) return tnotifier, fun