def test_pub_sub_add_rm(self): """Test adding and removing publishers. """ from posttroll.publisher import Publish from posttroll.subscriber import Subscribe time.sleep(4) with Subscribe("this_data", "counter", True) as sub: self.assertEqual(len(sub.sub_addr), 0) with Publish("data_provider", 0, ["this_data"], nameservers=self.nameservers): time.sleep(4) six.next(sub.recv(2)) self.assertEqual(len(sub.sub_addr), 1) time.sleep(3) for msg in sub.recv(2): if msg is None: break time.sleep(3) self.assertEqual(len(sub.sub_addr), 0) with Publish("data_provider_2", 0, ["another_data"], nameservers=self.nameservers): time.sleep(4) six.next(sub.recv(2)) self.assertEqual(len(sub.sub_addr), 0)
def run(self): with Subscribe("", TOPIC, True) as sub: for new_msg in sub.recv(): try: logger.debug("received message (%d)", len(self.msgs)) queue_msg(self.msgs, new_msg) except Exception: logger.exception("Can't queue message.")
def loop(self): with Subscribe(services=[""], addr_listener=True, topics=["pytroll://heart/minion"]) as sub: for msg in sub.recv(1): if msg: print msg.data self.pid_url[msg.data["pid"]] = msg.data["url"] self.watcher.reset_timer(msg.data["pid"])
def l1c_runner(config_filename, service_name): """The live runner for the NWCSAF/PPS l1c product generation.""" LOG.info("Start the NWCSAF/PPS level-1c runner - Service = %s", service_name) l1c_proc = L1cProcessor(config_filename, service_name) publish_name = service_name + '-runner' with Subscribe('', l1c_proc.subscribe_topics, True) as sub: with Publish(publish_name, 0, nameservers=l1c_proc.nameservers) as pub: _run_subscribe_publisher(l1c_proc, service_name, sub, pub)
def main(): """Make cat run.""" opts = arg_parse() setup_logging(opts.log, opts.verbose) cfg = RawConfigParser() cfg.read(opts.config) config = dict(cfg.items(opts.config_item)) services = '' if 'services' in config: services = config['services'].split(',') nameservers = [] if 'nameservers' in config: nameservers = config['nameservers'].split(',') publish_port = 0 if 'publish_port' in config: publish_port = int(config['publish_port']) sub_nameserver = 'localhost' if 'subscriber_nameserver' in config: sub_nameserver = config['subscriber_nameserver'] sub_addresses = None if 'subscriber_addresses' in config: sub_addresses = config['subscriber_addresses'].split(',') try: with Publish("cat_" + opts.config_item, port=publish_port, nameservers=nameservers) as pub: with Subscribe(services, topics=config["topic"], addr_listener=True, addresses=sub_addresses, nameserver=sub_nameserver) as sub: for msg in sub.recv(2): if msg is None: continue if msg.type == "collection": new_msg = str(process_message(msg, config)) if new_msg is None: continue logger.info("Sending %s", new_msg) pub.send(new_msg) except KeyboardInterrupt: logging.shutdown() finally: print("Thank you for using pytroll/cat!" "See you soon on pytroll.org.")
def record(self): """Log stuff.""" try: with Subscribe("", addr_listener=True) as sub: for msg in sub.recv(timeout=1): if msg: logger.debug("got msg %s", str(msg)) self.insert_line(msg) if not self.loop: logger.info("Stop recording") break except: logger.exception("Something went wrong in record") raise
def record(self): """Log stuff.""" try: with Subscribe("", addr_listener=True) as sub: for msg in sub.recv(timeout=1): if msg: logger.debug("got msg %s", str(msg)) if msg.type in ['collection', 'file', 'dataset']: self.insert_files(msg) if not self.loop: logger.info("Stop recording") break except Exception: logger.exception("Something went wrong in record") raise
def test_pub_sub_ctx(self): """Test publish and subscribe. """ with Publish("data_provider", 0, ["this_data"]) as pub: with Subscribe("this_data", "counter") as sub: for counter in range(5): message = Message("/counter", "info", str(counter)) pub.send(str(message)) time.sleep(1) msg = sub.recv(2).next() if msg is not None: self.assertEquals(str(msg), str(message)) tested = True self.assertTrue(tested)
def seviri_l1c_runner(options, service_name="unknown"): """The live runner for the SEVIRI l1c product generation""" LOG.info("Start the SEVIRI l1C runner...") LOG.debug("Listens for messages of type: %s", str(options['message_types'])) ncpus_available = cpu_count() LOG.info("Number of CPUs available = " + str(ncpus_available)) ncpus = int(options.get('num_of_cpus', 1)) LOG.info("Will use %d CPUs when running the CSPP SEVIRI instances", ncpus) af_proc = ActiveL1cProcessor(ncpus) with Subscribe('', options['message_types'], True) as sub: with Publish('seviri_l1c_runner', 0) as publisher: while True: count = 0 af_proc.initialise(service_name) for msg in sub.recv(): count = count + 1 status = af_proc.run(msg) if not status: break # end the loop and reinitialize ! LOG.debug("Received message data = %s", str(af_proc.message_data)) LOG.info( "Get the results from the multiptocessing pool-run") for res in af_proc.cspp_results: tmp_result_file = res.get() af_proc.result_file = tmp_result_file af_files = af_proc.deliver_output_file() if af_proc.result_home == af_proc.working_home: LOG.info( "home_dir = working_dir no cleaning necessary") else: LOG.info("Cleaning up directory %s", af_proc.working_home) cleanup_workdir(af_proc.working_home + '/') publish_l1c(publisher, af_files, af_proc.message_data, orbit=af_proc.orbit_number, publish_topic=af_proc.publish_topic, environment=af_proc.environment, site=af_proc.site) LOG.info("L1C processing has completed.")
def run(self): with Subscribe('', topics=self.attrs['listen'], addr_listener=True) as sub: for msg in sub.recv(1): if msg is None: if not self.loop: break else: continue # check that files are local for uri in gen_dict_extract(msg.data, 'uri'): urlobj = urlparse(uri) if (urlobj.scheme not in ['', 'file'] and not socket.gethostbyname( urlobj.netloc) in get_local_ips()): break else: LOGGER.debug('We have a match: %s', str(msg)) #pathname = unpack(orig_pathname, **attrs) info = self.attrs.get("info", {}) if info: info = dict((elt.strip().split('=') for elt in info.split(";"))) for infokey, infoval in info.items(): if "," in infoval: info[infokey] = infoval.split(",") # info.update(parse(attrs["origin"], orig_pathname)) # info['uri'] = pathname # info['uid'] = os.path.basename(pathname) info.update(msg.data) info['request_address'] = self.attrs.get( "request_address", get_own_ip()) + ":" + self.attrs["request_port"] old_data = msg.data msg = Message(self.attrs["topic"], msg.type, info) self.publisher.send(str(msg)) with file_cache_lock: for filename in gen_dict_extract(old_data, 'uid'): file_cache.appendleft(self.attrs["topic"] + '/' + filename) LOGGER.debug("Message sent: " + str(msg)) if not self.loop: break
def test_pub_sub_add_rm(self): """Test adding and removing publishers. """ with Subscribe("this_data", "counter", True) as sub: time.sleep(11) self.assertEquals(len(sub.sub_addr), 0) with Publish("data_provider", 0, ["this_data"]): time.sleep(4) sub.recv(2).next() self.assertEquals(len(sub.sub_addr), 1) time.sleep(3) for msg in sub.recv(2): if msg is None: break time.sleep(3) self.assertEquals(len(sub.sub_addr), 0)
def test_pub_sub_ctx(self): """Test publish and subscribe. """ from posttroll.message import Message from posttroll.publisher import Publish from posttroll.subscriber import Subscribe with Publish("data_provider", 0, ["this_data"]) as pub: with Subscribe("this_data", "counter") as sub: for counter in range(5): message = Message("/counter", "info", str(counter)) pub.send(str(message)) time.sleep(1) msg = six.next(sub.recv(2)) if msg is not None: self.assertEqual(str(msg), str(message)) tested = True sub.close() self.assertTrue(tested)
def product_filter_live_runner(options): """Listens and triggers processing""" LOG.info("*** Start the (EUMETCast) Product-filter runner:") LOG.debug("Listens for messages of type: %s", str(options['message_types'])) with Subscribe('', options['message_types'], True) as subscr: with Publish('product_filter_runner', 0) as publisher: file_reg = {} for msg in subscr.recv(): file_reg = start_product_filtering(file_reg, msg, options, publisher=publisher) # Cleanup in file registry (keep only the last 5): keys = list(file_reg.keys()) if len(keys) > 5: keys.sort() file_reg.pop(keys[0])
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger logger = tutil.setup_logging("msg_pub errors") context = zmq.Context() socket = context.socket(zmq.PUB) socket.setsockopt(zmq.TCP_KEEPALIVE, 1) socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 60) socket.setsockopt(zmq.TCP_KEEPALIVE_CNT, 20) socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 60) socket.bind("tcp://*:29092") logger.debug("Listening for subscribers.") with Subscribe("", "", True) as sub: for msg in sub.recv(): msg_string = msg.encode() logger.debug("sending msg: %s", msg_string) socket.send(bytes(msg_string, "UTF-8"))
def log(self): """Log stuff. """ with Subscribe(services=[""], addr_listener=True) as sub: for msg in sub.recv(1): if msg: if msg.type in [ "log.debug", "log.info", "log.warning", "log.error", "log.critical" ]: getattr(LOGGER, msg.type[4:])(msg.subject + " " + msg.sender + " " + str(msg.data) + " " + str(msg.time)) elif msg.binary: LOGGER.debug("%s %s %s [binary] %s", msg.subject, msg.sender, msg.type, str(msg.time)) else: LOGGER.debug("%s %s %s %s %s", msg.subject, msg.sender, msg.type, str(msg.data), str(msg.time)) if not self.loop: LOGGER.info("Stop logging") break
import logging from logger import PytrollFormatter, PytrollHandler logger = logging.getLogger("hrpt2") logger.setLevel(logging.DEBUG) try: with Publish("l2prod", 0, ["HRPT/2"]) as pub: ch = PytrollHandler(pub) ch.setLevel(logging.DEBUG) formatter = PytrollFormatter("/oper/polar/gds") ch.setFormatter(formatter) # add ch to logger logger.addHandler(ch) with Subscribe("HRPT/1b", "EOS/1") as sub1: for msg in sub1.recv(): ##data = msg.data ##if data["type"] != "HRPT 1b": ## continue logger.error("hej") print "Consumer got", msg if msg is not None and msg.type == "file": data = msg.data data["type"] = "HRPT 2" data["format"] = "Pytroll's netcdf" print "publishing", Message('/dc/polar/gds', "file", data).encode()
cfg = RawConfigParser() cfg.read(opts.config) config = dict(cfg.items(opts.config_item)) services = '' if 'services' in config: services = config['services'].split(',') nameservers = [] if 'nameservers' in config: nameservers = config['nameservers'].split(',') try: with Publish("cat_" + opts.config_item, nameservers=nameservers) as pub: with Subscribe(services, config["topic"], True) as sub: for msg in sub.recv(2): if msg is None: continue if msg.type == "collection": new_msg = str(process_message(msg, config)) if new_msg is None: continue LOG.info("Sending %s", new_msg) pub.send(new_msg) except KeyboardInterrupt: logging.shutdown() finally: print("Thank you for using pytroll/cat!" "See you soon on pytroll.org.")
handler.setLevel(loglevel) logging.getLogger('').setLevel(loglevel) logging.getLogger('').addHandler(handler) logging.getLogger("posttroll").setLevel(logging.INFO) LOG = logging.getLogger("cat") cfg = RawConfigParser() cfg.read(opts.config) config = dict(cfg.items(opts.config_item)) if 'service' not in config: config['service'] = '' try: with Publish("cat") as pub: with Subscribe(config['service'], config["topic"], True) as sub: for msg in sub.recv(2): if msg is None: continue if msg.type == "collection": new_msg = str(process_message(msg, config)) if new_msg is None: continue LOG.info("Sending %s", new_msg) pub.send(new_msg) except KeyboardInterrupt: logging.shutdown() finally: print("Thank you for using pytroll/cat!" "See you soon on pytroll.org.")
" %(name)s] %(message)s", '%Y-%m-%d %H:%M:%S')) if opts.verbose: loglevel = logging.DEBUG else: loglevel = logging.INFO handler.setLevel(loglevel) logging.getLogger('').setLevel(loglevel) logging.getLogger('').addHandler(handler) logging.getLogger("posttroll").setLevel(logging.INFO) logger = logging.getLogger("catter") subjects = list( set([config.get(section, "subject") for section in config.sections()])) logger.info("Listening to %s", str(subjects)) with Publish("catter") as pub: with Subscribe("", subjects, addr_listener=True) as sub: logger.info("Ready, waiting for messages") for msg in sub.recv(): logger.info("Got message for %s", msg.subject) if msg.type == "collection": mda = msg.data[0].copy() section = (mda["platform"] + " " + mda["number"] + "/" + mda["level"]) if section not in config.sections(): logger.debug("Skipping %s", section) continue logger.debug("Starting catting for %s", section) cat = config.get(section, "cat") pattern = config.get(section, "pattern") mda["proc_time"] = datetime.utcnow() mda["end_time"] = msg.data[-1]["end_time"]
# Pytroll is distributed in the hope that it will be useful, but WITHOUT ANY # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. See the GNU General Public License for more details. # You should have received a copy of the GNU General Public License along with # pytroll. If not, see <http://www.gnu.org/licenses/>. """A very stupid consumer. """ from posttroll.subscriber import Subscribe from posttroll.publisher import Publish from posttroll.message import Message try: with Publish("l2prod", "HRPT 3", 9003) as pub: with Subscribe("HRPT 2", "NWP") as sub1: for msg in sub1.recv(): print "Consumer got", msg if msg is not None and msg.type == "file": data = msg.data if data["type"] == "HRPT 2": data["type"] = "HRPT 3" data["format"] = "Pytroll's netcdf" print "publishing", Message('/dc/polar/gds', "file", data).encode() pub.send( Message('/dc/polar/gds', "file", data).encode()) except KeyboardInterrupt: print "terminating consumer..."
def run(self): """Start listening to messages.""" with Subscribe('', topics=self.attrs['listen'], addr_listener=True) as sub: self._run(sub)
# Pytroll is distributed in the hope that it will be useful, but WITHOUT ANY # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. See the GNU General Public License for more details. # You should have received a copy of the GNU General Public License along with # pytroll. If not, see <http://www.gnu.org/licenses/>. """A very stupid consumer. """ from posttroll.subscriber import Subscribe from posttroll.publisher import Publish #from posttroll.message import Message try: with Publish("cloudtype_e", "msg lvl4", 9003) as pub: with Subscribe("HRIT lvl1.5", "NWCSAF", timeout=60) as sub1: print "ok, let's go" for msg in sub1.recv(): if (msg is not None and msg.data["type"] == "HRIT lvl1.5" and msg.data["segment_number"] == "EPI"): print "Consumer got", msg if (msg is not None and msg.data["type"].startswith("NWCSAF")): print "Consumer got", msg # if msg is not None and msg.type == "file": # data = msg.data # if data["type"] == "HRPT 2": # data["type"] = "HRPT 3" # data["format"] = "Pytroll's netcdf" # print "publishing", Message('/dc/polar/gds', "file", # data).encode()
loglevel = logging.DEBUG else: loglevel = logging.INFO handler.setLevel(loglevel) logging.getLogger('').setLevel(loglevel) logging.getLogger('').addHandler(handler) logging.getLogger("posttroll").setLevel(logging.INFO) LOG = logging.getLogger("cat") cfg = RawConfigParser() cfg.read(opts.config) config = dict(cfg.items(opts.config_item)) try: with Publish("cat") as pub: with Subscribe('', config["topic"], True) as sub: for msg in sub.recv(2): if msg is None: continue if msg.type == "collection": new_msg = str(process_message(msg, config)) if new_msg is None: continue LOG.info("Sending %s", new_msg) pub.send(new_msg) except KeyboardInterrupt: logging.shutdown() finally: print "Thank you for using pytroll/cat!" \ "See you soon on pytroll.org."