def test_pub_sub_add_rm(self): """Test adding and removing publishers. """ from posttroll.publisher import Publish from posttroll.subscriber import Subscribe time.sleep(4) with Subscribe("this_data", "counter", True) as sub: self.assertEqual(len(sub.sub_addr), 0) with Publish("data_provider", 0, ["this_data"], nameservers=self.nameservers): time.sleep(4) six.next(sub.recv(2)) self.assertEqual(len(sub.sub_addr), 1) time.sleep(3) for msg in sub.recv(2): if msg is None: break time.sleep(3) self.assertEqual(len(sub.sub_addr), 0) with Publish("data_provider_2", 0, ["another_data"], nameservers=self.nameservers): time.sleep(4) six.next(sub.recv(2)) self.assertEqual(len(sub.sub_addr), 0)
def test_pub_addresses(self): """Test retrieving addresses. """ from posttroll.ns import get_pub_addresses from posttroll.publisher import Publish with Publish("data_provider", 0, ["this_data"], nameservers=self.nameservers): time.sleep(3) res = get_pub_addresses(["this_data"]) self.assertEqual(len(res), 1) expected = { u'status': True, u'service': [u'data_provider', u'this_data'], u'name': u'address' } for key, val in expected.items(): self.assertEqual(res[0][key], val) self.assertTrue("receive_time" in res[0]) self.assertTrue("URI" in res[0]) res = get_pub_addresses(["data_provider"]) self.assertEqual(len(res), 1) expected = { u'status': True, u'service': [u'data_provider', u'this_data'], u'name': u'address' } for key, val in expected.items(): self.assertEqual(res[0][key], val) self.assertTrue("receive_time" in res[0]) self.assertTrue("URI" in res[0])
def test_pub_addresses(self): """Test retrieving addresses.""" from posttroll.ns import get_pub_addresses from posttroll.publisher import Publish with Publish(six.text_type("data_provider"), 0, ["this_data"], broadcast_interval=0.1): time.sleep(.3) res = get_pub_addresses(["this_data"], timeout=.5) self.assertEqual(len(res), 1) expected = { u'status': True, u'service': [u'data_provider', u'this_data'], u'name': u'address' } for key, val in expected.items(): self.assertEqual(res[0][key], val) self.assertTrue("receive_time" in res[0]) self.assertTrue("URI" in res[0]) res = get_pub_addresses([six.text_type("data_provider")]) self.assertEqual(len(res), 1) expected = { u'status': True, u'service': [u'data_provider', u'this_data'], u'name': u'address' } for key, val in expected.items(): self.assertEqual(res[0][key], val) self.assertTrue("receive_time" in res[0]) self.assertTrue("URI" in res[0])
def receive_from_zmq(host, port, station, environment, excluded_platforms, days=1): """Receive 2met! messages from zeromq. """ # socket = Subscriber(["tcp://localhost:9331"], ["2met!"]) sock = GMCSubscriber(host, port) msg_rec = MessageReceiver(host, excluded_platforms) with Publish("receiver", 0, ["HRPT 0", "PDS", "RDR", "EPS 0"]) as pub: for rawmsg in sock.recv(): # TODO: # - Watch for idle time in order to detect a hangout LOGGER.debug("receive from 2met! %s", str(rawmsg)) string = TwoMetMessage(rawmsg) to_send = msg_rec.receive(string) if to_send is None: continue subject = "/".join( ("", to_send['format'], to_send['data_processing_level'], station, environment, "polar", "direct_readout")) msg = Message(subject, "file", to_send).encode() LOGGER.debug("publishing %s", str(msg)) pub.send(msg) if days: msg_rec.clean_passes(days)
def l1c_runner(config_filename, service_name): """The live runner for the NWCSAF/PPS l1c product generation.""" LOG.info("Start the NWCSAF/PPS level-1c runner - Service = %s", service_name) l1c_proc = L1cProcessor(config_filename, service_name) publish_name = service_name + '-runner' with Subscribe('', l1c_proc.subscribe_topics, True) as sub: with Publish(publish_name, 0, nameservers=l1c_proc.nameservers) as pub: _run_subscribe_publisher(l1c_proc, service_name, sub, pub)
def send_message(topic, msg_type, msg_data, nameservers=None, port=0): """Send monitoring message""" if nameservers is None: nameservers = [] if not isinstance(nameservers, list): nameservers = [nameservers] with Publish("trollflow-sat", port=port, nameservers=nameservers) as pub: msg = Message(topic, msg_type, msg_data) pub.send(str(msg))
def run(self): with Publish(self.runner_name, 0, self.publish_topic) as publisher: while self.loop: retv = self.queue.get() if retv != None: LOG.info("Publish the files...") publisher.send(retv)
def test_pub_unicode(self): from posttroll.message import Message from posttroll.publisher import Publish message = Message("/pџтяöll", "info", 'hej') with Publish("a_service", 9000) as pub: try: pub.send(message.encode()) except UnicodeDecodeError: self.fail("Sending raised UnicodeDecodeError unexpectedly!")
def run(self): LOG.debug("Using as nameservers: %s", OPTIONS['nameservers']) with Publish('ears_iasi_lvl2_converter', 0, ['netCDF/3', ], nameservers=OPTIONS['nameservers']) as publisher: while self.loop: retv = self.queue.get() if retv != None: LOG.info("Publish the IASI level-2 netcdf file") publisher.send(retv)
def _get_port(min_port=None, max_port=None): from zmq.error import ZMQError from posttroll.publisher import Publish try: # Create a publisher to a port selected randomly from # the given range with Publish("a_service", min_port=min_port, max_port=max_port) as pub: return pub.port_number except ZMQError: return False
def main(): """Make cat run.""" opts = arg_parse() setup_logging(opts.log, opts.verbose) cfg = RawConfigParser() cfg.read(opts.config) config = dict(cfg.items(opts.config_item)) services = '' if 'services' in config: services = config['services'].split(',') nameservers = [] if 'nameservers' in config: nameservers = config['nameservers'].split(',') publish_port = 0 if 'publish_port' in config: publish_port = int(config['publish_port']) sub_nameserver = 'localhost' if 'subscriber_nameserver' in config: sub_nameserver = config['subscriber_nameserver'] sub_addresses = None if 'subscriber_addresses' in config: sub_addresses = config['subscriber_addresses'].split(',') try: with Publish("cat_" + opts.config_item, port=publish_port, nameservers=nameservers) as pub: with Subscribe(services, topics=config["topic"], addr_listener=True, addresses=sub_addresses, nameserver=sub_nameserver) as sub: for msg in sub.recv(2): if msg is None: continue if msg.type == "collection": new_msg = str(process_message(msg, config)) if new_msg is None: continue logger.info("Sending %s", new_msg) pub.send(new_msg) except KeyboardInterrupt: logging.shutdown() finally: print("Thank you for using pytroll/cat!" "See you soon on pytroll.org.")
def run(self): with Publish('modis_dr_runner', 0, [ 'EOS/1B', ]) as publisher: while self.loop: retv = self.queue.get() if retv != None: LOG.info("Publish the files...") publisher.send(retv)
def run(self): with Publish('ears_iasi_lvl2_converter', 0, [ 'netCDF/3', ]) as publisher: while self.loop: retv = self.queue.get() if retv != None: LOG.info("Publish the IASI level-2 netcdf file") publisher.send(retv)
def run(self): with Publish('fengyun3_dr_runner', 0, [ self.publish_topic, ], nameservers=self.nameservers) as publisher: while self.loop: retv = self.queue.get() if retv != None: LOG.info("Publish the files...") publisher.send(retv)
def viirs_active_fire_runner(options, service_name): """The live runner for the CSPP VIIRS AF product generation""" from multiprocessing import cpu_count LOG.info("Start the VIIRS active fire runner...") LOG.debug("Listens for messages of type: %s", str(options['message_types'])) ncpus_available = cpu_count() LOG.info("Number of CPUs available = " + str(ncpus_available)) ncpus = int(OPTIONS.get('ncpus', 1)) LOG.info( "Will use %d CPUs when running the CSPP VIIRS Active Fires instances", ncpus) viirs_af_proc = ViirsActiveFiresProcessor(ncpus) with posttroll.subscriber.Subscribe('', options['message_types'], True) as subscr: with Publish('viirs_active_fire_runner', 0) as publisher: while True: viirs_af_proc.initialise(service_name) # for msg in subscr.recv(timeout=300): for msg in subscr.recv(): status = viirs_af_proc.run(msg) if not status: break # end the loop and reinitialize ! LOG.debug("Received message data = %s", str(viirs_af_proc.message_data)) LOG.info("Get the results from the multiptocessing pool-run") for res in viirs_af_proc.cspp_results: working_dir, tmp_result_files = res.get() viirs_af_proc.result_files = tmp_result_files af_files = viirs_af_proc.deliver_output_files() LOG.info("Cleaning up directory %s", working_dir) cleanup_cspp_workdir(working_dir) publish_af(publisher, af_files, viirs_af_proc.message_data, orbit=viirs_af_proc.orbit_number, publish_topic=viirs_af_proc.publish_topic, environment=viirs_af_proc.environment, site=viirs_af_proc.site) LOG.info("SDR processing has completed.") return
def sst_live_runner(): """Listens and triggers processing""" LOG.info("*** Start the OSISAF SST post-processing runner:") with posttroll.subscriber.Subscribe('', ['OSISAF/GHRSST', ], True) as subscr: with Publish('sst_runner', 0) as publisher: sstfile = {} for msg in subscr.recv(): sstfile = start_sst_processing( sstfile, msg, publisher=publisher) # Cleanup in sstfile registry (keep only the last 5): keys = sstfile.keys() if len(keys) > 5: keys.sort() sstfile.pop(keys[0])
def test_pub_sub_ctx(self): """Test publish and subscribe. """ with Publish("data_provider", 0, ["this_data"]) as pub: with Subscribe("this_data", "counter") as sub: for counter in range(5): message = Message("/counter", "info", str(counter)) pub.send(str(message)) time.sleep(1) msg = sub.recv(2).next() if msg is not None: self.assertEquals(str(msg), str(message)) tested = True self.assertTrue(tested)
def run(self): with Publish('PPS', 0, nameservers=self.nameservers) as publisher: time.sleep(WAIT_SECONDS_TO_ALLOW_PUBLISHER_TO_BE_REGISTERED) while True: retv = self.queue.get() if retv is not None: LOG.info("Publish the message...") publisher.send(retv) LOG.info("Message published!") else: time.sleep(1.0) break
def publish_new_files(bucket, config): """Publish files newly arrived in bucket.""" with Publish("s3_stalker") as pub: time_back = config['timedelta'] subject = config['subject'] pattern = config.get('file_pattern') with sleeper(2.5): set_last_fetch(datetime.now(tz.UTC) - timedelta(**time_back)) s3_kwargs = config['s3_kwargs'] fs, files = get_last_files(bucket, pattern=pattern, **s3_kwargs) messages = filelist_unzip_to_messages(fs, files, subject) for message in messages: logger.info("Publishing %s", str(message)) pub.send(str(message))
def run(args, conf): config_items = get_config_items(args, conf) LOGGER.debug("Setting up posttroll connection...") with Publish("remover") as pub: time.sleep(3) LOGGER.debug("Ready") tot_size = 0 tot_files = 0 for section in config_items: size, num_files = clean_section(pub, section, conf, is_dry_run=args.dry_run) tot_size += size tot_files += num_files LOGGER.info("# removed files: %s", tot_files) LOGGER.info("MB removed: %s", tot_size / 1000000)
def seviri_l1c_runner(options, service_name="unknown"): """The live runner for the SEVIRI l1c product generation""" LOG.info("Start the SEVIRI l1C runner...") LOG.debug("Listens for messages of type: %s", str(options['message_types'])) ncpus_available = cpu_count() LOG.info("Number of CPUs available = " + str(ncpus_available)) ncpus = int(options.get('num_of_cpus', 1)) LOG.info("Will use %d CPUs when running the CSPP SEVIRI instances", ncpus) af_proc = ActiveL1cProcessor(ncpus) with Subscribe('', options['message_types'], True) as sub: with Publish('seviri_l1c_runner', 0) as publisher: while True: count = 0 af_proc.initialise(service_name) for msg in sub.recv(): count = count + 1 status = af_proc.run(msg) if not status: break # end the loop and reinitialize ! LOG.debug("Received message data = %s", str(af_proc.message_data)) LOG.info( "Get the results from the multiptocessing pool-run") for res in af_proc.cspp_results: tmp_result_file = res.get() af_proc.result_file = tmp_result_file af_files = af_proc.deliver_output_file() if af_proc.result_home == af_proc.working_home: LOG.info( "home_dir = working_dir no cleaning necessary") else: LOG.info("Cleaning up directory %s", af_proc.working_home) cleanup_workdir(af_proc.working_home + '/') publish_l1c(publisher, af_files, af_proc.message_data, orbit=af_proc.orbit_number, publish_topic=af_proc.publish_topic, environment=af_proc.environment, site=af_proc.site) LOG.info("L1C processing has completed.")
def test_pub_sub_add_rm(self): """Test adding and removing publishers. """ with Subscribe("this_data", "counter", True) as sub: time.sleep(11) self.assertEquals(len(sub.sub_addr), 0) with Publish("data_provider", 0, ["this_data"]): time.sleep(4) sub.recv(2).next() self.assertEquals(len(sub.sub_addr), 1) time.sleep(3) for msg in sub.recv(2): if msg is None: break time.sleep(3) self.assertEquals(len(sub.sub_addr), 0)
def run(self): with Publish( 'PPS', 0, ) as publisher: time.sleep(WAIT_NSECS_PPS_PUBLISH) while True: retv = self.queue.get() if retv != None: LOG.info("Publish the message...") publisher.send(retv) LOG.info("Message published!") else: break
def modis_live_runner(): """Listens and triggers processing""" # Roll over log files at application start: try: LOG.handlers[0].doRollover() except AttributeError: LOG.warning("No log rotation supported for this handler...") except IndexError: LOG.debug("No handlers to rollover") LOG.info("*** Start the MODIS level-1 runner:") with posttroll.subscriber.Subscribe('receiver', ['PDS/0', ], True) as subscr: with Publish('modis_dr_runner', 0) as publisher: modisfiles = {} for msg in subscr.recv(): modisfiles = start_modis_lvl1_processing(modisfiles, publisher, msg)
def aapp_rolling_runner(): """The AAPP runner. Listens and triggers processing on Metop/NOAA HRPT level 0 files dispatched from Nimbus.""" LOG.info("*** Start the NOAA/Metop HRPT AAPP runner:") aapp_proc = AappLvl1Processor() with posttroll.subscriber.Subscribe('', ['HRPT/0', 'EPS/0'], True) as subscr: with Publish('aapp_runner', 0) as publisher: while True: aapp_proc.initialise() for msg in subscr.recv(timeout=90): status = aapp_proc.run(msg) if not status: break # end the loop and reinitialize! tobj = aapp_proc.starttime LOG.info("Time used in sub-dir name: " + str(tobj.strftime("%Y-%m-%d %H:%M"))) if aapp_proc.platform_name.startswith('Metop'): subd = create_subdirname(tobj, aapp_proc.platform_name, aapp_proc.orbit) LOG.info("Create sub-directory for level-1 files: %s" % str(subd)) level1_files = aapp_proc.pack_aapplvl1_files(subd) else: LOG.info("Move sub-directory with NOAA level-1 files") LOG.debug("Orbit BEFORE call to move_lvl1dir: " + str(aapp_proc.orbit)) level1_files = aapp_proc.move_lvl1dir() LOG.debug("Orbit AFTER call to move_lvl1dir: " + str(aapp_proc.orbit)) publish_level1(publisher, level1_files, aapp_proc.platform_name, aapp_proc.orbit, aapp_proc.starttime, aapp_proc.endtime) if aapp_proc.working_dir: LOG.info("Cleaning up directory %s" % aapp_proc.working_dir) aapp_proc.cleanup_aapp_workdir() return
def receive_from_zmq(host, port, station, environment, excluded_platforms, target_server, ftp_prefix, topic_postfix, publish_port=0, nameservers=None, days=1): """Receive 2met! messages from zeromq.""" logger.debug("target_server: %s", str(target_server)) logger.debug("ftp_prefix: %s", str(ftp_prefix)) logger.debug("topic_postfix: %s", str(topic_postfix)) logger.debug("station %s", str(station)) logger.debug(type(target_server)) # socket = Subscriber(["tcp://localhost:9331"], ["2met!"]) sock = GMCSubscriber(host, port) msg_rec = MessageReceiver(host, excluded_platforms, target_server, ftp_prefix) with Publish("receiver", port=publish_port, aliases=["HRPT 0", "PDS", "RDR", "EPS 0"], nameservers=nameservers) as pub: for rawmsg in sock.recv(): # TODO: # - Watch for idle time in order to detect a hangout logger.debug("receive from 2met! %s", str(rawmsg)) string = TwoMetMessage(rawmsg) to_send = msg_rec.receive(string) logger.debug("to_send: %s", str(to_send)) if to_send is None: continue if topic_postfix is not None: subject = "/".join(("", to_send['format'], to_send['data_processing_level'], topic_postfix)) else: subject = "/".join(("", to_send['format'], to_send['data_processing_level'], station, environment, "polar", "direct_readout")) logger.debug("Subject: %s", str(subject)) msg = Message(subject, "file", to_send).encode() logger.debug("publishing %s", str(msg)) pub.send(msg) if days: msg_rec.clean_passes(days)
def zipcollector_live_runner(options): """Listen and trigger processing.""" logger.info("*** Start the zipcollector runner:") logger.debug("Listens for messages of type: %s", options['message_type']) with posttroll.subscriber.Subscribe('', [ options['message_type'], ], True) as subscr: with Publish('zipcollector_runner', 0) as publisher: file_reg = {} for msg in subscr.recv(): file_reg = start_zipcollector(file_reg, msg, options, publisher=publisher) # Cleanup in file registry (keep only the last 5): keys = list(file_reg.keys()) if len(keys) > 5: keys.sort() file_reg.pop(keys[0])
def product_filter_live_runner(options): """Listens and triggers processing""" LOG.info("*** Start the (EUMETCast) Product-filter runner:") LOG.debug("Listens for messages of type: %s", str(options['message_types'])) with Subscribe('', options['message_types'], True) as subscr: with Publish('product_filter_runner', 0) as publisher: file_reg = {} for msg in subscr.recv(): file_reg = start_product_filtering(file_reg, msg, options, publisher=publisher) # Cleanup in file registry (keep only the last 5): keys = list(file_reg.keys()) if len(keys) > 5: keys.sort() file_reg.pop(keys[0])
def test_pub_sub_ctx(self): """Test publish and subscribe. """ from posttroll.message import Message from posttroll.publisher import Publish from posttroll.subscriber import Subscribe with Publish("data_provider", 0, ["this_data"]) as pub: with Subscribe("this_data", "counter") as sub: for counter in range(5): message = Message("/counter", "info", str(counter)) pub.send(str(message)) time.sleep(1) msg = six.next(sub.recv(2)) if msg is not None: self.assertEqual(str(msg), str(message)) tested = True sub.close() self.assertTrue(tested)
def run(self): """Run the thread.""" self._loop = True # Get save settings kwargs = self._save_settings.copy() # Initialize publisher context with Publish("l2producer", port=self._port, nameservers=self._nameservers) as self.pub: while self._loop: if self.queue is not None: try: data = self.queue.get(True, 1) if self.prev_lock is not None: self.logger.debug( "Writer acquires lock of " "previous worker: %s", str(self.prev_lock)) utils.acquire_lock(self.prev_lock) self.queue.task_done() except queue_empty: continue if data is None: self._compute() self.data = [] self.messages = [] else: self._process(data, **kwargs) # After all the items have been processed, release the # lock for the previous worker if self.prev_lock is not None: utils.release_locks( [self.prev_lock], self.logger.debug, "Writer releses lock of " "previous worker: %s" % str(self.prev_lock)) else: time.sleep(1)