def test_pub_sub_add_rm(self):
"""Test adding and removing publishers.
"""
from posttroll.publisher import Publish
from posttroll.subscriber import Subscribe
time.sleep(4)
with Subscribe("this_data", "counter", True) as sub:
self.assertEqual(len(sub.sub_addr), 0)
with Publish("data_provider",
0, ["this_data"],
nameservers=self.nameservers):
time.sleep(4)
six.next(sub.recv(2))
self.assertEqual(len(sub.sub_addr), 1)
time.sleep(3)
for msg in sub.recv(2):
if msg is None:
break
time.sleep(3)
self.assertEqual(len(sub.sub_addr), 0)
with Publish("data_provider_2",
0, ["another_data"],
nameservers=self.nameservers):
time.sleep(4)
six.next(sub.recv(2))
self.assertEqual(len(sub.sub_addr), 0)
def test_pub_addresses(self):
"""Test retrieving addresses.
"""
from posttroll.ns import get_pub_addresses
from posttroll.publisher import Publish
with Publish("data_provider",
0, ["this_data"],
nameservers=self.nameservers):
time.sleep(3)
res = get_pub_addresses(["this_data"])
self.assertEqual(len(res), 1)
expected = {
u'status': True,
u'service': [u'data_provider', u'this_data'],
u'name': u'address'
}
for key, val in expected.items():
self.assertEqual(res[0][key], val)
self.assertTrue("receive_time" in res[0])
self.assertTrue("URI" in res[0])
res = get_pub_addresses(["data_provider"])
self.assertEqual(len(res), 1)
expected = {
u'status': True,
u'service': [u'data_provider', u'this_data'],
u'name': u'address'
}
for key, val in expected.items():
self.assertEqual(res[0][key], val)
self.assertTrue("receive_time" in res[0])
self.assertTrue("URI" in res[0])
def test_pub_addresses(self):
"""Test retrieving addresses."""
from posttroll.ns import get_pub_addresses
from posttroll.publisher import Publish
with Publish(six.text_type("data_provider"),
0, ["this_data"],
broadcast_interval=0.1):
time.sleep(.3)
res = get_pub_addresses(["this_data"], timeout=.5)
self.assertEqual(len(res), 1)
expected = {
u'status': True,
u'service': [u'data_provider', u'this_data'],
u'name': u'address'
}
for key, val in expected.items():
self.assertEqual(res[0][key], val)
self.assertTrue("receive_time" in res[0])
self.assertTrue("URI" in res[0])
res = get_pub_addresses([six.text_type("data_provider")])
self.assertEqual(len(res), 1)
expected = {
u'status': True,
u'service': [u'data_provider', u'this_data'],
u'name': u'address'
}
for key, val in expected.items():
self.assertEqual(res[0][key], val)
self.assertTrue("receive_time" in res[0])
self.assertTrue("URI" in res[0])
def receive_from_zmq(host,
port,
station,
environment,
excluded_platforms,
days=1):
"""Receive 2met! messages from zeromq.
"""
# socket = Subscriber(["tcp://localhost:9331"], ["2met!"])
sock = GMCSubscriber(host, port)
msg_rec = MessageReceiver(host, excluded_platforms)
with Publish("receiver", 0, ["HRPT 0", "PDS", "RDR", "EPS 0"]) as pub:
for rawmsg in sock.recv():
# TODO:
# - Watch for idle time in order to detect a hangout
LOGGER.debug("receive from 2met! %s", str(rawmsg))
string = TwoMetMessage(rawmsg)
to_send = msg_rec.receive(string)
if to_send is None:
continue
subject = "/".join(
("", to_send['format'], to_send['data_processing_level'],
station, environment, "polar", "direct_readout"))
msg = Message(subject, "file", to_send).encode()
LOGGER.debug("publishing %s", str(msg))
pub.send(msg)
if days:
msg_rec.clean_passes(days)
def l1c_runner(config_filename, service_name):
"""The live runner for the NWCSAF/PPS l1c product generation."""
LOG.info("Start the NWCSAF/PPS level-1c runner - Service = %s", service_name)
l1c_proc = L1cProcessor(config_filename, service_name)
publish_name = service_name + '-runner'
with Subscribe('', l1c_proc.subscribe_topics, True) as sub:
with Publish(publish_name, 0, nameservers=l1c_proc.nameservers) as pub:
_run_subscribe_publisher(l1c_proc, service_name, sub, pub)
def send_message(topic, msg_type, msg_data, nameservers=None, port=0):
"""Send monitoring message"""
if nameservers is None:
nameservers = []
if not isinstance(nameservers, list):
nameservers = [nameservers]
with Publish("trollflow-sat", port=port, nameservers=nameservers) as pub:
msg = Message(topic, msg_type, msg_data)
pub.send(str(msg))
def run(self):
with Publish(self.runner_name, 0, self.publish_topic) as publisher:
while self.loop:
retv = self.queue.get()
if retv != None:
LOG.info("Publish the files...")
publisher.send(retv)
def test_pub_unicode(self):
from posttroll.message import Message
from posttroll.publisher import Publish
message = Message("/pџтяöll", "info", 'hej')
with Publish("a_service", 9000) as pub:
try:
pub.send(message.encode())
except UnicodeDecodeError:
self.fail("Sending raised UnicodeDecodeError unexpectedly!")
def run(self):
LOG.debug("Using as nameservers: %s", OPTIONS['nameservers'])
with Publish('ears_iasi_lvl2_converter', 0, ['netCDF/3', ], nameservers=OPTIONS['nameservers']) as publisher:
while self.loop:
retv = self.queue.get()
if retv != None:
LOG.info("Publish the IASI level-2 netcdf file")
publisher.send(retv)
def _get_port(min_port=None, max_port=None):
from zmq.error import ZMQError
from posttroll.publisher import Publish
try:
# Create a publisher to a port selected randomly from
# the given range
with Publish("a_service", min_port=min_port, max_port=max_port) as pub:
return pub.port_number
except ZMQError:
return False
def main():
"""Make cat run."""
opts = arg_parse()
setup_logging(opts.log, opts.verbose)
cfg = RawConfigParser()
cfg.read(opts.config)
config = dict(cfg.items(opts.config_item))
services = ''
if 'services' in config:
services = config['services'].split(',')
nameservers = []
if 'nameservers' in config:
nameservers = config['nameservers'].split(',')
publish_port = 0
if 'publish_port' in config:
publish_port = int(config['publish_port'])
sub_nameserver = 'localhost'
if 'subscriber_nameserver' in config:
sub_nameserver = config['subscriber_nameserver']
sub_addresses = None
if 'subscriber_addresses' in config:
sub_addresses = config['subscriber_addresses'].split(',')
try:
with Publish("cat_" + opts.config_item,
port=publish_port,
nameservers=nameservers) as pub:
with Subscribe(services,
topics=config["topic"],
addr_listener=True,
addresses=sub_addresses,
nameserver=sub_nameserver) as sub:
for msg in sub.recv(2):
if msg is None:
continue
if msg.type == "collection":
new_msg = str(process_message(msg, config))
if new_msg is None:
continue
logger.info("Sending %s", new_msg)
pub.send(new_msg)
except KeyboardInterrupt:
logging.shutdown()
finally:
print("Thank you for using pytroll/cat!"
"See you soon on pytroll.org.")
def run(self):
with Publish('modis_dr_runner', 0, [
'EOS/1B',
]) as publisher:
while self.loop:
retv = self.queue.get()
if retv != None:
LOG.info("Publish the files...")
publisher.send(retv)
def run(self):
with Publish('ears_iasi_lvl2_converter', 0, [
'netCDF/3',
]) as publisher:
while self.loop:
retv = self.queue.get()
if retv != None:
LOG.info("Publish the IASI level-2 netcdf file")
publisher.send(retv)
def run(self):
with Publish('fengyun3_dr_runner',
0, [
self.publish_topic,
],
nameservers=self.nameservers) as publisher:
while self.loop:
retv = self.queue.get()
if retv != None:
LOG.info("Publish the files...")
publisher.send(retv)
def viirs_active_fire_runner(options, service_name):
"""The live runner for the CSPP VIIRS AF product generation"""
from multiprocessing import cpu_count
LOG.info("Start the VIIRS active fire runner...")
LOG.debug("Listens for messages of type: %s",
str(options['message_types']))
ncpus_available = cpu_count()
LOG.info("Number of CPUs available = " + str(ncpus_available))
ncpus = int(OPTIONS.get('ncpus', 1))
LOG.info(
"Will use %d CPUs when running the CSPP VIIRS Active Fires instances",
ncpus)
viirs_af_proc = ViirsActiveFiresProcessor(ncpus)
with posttroll.subscriber.Subscribe('', options['message_types'],
True) as subscr:
with Publish('viirs_active_fire_runner', 0) as publisher:
while True:
viirs_af_proc.initialise(service_name)
# for msg in subscr.recv(timeout=300):
for msg in subscr.recv():
status = viirs_af_proc.run(msg)
if not status:
break # end the loop and reinitialize !
LOG.debug("Received message data = %s",
str(viirs_af_proc.message_data))
LOG.info("Get the results from the multiptocessing pool-run")
for res in viirs_af_proc.cspp_results:
working_dir, tmp_result_files = res.get()
viirs_af_proc.result_files = tmp_result_files
af_files = viirs_af_proc.deliver_output_files()
LOG.info("Cleaning up directory %s", working_dir)
cleanup_cspp_workdir(working_dir)
publish_af(publisher,
af_files,
viirs_af_proc.message_data,
orbit=viirs_af_proc.orbit_number,
publish_topic=viirs_af_proc.publish_topic,
environment=viirs_af_proc.environment,
site=viirs_af_proc.site)
LOG.info("SDR processing has completed.")
return
def sst_live_runner():
"""Listens and triggers processing"""
LOG.info("*** Start the OSISAF SST post-processing runner:")
with posttroll.subscriber.Subscribe('', ['OSISAF/GHRSST', ], True) as subscr:
with Publish('sst_runner', 0) as publisher:
sstfile = {}
for msg in subscr.recv():
sstfile = start_sst_processing(
sstfile, msg, publisher=publisher)
# Cleanup in sstfile registry (keep only the last 5):
keys = sstfile.keys()
if len(keys) > 5:
keys.sort()
sstfile.pop(keys[0])
def test_pub_sub_ctx(self):
"""Test publish and subscribe.
"""
with Publish("data_provider", 0, ["this_data"]) as pub:
with Subscribe("this_data", "counter") as sub:
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
def run(self):
with Publish('PPS', 0, nameservers=self.nameservers) as publisher:
time.sleep(WAIT_SECONDS_TO_ALLOW_PUBLISHER_TO_BE_REGISTERED)
while True:
retv = self.queue.get()
if retv is not None:
LOG.info("Publish the message...")
publisher.send(retv)
LOG.info("Message published!")
else:
time.sleep(1.0)
break
def publish_new_files(bucket, config):
"""Publish files newly arrived in bucket."""
with Publish("s3_stalker") as pub:
time_back = config['timedelta']
subject = config['subject']
pattern = config.get('file_pattern')
with sleeper(2.5):
set_last_fetch(datetime.now(tz.UTC) - timedelta(**time_back))
s3_kwargs = config['s3_kwargs']
fs, files = get_last_files(bucket, pattern=pattern, **s3_kwargs)
messages = filelist_unzip_to_messages(fs, files, subject)
for message in messages:
logger.info("Publishing %s", str(message))
pub.send(str(message))
def run(args, conf):
config_items = get_config_items(args, conf)
LOGGER.debug("Setting up posttroll connection...")
with Publish("remover") as pub:
time.sleep(3)
LOGGER.debug("Ready")
tot_size = 0
tot_files = 0
for section in config_items:
size, num_files = clean_section(pub, section, conf,
is_dry_run=args.dry_run)
tot_size += size
tot_files += num_files
LOGGER.info("# removed files: %s", tot_files)
LOGGER.info("MB removed: %s", tot_size / 1000000)
def seviri_l1c_runner(options, service_name="unknown"):
"""The live runner for the SEVIRI l1c product generation"""
LOG.info("Start the SEVIRI l1C runner...")
LOG.debug("Listens for messages of type: %s",
str(options['message_types']))
ncpus_available = cpu_count()
LOG.info("Number of CPUs available = " + str(ncpus_available))
ncpus = int(options.get('num_of_cpus', 1))
LOG.info("Will use %d CPUs when running the CSPP SEVIRI instances", ncpus)
af_proc = ActiveL1cProcessor(ncpus)
with Subscribe('', options['message_types'], True) as sub:
with Publish('seviri_l1c_runner', 0) as publisher:
while True:
count = 0
af_proc.initialise(service_name)
for msg in sub.recv():
count = count + 1
status = af_proc.run(msg)
if not status:
break # end the loop and reinitialize !
LOG.debug("Received message data = %s",
str(af_proc.message_data))
LOG.info(
"Get the results from the multiptocessing pool-run")
for res in af_proc.cspp_results:
tmp_result_file = res.get()
af_proc.result_file = tmp_result_file
af_files = af_proc.deliver_output_file()
if af_proc.result_home == af_proc.working_home:
LOG.info(
"home_dir = working_dir no cleaning necessary")
else:
LOG.info("Cleaning up directory %s",
af_proc.working_home)
cleanup_workdir(af_proc.working_home + '/')
publish_l1c(publisher,
af_files,
af_proc.message_data,
orbit=af_proc.orbit_number,
publish_topic=af_proc.publish_topic,
environment=af_proc.environment,
site=af_proc.site)
LOG.info("L1C processing has completed.")
def test_pub_sub_add_rm(self):
"""Test adding and removing publishers.
"""
with Subscribe("this_data", "counter", True) as sub:
time.sleep(11)
self.assertEquals(len(sub.sub_addr), 0)
with Publish("data_provider", 0, ["this_data"]):
time.sleep(4)
sub.recv(2).next()
self.assertEquals(len(sub.sub_addr), 1)
time.sleep(3)
for msg in sub.recv(2):
if msg is None:
break
time.sleep(3)
self.assertEquals(len(sub.sub_addr), 0)
def run(self):
with Publish(
'PPS',
0,
) as publisher:
time.sleep(WAIT_NSECS_PPS_PUBLISH)
while True:
retv = self.queue.get()
if retv != None:
LOG.info("Publish the message...")
publisher.send(retv)
LOG.info("Message published!")
else:
break
def modis_live_runner():
"""Listens and triggers processing"""
# Roll over log files at application start:
try:
LOG.handlers[0].doRollover()
except AttributeError:
LOG.warning("No log rotation supported for this handler...")
except IndexError:
LOG.debug("No handlers to rollover")
LOG.info("*** Start the MODIS level-1 runner:")
with posttroll.subscriber.Subscribe('receiver', ['PDS/0', ], True) as subscr:
with Publish('modis_dr_runner', 0) as publisher:
modisfiles = {}
for msg in subscr.recv():
modisfiles = start_modis_lvl1_processing(modisfiles,
publisher, msg)
def aapp_rolling_runner():
"""The AAPP runner. Listens and triggers processing on Metop/NOAA HRPT
level 0 files dispatched from Nimbus."""
LOG.info("*** Start the NOAA/Metop HRPT AAPP runner:")
aapp_proc = AappLvl1Processor()
with posttroll.subscriber.Subscribe('', ['HRPT/0', 'EPS/0'],
True) as subscr:
with Publish('aapp_runner', 0) as publisher:
while True:
aapp_proc.initialise()
for msg in subscr.recv(timeout=90):
status = aapp_proc.run(msg)
if not status:
break # end the loop and reinitialize!
tobj = aapp_proc.starttime
LOG.info("Time used in sub-dir name: " +
str(tobj.strftime("%Y-%m-%d %H:%M")))
if aapp_proc.platform_name.startswith('Metop'):
subd = create_subdirname(tobj, aapp_proc.platform_name,
aapp_proc.orbit)
LOG.info("Create sub-directory for level-1 files: %s" %
str(subd))
level1_files = aapp_proc.pack_aapplvl1_files(subd)
else:
LOG.info("Move sub-directory with NOAA level-1 files")
LOG.debug("Orbit BEFORE call to move_lvl1dir: " +
str(aapp_proc.orbit))
level1_files = aapp_proc.move_lvl1dir()
LOG.debug("Orbit AFTER call to move_lvl1dir: " +
str(aapp_proc.orbit))
publish_level1(publisher, level1_files,
aapp_proc.platform_name, aapp_proc.orbit,
aapp_proc.starttime, aapp_proc.endtime)
if aapp_proc.working_dir:
LOG.info("Cleaning up directory %s" %
aapp_proc.working_dir)
aapp_proc.cleanup_aapp_workdir()
return
def receive_from_zmq(host, port, station, environment, excluded_platforms,
target_server, ftp_prefix, topic_postfix,
publish_port=0, nameservers=None, days=1):
"""Receive 2met! messages from zeromq."""
logger.debug("target_server: %s", str(target_server))
logger.debug("ftp_prefix: %s", str(ftp_prefix))
logger.debug("topic_postfix: %s", str(topic_postfix))
logger.debug("station %s", str(station))
logger.debug(type(target_server))
# socket = Subscriber(["tcp://localhost:9331"], ["2met!"])
sock = GMCSubscriber(host, port)
msg_rec = MessageReceiver(host, excluded_platforms,
target_server, ftp_prefix)
with Publish("receiver", port=publish_port,
aliases=["HRPT 0", "PDS", "RDR", "EPS 0"],
nameservers=nameservers) as pub:
for rawmsg in sock.recv():
# TODO:
# - Watch for idle time in order to detect a hangout
logger.debug("receive from 2met! %s", str(rawmsg))
string = TwoMetMessage(rawmsg)
to_send = msg_rec.receive(string)
logger.debug("to_send: %s", str(to_send))
if to_send is None:
continue
if topic_postfix is not None:
subject = "/".join(("", to_send['format'],
to_send['data_processing_level'],
topic_postfix))
else:
subject = "/".join(("", to_send['format'],
to_send['data_processing_level'],
station, environment,
"polar", "direct_readout"))
logger.debug("Subject: %s", str(subject))
msg = Message(subject,
"file",
to_send).encode()
logger.debug("publishing %s", str(msg))
pub.send(msg)
if days:
msg_rec.clean_passes(days)
def zipcollector_live_runner(options):
"""Listen and trigger processing."""
logger.info("*** Start the zipcollector runner:")
logger.debug("Listens for messages of type: %s", options['message_type'])
with posttroll.subscriber.Subscribe('', [
options['message_type'],
], True) as subscr:
with Publish('zipcollector_runner', 0) as publisher:
file_reg = {}
for msg in subscr.recv():
file_reg = start_zipcollector(file_reg,
msg,
options,
publisher=publisher)
# Cleanup in file registry (keep only the last 5):
keys = list(file_reg.keys())
if len(keys) > 5:
keys.sort()
file_reg.pop(keys[0])
def product_filter_live_runner(options):
"""Listens and triggers processing"""
LOG.info("*** Start the (EUMETCast) Product-filter runner:")
LOG.debug("Listens for messages of type: %s",
str(options['message_types']))
with Subscribe('', options['message_types'], True) as subscr:
with Publish('product_filter_runner', 0) as publisher:
file_reg = {}
for msg in subscr.recv():
file_reg = start_product_filtering(file_reg,
msg,
options,
publisher=publisher)
# Cleanup in file registry (keep only the last 5):
keys = list(file_reg.keys())
if len(keys) > 5:
keys.sort()
file_reg.pop(keys[0])
def test_pub_sub_ctx(self):
"""Test publish and subscribe.
"""
from posttroll.message import Message
from posttroll.publisher import Publish
from posttroll.subscriber import Subscribe
with Publish("data_provider", 0, ["this_data"]) as pub:
with Subscribe("this_data", "counter") as sub:
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = six.next(sub.recv(2))
if msg is not None:
self.assertEqual(str(msg), str(message))
tested = True
sub.close()
self.assertTrue(tested)
def run(self):
"""Run the thread."""
self._loop = True
# Get save settings
kwargs = self._save_settings.copy()
# Initialize publisher context
with Publish("l2producer",
port=self._port,
nameservers=self._nameservers) as self.pub:
while self._loop:
if self.queue is not None:
try:
data = self.queue.get(True, 1)
if self.prev_lock is not None:
self.logger.debug(
"Writer acquires lock of "
"previous worker: %s", str(self.prev_lock))
utils.acquire_lock(self.prev_lock)
self.queue.task_done()
except queue_empty:
continue
if data is None:
self._compute()
self.data = []
self.messages = []
else:
self._process(data, **kwargs)
# After all the items have been processed, release the
# lock for the previous worker
if self.prev_lock is not None:
utils.release_locks(
[self.prev_lock], self.logger.debug,
"Writer releses lock of "
"previous worker: %s" % str(self.prev_lock))
else:
time.sleep(1)