def test_pub_sub_add_rm(self):
"""Test adding and removing publishers.
"""
from posttroll.publisher import Publish
from posttroll.subscriber import Subscribe
time.sleep(4)
with Subscribe("this_data", "counter", True) as sub:
self.assertEqual(len(sub.sub_addr), 0)
with Publish("data_provider",
0, ["this_data"],
nameservers=self.nameservers):
time.sleep(4)
six.next(sub.recv(2))
self.assertEqual(len(sub.sub_addr), 1)
time.sleep(3)
for msg in sub.recv(2):
if msg is None:
break
time.sleep(3)
self.assertEqual(len(sub.sub_addr), 0)
with Publish("data_provider_2",
0, ["another_data"],
nameservers=self.nameservers):
time.sleep(4)
six.next(sub.recv(2))
self.assertEqual(len(sub.sub_addr), 0)
def run(self):
with Subscribe("", TOPIC, True) as sub:
for new_msg in sub.recv():
try:
logger.debug("received message (%d)", len(self.msgs))
queue_msg(self.msgs, new_msg)
except Exception:
logger.exception("Can't queue message.")
def loop(self):
with Subscribe(services=[""],
addr_listener=True,
topics=["pytroll://heart/minion"]) as sub:
for msg in sub.recv(1):
if msg:
print msg.data
self.pid_url[msg.data["pid"]] = msg.data["url"]
self.watcher.reset_timer(msg.data["pid"])
def l1c_runner(config_filename, service_name):
"""The live runner for the NWCSAF/PPS l1c product generation."""
LOG.info("Start the NWCSAF/PPS level-1c runner - Service = %s", service_name)
l1c_proc = L1cProcessor(config_filename, service_name)
publish_name = service_name + '-runner'
with Subscribe('', l1c_proc.subscribe_topics, True) as sub:
with Publish(publish_name, 0, nameservers=l1c_proc.nameservers) as pub:
_run_subscribe_publisher(l1c_proc, service_name, sub, pub)
def main():
"""Make cat run."""
opts = arg_parse()
setup_logging(opts.log, opts.verbose)
cfg = RawConfigParser()
cfg.read(opts.config)
config = dict(cfg.items(opts.config_item))
services = ''
if 'services' in config:
services = config['services'].split(',')
nameservers = []
if 'nameservers' in config:
nameservers = config['nameservers'].split(',')
publish_port = 0
if 'publish_port' in config:
publish_port = int(config['publish_port'])
sub_nameserver = 'localhost'
if 'subscriber_nameserver' in config:
sub_nameserver = config['subscriber_nameserver']
sub_addresses = None
if 'subscriber_addresses' in config:
sub_addresses = config['subscriber_addresses'].split(',')
try:
with Publish("cat_" + opts.config_item,
port=publish_port,
nameservers=nameservers) as pub:
with Subscribe(services,
topics=config["topic"],
addr_listener=True,
addresses=sub_addresses,
nameserver=sub_nameserver) as sub:
for msg in sub.recv(2):
if msg is None:
continue
if msg.type == "collection":
new_msg = str(process_message(msg, config))
if new_msg is None:
continue
logger.info("Sending %s", new_msg)
pub.send(new_msg)
except KeyboardInterrupt:
logging.shutdown()
finally:
print("Thank you for using pytroll/cat!"
"See you soon on pytroll.org.")
def record(self):
"""Log stuff."""
try:
with Subscribe("", addr_listener=True) as sub:
for msg in sub.recv(timeout=1):
if msg:
logger.debug("got msg %s", str(msg))
self.insert_line(msg)
if not self.loop:
logger.info("Stop recording")
break
except:
logger.exception("Something went wrong in record")
raise
def record(self):
"""Log stuff."""
try:
with Subscribe("", addr_listener=True) as sub:
for msg in sub.recv(timeout=1):
if msg:
logger.debug("got msg %s", str(msg))
if msg.type in ['collection', 'file', 'dataset']:
self.insert_files(msg)
if not self.loop:
logger.info("Stop recording")
break
except Exception:
logger.exception("Something went wrong in record")
raise
def test_pub_sub_ctx(self):
"""Test publish and subscribe.
"""
with Publish("data_provider", 0, ["this_data"]) as pub:
with Subscribe("this_data", "counter") as sub:
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
def seviri_l1c_runner(options, service_name="unknown"):
"""The live runner for the SEVIRI l1c product generation"""
LOG.info("Start the SEVIRI l1C runner...")
LOG.debug("Listens for messages of type: %s",
str(options['message_types']))
ncpus_available = cpu_count()
LOG.info("Number of CPUs available = " + str(ncpus_available))
ncpus = int(options.get('num_of_cpus', 1))
LOG.info("Will use %d CPUs when running the CSPP SEVIRI instances", ncpus)
af_proc = ActiveL1cProcessor(ncpus)
with Subscribe('', options['message_types'], True) as sub:
with Publish('seviri_l1c_runner', 0) as publisher:
while True:
count = 0
af_proc.initialise(service_name)
for msg in sub.recv():
count = count + 1
status = af_proc.run(msg)
if not status:
break # end the loop and reinitialize !
LOG.debug("Received message data = %s",
str(af_proc.message_data))
LOG.info(
"Get the results from the multiptocessing pool-run")
for res in af_proc.cspp_results:
tmp_result_file = res.get()
af_proc.result_file = tmp_result_file
af_files = af_proc.deliver_output_file()
if af_proc.result_home == af_proc.working_home:
LOG.info(
"home_dir = working_dir no cleaning necessary")
else:
LOG.info("Cleaning up directory %s",
af_proc.working_home)
cleanup_workdir(af_proc.working_home + '/')
publish_l1c(publisher,
af_files,
af_proc.message_data,
orbit=af_proc.orbit_number,
publish_topic=af_proc.publish_topic,
environment=af_proc.environment,
site=af_proc.site)
LOG.info("L1C processing has completed.")
def run(self):
with Subscribe('', topics=self.attrs['listen'],
addr_listener=True) as sub:
for msg in sub.recv(1):
if msg is None:
if not self.loop:
break
else:
continue
# check that files are local
for uri in gen_dict_extract(msg.data, 'uri'):
urlobj = urlparse(uri)
if (urlobj.scheme not in ['', 'file']
and not socket.gethostbyname(
urlobj.netloc) in get_local_ips()):
break
else:
LOGGER.debug('We have a match: %s', str(msg))
#pathname = unpack(orig_pathname, **attrs)
info = self.attrs.get("info", {})
if info:
info = dict((elt.strip().split('=')
for elt in info.split(";")))
for infokey, infoval in info.items():
if "," in infoval:
info[infokey] = infoval.split(",")
# info.update(parse(attrs["origin"], orig_pathname))
# info['uri'] = pathname
# info['uid'] = os.path.basename(pathname)
info.update(msg.data)
info['request_address'] = self.attrs.get(
"request_address",
get_own_ip()) + ":" + self.attrs["request_port"]
old_data = msg.data
msg = Message(self.attrs["topic"], msg.type, info)
self.publisher.send(str(msg))
with file_cache_lock:
for filename in gen_dict_extract(old_data, 'uid'):
file_cache.appendleft(self.attrs["topic"] + '/' +
filename)
LOGGER.debug("Message sent: " + str(msg))
if not self.loop:
break
def test_pub_sub_add_rm(self):
"""Test adding and removing publishers.
"""
with Subscribe("this_data", "counter", True) as sub:
time.sleep(11)
self.assertEquals(len(sub.sub_addr), 0)
with Publish("data_provider", 0, ["this_data"]):
time.sleep(4)
sub.recv(2).next()
self.assertEquals(len(sub.sub_addr), 1)
time.sleep(3)
for msg in sub.recv(2):
if msg is None:
break
time.sleep(3)
self.assertEquals(len(sub.sub_addr), 0)
def test_pub_sub_ctx(self):
"""Test publish and subscribe.
"""
from posttroll.message import Message
from posttroll.publisher import Publish
from posttroll.subscriber import Subscribe
with Publish("data_provider", 0, ["this_data"]) as pub:
with Subscribe("this_data", "counter") as sub:
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = six.next(sub.recv(2))
if msg is not None:
self.assertEqual(str(msg), str(message))
tested = True
sub.close()
self.assertTrue(tested)
def product_filter_live_runner(options):
"""Listens and triggers processing"""
LOG.info("*** Start the (EUMETCast) Product-filter runner:")
LOG.debug("Listens for messages of type: %s",
str(options['message_types']))
with Subscribe('', options['message_types'], True) as subscr:
with Publish('product_filter_runner', 0) as publisher:
file_reg = {}
for msg in subscr.recv():
file_reg = start_product_filtering(file_reg,
msg,
options,
publisher=publisher)
# Cleanup in file registry (keep only the last 5):
keys = list(file_reg.keys())
if len(keys) > 5:
keys.sort()
file_reg.pop(keys[0])
def main():
# let ctrl-c work as it should.
signal.signal(signal.SIGINT, signal.SIG_DFL)
global logger
logger = tutil.setup_logging("msg_pub errors")
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.setsockopt(zmq.TCP_KEEPALIVE, 1)
socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 60)
socket.setsockopt(zmq.TCP_KEEPALIVE_CNT, 20)
socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 60)
socket.bind("tcp://*:29092")
logger.debug("Listening for subscribers.")
with Subscribe("", "", True) as sub:
for msg in sub.recv():
msg_string = msg.encode()
logger.debug("sending msg: %s", msg_string)
socket.send(bytes(msg_string, "UTF-8"))
def log(self):
"""Log stuff.
"""
with Subscribe(services=[""], addr_listener=True) as sub:
for msg in sub.recv(1):
if msg:
if msg.type in [
"log.debug", "log.info", "log.warning",
"log.error", "log.critical"
]:
getattr(LOGGER,
msg.type[4:])(msg.subject + " " + msg.sender +
" " + str(msg.data) + " " +
str(msg.time))
elif msg.binary:
LOGGER.debug("%s %s %s [binary] %s", msg.subject,
msg.sender, msg.type, str(msg.time))
else:
LOGGER.debug("%s %s %s %s %s", msg.subject, msg.sender,
msg.type, str(msg.data), str(msg.time))
if not self.loop:
LOGGER.info("Stop logging")
break
import logging
from logger import PytrollFormatter, PytrollHandler
logger = logging.getLogger("hrpt2")
logger.setLevel(logging.DEBUG)
try:
with Publish("l2prod", 0, ["HRPT/2"]) as pub:
ch = PytrollHandler(pub)
ch.setLevel(logging.DEBUG)
formatter = PytrollFormatter("/oper/polar/gds")
ch.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)
with Subscribe("HRPT/1b", "EOS/1") as sub1:
for msg in sub1.recv():
##data = msg.data
##if data["type"] != "HRPT 1b":
## continue
logger.error("hej")
print "Consumer got", msg
if msg is not None and msg.type == "file":
data = msg.data
data["type"] = "HRPT 2"
data["format"] = "Pytroll's netcdf"
print "publishing", Message('/dc/polar/gds', "file",
data).encode()
cfg = RawConfigParser()
cfg.read(opts.config)
config = dict(cfg.items(opts.config_item))
services = ''
if 'services' in config:
services = config['services'].split(',')
nameservers = []
if 'nameservers' in config:
nameservers = config['nameservers'].split(',')
try:
with Publish("cat_" + opts.config_item,
nameservers=nameservers) as pub:
with Subscribe(services, config["topic"], True) as sub:
for msg in sub.recv(2):
if msg is None:
continue
if msg.type == "collection":
new_msg = str(process_message(msg, config))
if new_msg is None:
continue
LOG.info("Sending %s", new_msg)
pub.send(new_msg)
except KeyboardInterrupt:
logging.shutdown()
finally:
print("Thank you for using pytroll/cat!"
"See you soon on pytroll.org.")
handler.setLevel(loglevel)
logging.getLogger('').setLevel(loglevel)
logging.getLogger('').addHandler(handler)
logging.getLogger("posttroll").setLevel(logging.INFO)
LOG = logging.getLogger("cat")
cfg = RawConfigParser()
cfg.read(opts.config)
config = dict(cfg.items(opts.config_item))
if 'service' not in config:
config['service'] = ''
try:
with Publish("cat") as pub:
with Subscribe(config['service'], config["topic"], True) as sub:
for msg in sub.recv(2):
if msg is None:
continue
if msg.type == "collection":
new_msg = str(process_message(msg, config))
if new_msg is None:
continue
LOG.info("Sending %s", new_msg)
pub.send(new_msg)
except KeyboardInterrupt:
logging.shutdown()
finally:
print("Thank you for using pytroll/cat!"
"See you soon on pytroll.org.")
" %(name)s] %(message)s", '%Y-%m-%d %H:%M:%S'))
if opts.verbose:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
handler.setLevel(loglevel)
logging.getLogger('').setLevel(loglevel)
logging.getLogger('').addHandler(handler)
logging.getLogger("posttroll").setLevel(logging.INFO)
logger = logging.getLogger("catter")
subjects = list(
set([config.get(section, "subject") for section in config.sections()]))
logger.info("Listening to %s", str(subjects))
with Publish("catter") as pub:
with Subscribe("", subjects, addr_listener=True) as sub:
logger.info("Ready, waiting for messages")
for msg in sub.recv():
logger.info("Got message for %s", msg.subject)
if msg.type == "collection":
mda = msg.data[0].copy()
section = (mda["platform"] + " " + mda["number"] + "/" +
mda["level"])
if section not in config.sections():
logger.debug("Skipping %s", section)
continue
logger.debug("Starting catting for %s", section)
cat = config.get(section, "cat")
pattern = config.get(section, "pattern")
mda["proc_time"] = datetime.utcnow()
mda["end_time"] = msg.data[-1]["end_time"]
# Pytroll is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
# You should have received a copy of the GNU General Public License along with
# pytroll. If not, see <http://www.gnu.org/licenses/>.
"""A very stupid consumer.
"""
from posttroll.subscriber import Subscribe
from posttroll.publisher import Publish
from posttroll.message import Message
try:
with Publish("l2prod", "HRPT 3", 9003) as pub:
with Subscribe("HRPT 2", "NWP") as sub1:
for msg in sub1.recv():
print "Consumer got", msg
if msg is not None and msg.type == "file":
data = msg.data
if data["type"] == "HRPT 2":
data["type"] = "HRPT 3"
data["format"] = "Pytroll's netcdf"
print "publishing", Message('/dc/polar/gds', "file",
data).encode()
pub.send(
Message('/dc/polar/gds', "file", data).encode())
except KeyboardInterrupt:
print "terminating consumer..."
def run(self):
"""Start listening to messages."""
with Subscribe('', topics=self.attrs['listen'],
addr_listener=True) as sub:
self._run(sub)
# Pytroll is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
# You should have received a copy of the GNU General Public License along with
# pytroll. If not, see <http://www.gnu.org/licenses/>.
"""A very stupid consumer.
"""
from posttroll.subscriber import Subscribe
from posttroll.publisher import Publish
#from posttroll.message import Message
try:
with Publish("cloudtype_e", "msg lvl4", 9003) as pub:
with Subscribe("HRIT lvl1.5", "NWCSAF", timeout=60) as sub1:
print "ok, let's go"
for msg in sub1.recv():
if (msg is not None and msg.data["type"] == "HRIT lvl1.5"
and msg.data["segment_number"] == "EPI"):
print "Consumer got", msg
if (msg is not None and msg.data["type"].startswith("NWCSAF")):
print "Consumer got", msg
# if msg is not None and msg.type == "file":
# data = msg.data
# if data["type"] == "HRPT 2":
# data["type"] = "HRPT 3"
# data["format"] = "Pytroll's netcdf"
# print "publishing", Message('/dc/polar/gds', "file",
# data).encode()
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
handler.setLevel(loglevel)
logging.getLogger('').setLevel(loglevel)
logging.getLogger('').addHandler(handler)
logging.getLogger("posttroll").setLevel(logging.INFO)
LOG = logging.getLogger("cat")
cfg = RawConfigParser()
cfg.read(opts.config)
config = dict(cfg.items(opts.config_item))
try:
with Publish("cat") as pub:
with Subscribe('', config["topic"], True) as sub:
for msg in sub.recv(2):
if msg is None:
continue
if msg.type == "collection":
new_msg = str(process_message(msg, config))
if new_msg is None:
continue
LOG.info("Sending %s", new_msg)
pub.send(new_msg)
except KeyboardInterrupt:
logging.shutdown()
finally:
print "Thank you for using pytroll/cat!" \
"See you soon on pytroll.org."
