def test_check_message_okay_message_ok(self):
"""Test for check if message is okay."""
input_msg = Message.decode(rawstr=TEST_INPUT_MSG)
result = check_message_okay(input_msg)
self.assertEqual(result, None)
input_msg = Message.decode(rawstr=TEST_INPUT_MSG_NO_DATASET)
result = check_message_okay(input_msg)
self.assertEqual(result, None)
def test_prepare_posttroll_message(setup_comm, get_config, gethostname):
"""Test setup the posttroll message."""
get_config.return_value = CONFIG_EXAMPLE
gethostname.return_value = "my.host.name"
myconfigfile = "/my/config/file/path"
myboarders_file = "/my/shape/file/with/country/boarders"
mymask_file = "/my/shape/file/with/polygons/to/filter/out"
afpp = ActiveFiresPostprocessing(myconfigfile, myboarders_file,
mymask_file)
test_filepath = "/my/geojson/file/path"
input_msg = Message.decode(rawstr=TEST_MSG)
res_msg = afpp._generate_output_message(test_filepath, input_msg)
assert res_msg.data['platform_name'] == 'NOAA-20'
assert res_msg.data['type'] == 'GEOJSON-filtered'
assert res_msg.data['format'] == 'geojson'
assert res_msg.data['product'] == 'afimg'
assert res_msg.subject == '/VIIRS/L2/Fires/PP/National'
assert res_msg.data['uri'] == 'ssh://my.host.name//my/geojson/file/path'
input_msg = Message.decode(rawstr=TEST_MSG)
fake_region_mask = {
'attributes': {
'Kod_omr': '9999',
'Testomr': 'Some area description'
}
}
res_msg = afpp._generate_output_message(test_filepath,
input_msg,
region=fake_region_mask)
assert res_msg.subject == '/VIIRS/L2/Fires/PP/Regional/9999'
msg_str = 'No fire detections for this granule'
result_messages = afpp._generate_no_fires_messages(input_msg, msg_str)
for (nat_or_reg, res_msg) in zip(['National', 'Regional'],
result_messages):
assert res_msg.data['info'] == msg_str
assert res_msg.subject == '/VIIRS/L2/Fires/PP/' + nat_or_reg
assert 'type' not in res_msg.data
assert 'format' not in res_msg.data
assert 'product' not in res_msg.data
assert 'uri' not in res_msg.data
def get_pub_address(name, timeout=10, nameserver="localhost"):
"""Get the address of the publisher for a given publisher *name* from the
nameserver on *nameserver* (localhost by default).
"""
# Socket to talk to server
socket = get_context().socket(REQ)
try:
socket.setsockopt(LINGER, timeout * 1000)
socket.connect("tcp://" + nameserver + ":" + str(PORT))
logger.debug('Connecting to %s',
"tcp://" + nameserver + ":" + str(PORT))
poller = Poller()
poller.register(socket, POLLIN)
message = Message("/oper/ns", "request", {"service": name})
socket.send_string(six.text_type(message))
# Get the reply.
sock = poller.poll(timeout=timeout * 1000)
if sock:
if sock[0][0] == socket:
message = Message.decode(socket.recv_string(NOBLOCK))
return message.data
else:
raise TimeoutError("Didn't get an address after %d seconds." %
timeout)
finally:
socket.close()
def run(self):
last_hb = datetime.now()
minutes = 2
while self._loop:
if datetime.now() - last_hb > timedelta(minutes=minutes):
logger.error("No heartbeat from " + str(self._pubaddress))
last_hb = datetime.now()
minutes = 1440
socks = dict(self._poller.poll(2000))
if (socks and
self._subsocket in socks and
socks[self._subsocket] == POLLIN):
message = Message.decode(self._subsocket.recv())
else:
continue
if message.type == "have":
sat = message.data["satellite"]
key = strp_isoformat(message.data["timecode"])
elevation = message.data["elevation"]
quality = message.data.get("quality", 100)
data = _MirrorGetter(self._req, sat, key)
self._holder.add(sat, key, elevation, quality, data)
if message.type == "heartbeat":
logger.debug("Got heartbeat from " + str(self._pubaddress)
+ ": " + str(message))
self._sched.mirror_next_pass = message.data["next_pass"]
last_hb = datetime.now()
minutes = 2
def get(self, timeout=None):
"""Return a published message or None if a timeout has happend.
"""
if timeout:
timeout *= 1000.
self.subscriber.connect(self.destination)
poller = zmq.Poller()
poller.register(self.subscriber, zmq.POLLIN)
try:
while (True):
try:
ret = poller.poll(timeout=timeout)
if ret:
if ret[0][0] == self.subscriber:
msg = Message.decode(
self.subscriber.recv(zmq.NOBLOCK))
yield msg
else:
logger.error("WHAT THE HECK")
else:
# timeout
yield None
except zmq.ZMQError:
logger.exception('recv failed')
finally:
poller.unregister(self.subscriber)
self.subscriber.close()
def get_address(data_type, timeout=2):
warnings.warn(
"nameclient.get_address shouldn't be used, " +
"use posttroll.subscriber.get_address instead...", DeprecationWarning)
context = zmq.Context()
# Socket to talk to server
socket = context.socket(zmq.REQ)
try:
socket.connect("tcp://localhost:5555")
msg = Message("/oper/ns", "request", {"type": data_type})
socket.send(str(msg))
# Get the reply.
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
s = poller.poll(timeout=timeout * 1000)
if s:
if s[0][0] == socket:
m = Message.decode(socket.recv(zmq.NOBLOCK))
return m.data
else:
raise RuntimeError("Unknown socket ?!?!?")
else:
raise TimeoutException("Didn't get an address after %d seconds." %
timeout)
print "Received reply to ", data_type, ": [", m, "]"
finally:
socket.close()
def get_pub_address(name, timeout=10, nameserver="localhost"):
"""Get the address of the publisher for a given publisher *name* from the
nameserver on *nameserver* (localhost by default).
"""
# Socket to talk to server
socket = context.socket(REQ)
try:
socket.setsockopt(LINGER, timeout * 1000)
socket.connect("tcp://" + nameserver + ":" + str(PORT))
poller = Poller()
poller.register(socket, POLLIN)
message = Message("/oper/ns", "request", {"service": name})
socket.send(str(message))
# Get the reply.
sock = poller.poll(timeout=timeout * 1000)
if sock:
if sock[0][0] == socket:
message = Message.decode(socket.recv(NOBLOCK))
return message.data
else:
raise TimeoutError("Didn't get an address after %d seconds."
% timeout)
finally:
socket.close()
def run(self):
print("starting SDR subscriber loop")
while True:
msg_bytes = self.socket.recv()
print("GOT SDR: {}".format(msg_bytes))
npnow = pd.to_datetime("now")
message = Message.decode(msg_bytes)
filename = os.path.basename(message.data["uri"])
file_time = datetime.strptime(filename[-69:-51],
"_d%Y%m%d_t%H%M%S")
npthen = pd.to_datetime(file_time)
delay = npnow - npthen
if self._sdrs.index.size > 0:
gap = npnow - self._sdrs.index[-1]
else:
gap = pd.Timedelta("0 seconds")
with self.lock:
self._sdrs.at[npnow] = (
message.data["segment"],
message.data["platform_name"],
pd.to_datetime(message.data["start_time"]),
pd.to_datetime(message.data["end_time"]),
message.data["orbit_number"],
pd.to_datetime(message.data["proctime"]),
message.data["uid"],
delay,
gap,
)
def _run(self):
port = broadcast_port
recv = MulticastReceiver(port).settimeout(2.0)
self._is_running = True
with Publish("address_receiver", ["addresses"], self._port) as pub:
try:
while self._do_run:
try:
data, fromaddr = recv()
del fromaddr
except SocketTimeout:
continue
msg = Message.decode(data)
name = msg.subject.split("/")[1]
if(msg.type == 'info' and
msg.subject.lower().endswith(self._subject)):
addr = msg.data["URI"]
metadata = copy.copy(msg.data)
metadata["name"] = name
if debug:
print 'receiving address', addr, name, metadata
if addr not in self._addresses:
pub.send(str(msg))
if debug:
print 'publishing address', addr, name, metadata
self._add(addr, metadata)
finally:
self._is_running = False
recv.close()
def test_metadata(self):
"""Test metadata encoding/decoding.
"""
metadata = copy.copy(SOME_METADATA)
msg = Message.decode(Message("/sat/polar/smb/level1", "file", data=metadata).encode())
self.assertTrue(msg.data == metadata, msg="Messaging, metadata decoding / encoding failed")
def test_create_publish_message(self, gethostname):
"""Test the creation of the publish message."""
gethostname.return_value = "my_local_server"
my_fake_level1c_file = '/my/level1c/file/path/level1c.nc'
input_msg = Message.decode(rawstr=TEST_INPUT_MSG)
result = prepare_l1c_message(my_fake_level1c_file,
input_msg.data,
orbit=99999)
expected = {
'data_type': 'MSG4',
'orig_platform_name': 'MSG4',
'start_time': datetime(2021, 5, 18, 14, 15),
'variant': '0DEG',
'series': 'MSG4',
'platform_name': 'Meteosat-11',
'channel': '',
'nominal_time': datetime(2021, 5, 18, 14, 15),
'compressed': '',
'origin': '172.18.0.248:9093',
'sensor': ['seviri'],
'uri': 'ssh://my_local_server/my/level1c/file/path/level1c.nc',
'uid': 'level1c.nc',
'format': 'PPS-L1C',
'type': 'NETCDF',
'data_processing_level': '1c'
}
self.assertDictEqual(result, expected)
def run(self):
last_hb = datetime.now()
minutes = 2
while self._loop:
if datetime.now() - last_hb > timedelta(minutes=minutes):
logger.error("No heartbeat from " + str(self._pubaddress))
last_hb = datetime.now()
minutes = 1440
socks = dict(self._poller.poll(2000))
if (socks and self._subsocket in socks
and socks[self._subsocket] == POLLIN):
message = Message.decode(self._subsocket.recv())
else:
continue
if message.type == "have":
sat = message.data["satellite"]
key = strp_isoformat(message.data["timecode"])
elevation = message.data["elevation"]
quality = message.data.get("quality", 100)
data = _MirrorGetter(self._req, sat, key)
self._holder.add(sat, key, elevation, quality, data)
if message.type == "heartbeat":
logger.debug("Got heartbeat from " + str(self._pubaddress) +
": " + str(message))
self._sched.mirror_next_pass = message.data["next_pass"]
last_hb = datetime.now()
minutes = 2
def get_address(data_type, timeout=2):
warnings.warn(
"nameclient.get_address shouldn't be used, " + "use posttroll.subscriber.get_address instead...",
DeprecationWarning,
)
context = zmq.Context()
# Socket to talk to server
socket = context.socket(zmq.REQ)
try:
socket.connect("tcp://localhost:5555")
msg = Message("/oper/ns", "request", {"type": data_type})
socket.send(str(msg))
# Get the reply.
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
s = poller.poll(timeout=timeout * 1000)
if s:
if s[0][0] == socket:
m = Message.decode(socket.recv(zmq.NOBLOCK))
return m.data
else:
raise RuntimeError("Unknown socket ?!?!?")
else:
raise TimeoutException("Didn't get an address after %d seconds." % timeout)
print "Received reply to ", data_type, ": [", m, "]"
finally:
socket.close()
def get(self, timeout=None):
"""Return a published message or None if a timeout has happend.
"""
if timeout:
timeout *= 1000.
self.subscriber.connect(self.destination)
poller = zmq.Poller()
poller.register(self.subscriber, zmq.POLLIN)
try:
while(True):
try:
ret = poller.poll(timeout=timeout)
if ret:
if ret[0][0] == self.subscriber:
msg = Message.decode(
self.subscriber.recv(zmq.NOBLOCK))
yield msg
else:
logger.error("WHAT THE HECK")
else:
# timeout
yield None
except zmq.ZMQError:
logger.exception('recv failed')
finally:
poller.unregister(self.subscriber)
self.subscriber.close()
def run(self, *args):
"""Run the listener and answer to requests.
"""
del args
arec = AddressReceiver(max_age=self._max_age)
arec.start()
port = PORT
try:
self.listener = context.socket(REP)
self.listener.bind("tcp://*:" + str(port))
poller = Poller()
poller.register(self.listener, POLLIN)
while self.loop:
socks = dict(poller.poll(1000))
if socks:
if socks.get(self.listener) == POLLIN:
msg = self.listener.recv()
else:
continue
logger.debug("Replying to request: " + str(msg))
msg = Message.decode(msg)
self.listener.send_unicode(str(get_active_address(
msg.data["service"], arec)))
except KeyboardInterrupt:
# Needed to stop the nameserver.
pass
finally:
arec.stop()
self.listener.close()
def get_messages(file_list):
messages = {}
for message_file in file_list:
print("reading {}".format(message_file))
with open(message_file, "r") as file:
message = Message.decode(file.read())
messages[get_key(message)] = message
return messages
def test_set_collection_area_id_not_in_message_and_not_in_config(self):
"""Test setting the collection area id."""
msg = Message.decode(self.message1)
config = self.config_mandatory
runconfig = DummyAappRunnerConfig(config, 'xl-band')
set_collection_area_id(msg.data, runconfig)
assert config['collection_area_id'] is None
def test_set_collection_area_id_present_in_config_but_not_in_message(self):
"""Test setting the collection area id."""
msg = Message.decode(self.message1)
config = self.config_complete
runconfig = DummyAappRunnerConfig(config, 'xl-band')
set_collection_area_id(msg.data, runconfig)
assert config['collection_area_id'] == 'euron1'
def test_get_options_regional_filtering(self, setup_comm, read_config, gethostname, netrc):
secrets = MyNetrcMock()
netrc.return_value = secrets
gethostname.return_value = 'default'
myconfigfile = "/my/config/file/path"
regstream = io.StringIO(REG_CONFIG)
read_config.return_value = yaml.load(regstream, Loader=yaml.UnsafeLoader)
this = EndUserNotifierRegional(myconfigfile)
expected = {'publish_topic': 'VIIRS/L2/MSB/Regional',
'subscribe_topics': ['VIIRS/L2/Fires/PP/Regional'],
'smtp_server': 'smtp.mydomain.xx', 'domain': 'mydomain.xx',
'sender': '*****@*****.**',
'recipients': {
'Area1-name': {'subject': 'My subject', 'Kod_omr': '0999', 'name': 'Name of my area 1',
'recipients': ['*****@*****.**'],
'recipients_attachment': ['*****@*****.**']},
'Area2-name': {'subject': 'My subject', 'Kod_omr': '0114', 'name':
'Name of my area 2', 'recipients': ['*****@*****.**'],
'recipients_attachment': ['*****@*****.**']}},
'max_number_of_fires_in_sms': 3, 'fire_data': ['power', 'observation_time'],
'unsubscribe': {'address': '*****@*****.**',
'text': 'Stop being bothered: Send a note to [email protected]'},
'unsubscribe_address': '*****@*****.**',
'unsubscribe_text': 'Stop being bothered: Send a note to [email protected]'}
self.assertDictEqual(expected, this.options)
assert this.smtp_server == 'smtp.mydomain.xx'
assert this.domain == 'mydomain.xx'
assert this.sender == '*****@*****.**'
# assert len(this.recipients) == 2
# recipients = this.recipients['Area1-name']
# expected = {'subject': 'My subject', 'knkod': '0999', 'name': 'Name of my area 1',
# 'recipients': ['*****@*****.**'],
# 'recipients_attachment': ['*****@*****.**']}
# self.assertDictEqual(expected, recipients)
# recipients = this.recipients['Area2-name']
# expected = {'subject': 'My subject', 'knkod': '0114', 'name': 'Name of my area 2',
# 'recipients': ['*****@*****.**'],
# 'recipients_attachment': ['*****@*****.**']}
# self.assertDictEqual(expected, recipients)
# assert this.subject is None
# assert this.max_number_of_fires_in_sms == 3
# self.assertListEqual(this.fire_data, ['power', 'observation_time'])
# assert this.unsubscribe_address == '*****@*****.**'
# assert this.input_topic == 'VIIRS/L2/Fires/PP/Regional'
# assert this.output_topic == 'VIIRS/L2/MSB/Regional'
input_msg = Message.decode(rawstr=REGIONAL_TEST_MESSAGE)
def test_metadata(self):
"""Test metadata encoding/decoding.
"""
metadata = copy.copy(SOME_METADATA)
msg = Message.decode(
Message('/sat/polar/smb/level1', 'file', data=metadata).encode())
self.assertTrue(msg.data == metadata,
msg='Messaging, metadata decoding / encoding failed')
def test_check_message_okay_message_has_no_start_time(self):
"""Test that message is not okay if it does not contain start_time."""
input_msg = Message.decode(rawstr=TEST_INPUT_MSG_NO_START_TIME)
with pytest.raises(MessageContentMissing) as exec_info:
_ = check_message_okay(input_msg)
exception_raised = exec_info.value
self.assertEqual("Message is lacking crucial fields: start_time",
str(exception_raised))
def test_encode_decode(self):
"""Test the encoding/decoding of the message class.
"""
msg1 = Message("/test/whatup/doc", "info", data="not much to say")
sender = "%s@%s" % (msg1.user, msg1.host)
self.assertTrue(sender == msg1.sender, msg="Messaging, decoding user, host from sender failed")
msg2 = Message.decode(msg1.encode())
self.assertTrue(str(msg2) == str(msg1), msg="Messaging, encoding, decoding failed")
def test_check_message_okay_message_has_no_dataset(self):
"""Test that message is not okay if it is not a dataset."""
input_msg = Message.decode(rawstr=TEST_INPUT_MSG_NO_DATASET_NO_FILE)
with pytest.raises(MessageTypeNotSupported) as exec_info:
_ = check_message_okay(input_msg)
exception_raised = exec_info.value
self.assertEqual("Not a dataset or a file, don't do anything...",
str(exception_raised))
def test_decode(self):
"""Test the decoding of a message.
"""
rawstr = (_MAGICK +
r'/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01' +
r' text/ascii "what' + r"'" + r's up doc"')
msg = Message.decode(rawstr)
self.assertTrue(str(msg) == rawstr,
msg='Messaging, decoding of message failed')
def test_decode(self):
"""Test the decoding of a message.
"""
rawstr = (
_MAGICK +
r'/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01' +
r' text/ascii "what' + r"'" + r's up doc"')
msg = Message.decode(rawstr)
self.assertTrue(str(msg) == rawstr,
msg='Messaging, decoding of message failed')
def test_decode(self):
"""Test the decoding of a message.
"""
rawstr = (
_MAGICK
+ "/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01"
+ ' application/json "what\'s up doc"'
)
msg = Message.decode(rawstr)
self.assertTrue(str(msg) == rawstr, msg="Messaging, decoding of message failed")
def test_decode(self):
"""Test the decoding of a message.
"""
rawstr = (
_MAGICK +
'/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01' +
' application/json "what\'s up doc"')
msg = Message.decode(rawstr)
self.assertTrue(str(msg) == rawstr,
msg='Messaging, decoding of message failed')
def test_encode_decode(self):
"""Test the encoding/decoding of the message class.
"""
msg1 = Message('/test/whatup/doc', 'info', data='not much to say')
sender = '%s@%s' % (msg1.user, msg1.host)
self.assertTrue(sender == msg1.sender,
msg='Messaging, decoding user, host from sender failed')
msg2 = Message.decode(msg1.encode())
self.assertTrue(str(msg2) == str(msg1),
msg='Messaging, encoding, decoding failed')
def test_encode_decode(self):
"""Test the encoding/decoding of the message class.
"""
msg1 = Message('/test/whatup/doc', 'info', data='not much to say')
sender = '%s@%s' % (msg1.user, msg1.host)
self.assertTrue(
sender == msg1.sender,
msg='Messaging, decoding user, host from sender failed')
msg2 = Message.decode(msg1.encode())
self.assertTrue(str(msg2) == str(msg1),
msg='Messaging, encoding, decoding failed')
def recv(self, timeout=None):
"""Receive, optionally with *timeout* in seconds.
"""
if timeout:
timeout *= 1000.
for sub in list(self.subscribers) + self._hooks:
self.poller.register(sub, POLLIN)
self._loop = True
try:
while self._loop:
sleep(0)
try:
socks = dict(self.poller.poll(timeout=timeout))
if socks:
for sub in self.subscribers:
if sub in socks and socks[sub] == POLLIN:
m__ = Message.decode(sub.recv_string(NOBLOCK))
if not self._filter or self._filter(m__):
if self._translate:
url = urlsplit(self.sub_addr[sub])
host = url[1].split(":")[0]
m__.sender = (
m__.sender.split("@")[0] + "@" +
host)
yield m__
for sub in self._hooks:
if sub in socks and socks[sub] == POLLIN:
m__ = Message.decode(sub.recv_string(NOBLOCK))
self._hooks_cb[sub](m__)
else:
# timeout
yield None
except ZMQError as err:
LOGGER.exception("Receive failed: %s", str(err))
finally:
for sub in list(self.subscribers) + self._hooks:
self.poller.unregister(sub)
def recv(self, timeout=None):
"""Receive, optionally with *timeout* in seconds.
"""
if timeout:
timeout *= 1000.
for sub in list(self.subscribers) + self._hooks:
self.poller.register(sub, POLLIN)
self._loop = True
try:
while self._loop:
sleep(0)
try:
socks = dict(self.poller.poll(timeout=timeout))
if socks:
for sub in self.subscribers:
if sub in socks and socks[sub] == POLLIN:
m__ = Message.decode(sub.recv_string(NOBLOCK))
if not self._filter or self._filter(m__):
if self._translate:
url = urlsplit(self.sub_addr[sub])
host = url[1].split(":")[0]
m__.sender = (m__.sender.split("@")[0]
+ "@" + host)
yield m__
for sub in self._hooks:
if sub in socks and socks[sub] == POLLIN:
m__ = Message.decode(sub.recv_string(NOBLOCK))
self._hooks_cb[sub](m__)
else:
# timeout
yield None
except ZMQError as err:
LOGGER.exception("Receive failed: %s", str(err))
finally:
for sub in list(self.subscribers) + self._hooks:
self.poller.unregister(sub)
def test_orbit_number_from_msg_viirs(self, cpu_count, config):
"""Test use orbit number from message."""
cpu_count.return_value = 1
config.return_value = self.config_viirs_orbit_number_from_msg_ok
input_msg = Message.decode(rawstr=TEST_INPUT_MESSAGE_VIIRS_MSG)
with patch('nwcsafpps_runner.l1c_processing.ThreadPool'):
with tempfile.NamedTemporaryFile() as myconfig_file:
l1c_proc = L1cProcessor(myconfig_file.name, 'viirs-l1c')
l1c_proc.run(input_msg)
expected_orbit_number = TEST_VIIRS_MSG_DATA.get('orbit_number')
self.assertEqual(l1c_proc.orbit_number, expected_orbit_number)
def process_message(msg_bytes):
try:
message = Message.decode(msg_bytes)
publish_products(message)
except MessageError:
logger.exception("Message decode error.")
except NotImplementedError:
logger.exception("Crap. I accepted a message I can't process.")
except ValueError:
logger.exception("I got a message, but couldn't find the data")
except KeyError:
logger.exception("missing data, skipping")
logger.debug("Whew, that was hard. Let rest for 10 seconds.")
time.sleep(10)
def test_orbit_number_missing_in_msg_viirs(self, cpu_count, config):
"""Test use orbit number but missing in message."""
cpu_count.return_value = 1
config.return_value = self.config_viirs_orbit_number_from_msg_ok
input_msg = Message.decode(
rawstr=TEST_INPUT_MESSAGE_VIIRS_NO_ORBIT_MSG)
with self.assertLogs('nwcsafpps_runner.l1c_processing',
level='INFO') as cm:
with patch('nwcsafpps_runner.l1c_processing.ThreadPool'):
with tempfile.NamedTemporaryFile() as myconfig_file:
l1c_proc = L1cProcessor(myconfig_file.name, 'viirs-l1c')
l1c_proc.run(input_msg)
expected_orbit_number = 99999
self.assertEqual(l1c_proc.orbit_number, expected_orbit_number)
self.assertEqual(
cm.output[2],
'WARNING:nwcsafpps_runner.l1c_processing:You asked for orbit_number '
'from the message, but its not there. Keep init orbit.')
def main():
# let ctrl-c work as it should.
signal.signal(signal.SIGINT, signal.SIG_DFL)
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.setsockopt(zmq.TCP_KEEPALIVE, 1)
socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 60)
socket.setsockopt(zmq.TCP_KEEPALIVE_CNT, 20)
socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 60)
socket.setsockopt_string(zmq.SUBSCRIBE, "pytroll://AVO/viirs/sdr")
socket.connect(SDR_PUBLISHER)
while True:
msg_bytes = socket.recv()
message = Message.decode(msg_bytes)
filename = os.path.basename(message.data["uri"])
file_time = datetime.strptime(filename[-69:-51], "_d%Y%m%d_t%H%M%S")
age = datetime.now() - file_time
print("{} (retrieval took {})".format(filename,
humanize.naturaldelta(age)))
def recv(self, timeout=None):
if timeout:
timeout *= 1000.
for sub in self.subscribers:
self.poller.register(sub, zmq.POLLIN)
self._loop = True
try:
while(self._loop):
try:
s = dict(self.poller.poll(timeout=timeout))
if s:
for sub in self.subscribers:
if sub in s and s[sub] == zmq.POLLIN:
m__ = Message.decode(sub.recv(zmq.NOBLOCK))
if self._translate:
url = urlsplit(self.sub_addr[sub])
host = url[1].split(":")[0]
m__.sender = (m__.sender.split("@")[0]
+ "@" + host)
# Only accept pre-defined data types
try:
if (self._data_types and
(m__.data['type'] not in
self._data_types)):
continue
except (KeyError, TypeError):
pass
yield m__
else:
# timeout
yield None
except zmq.ZMQError:
print >>sys.stderr, 'receive failed'
finally:
for sub in self.subscribers:
self.poller.unregister(sub)
def _run(self):
"""Run the receiver.
"""
port = broadcast_port
recv = MulticastReceiver(port).settimeout(2.)
self._is_running = True
with Publish("address_receiver", self._port, ["addresses"]) as pub:
try:
while self._do_run:
try:
data, fromaddr = recv()
del fromaddr
except SocketTimeout:
logger.debug("Multicast socket timed out on recv!")
continue
finally:
self._check_age(pub, min_interval=self._max_age / 20)
if self._do_heartbeat:
pub.heartbeat(min_interval=29)
msg = Message.decode(data)
name = msg.subject.split("/")[1]
if(msg.type == 'info' and
msg.subject.lower().startswith(self._subject)):
addr = msg.data["URI"]
msg.data['status'] = True
metadata = copy.copy(msg.data)
metadata["name"] = name
logger.debug('receiving address ' + str(addr)
+ " " + str(name) + " " + str(metadata))
if addr not in self._addresses:
logger.info("nameserver: publish add '%s'",
str(msg))
pub.send(msg.encode())
self._add(addr, metadata)
finally:
self._is_running = False
recv.close()
def run(self, *args):
"""Run the listener and answer to requests.
"""
del args
arec = AddressReceiver(
max_age=self._max_age,
multicast_enabled=self._multicast_enabled,
restrict_to_localhost=self._restrict_to_localhost)
arec.start()
port = PORT
try:
with nslock:
self.listener = get_context().socket(REP)
self.listener.bind("tcp://*:" + str(port))
logger.debug('Listening on port %s', str(port))
poller = Poller()
poller.register(self.listener, POLLIN)
while self.loop:
with nslock:
socks = dict(poller.poll(1000))
if socks:
if socks.get(self.listener) == POLLIN:
msg = self.listener.recv_string()
else:
continue
logger.debug("Replying to request: " + str(msg))
msg = Message.decode(msg)
active_address = get_active_address(
msg.data["service"], arec)
self.listener.send_unicode(six.text_type(active_address))
except KeyboardInterrupt:
# Needed to stop the nameserver.
pass
finally:
arec.stop()
self.stop()
def recv(self, timeout=None):
"""Receive a message, timeout in seconds.
"""
if timeout:
timeout *= 1000
while self._loop:
with self._lock:
if not self._loop:
break
subs = dict(self._poller.poll(timeout=timeout))
if subs:
for sub in self.subscribers:
if sub in subs and subs[sub] == POLLIN:
msg = Message.decode(sub.recv())
if self._translate:
url = urlsplit(self.sub_addr[sub])
host = url[1].split(":")[0]
msg.sender = msg.sender.split("@")[0] + "@" + host
yield msg
else:
yield None
def recv(self, timeout=None):
"""Receive a message, timeout in seconds.
"""
if timeout:
timeout *= 1000
while(self._loop):
with self._lock:
if not self._loop:
break
subs = dict(self._poller.poll(timeout=timeout))
if subs:
for sub in self.subscribers:
if sub in subs and subs[sub] == POLLIN:
msg = Message.decode(sub.recv())
if self._translate:
url = urlsplit(self.sub_addr[sub])
host = url[1].split(":")[0]
msg.sender = (msg.sender.split("@")[0]
+ "@" + host)
yield msg
else:
yield None
def setUp(self):
self.config_complete = create_config_from_yaml(TEST_YAML_CONTENT_OK)
self.input_msg_dset = Message.decode(rawstr=TEST_INPUT_MSG_DSET)
self.input_msg_uri = Message.decode(rawstr=TEST_INPUT_MSG_URI)
def _run(self):
"""Run the receiver.
"""
port = broadcast_port
nameservers = []
if self._multicast_enabled:
recv = MulticastReceiver(port).settimeout(2.)
while True:
try:
recv = MulticastReceiver(port).settimeout(2.)
LOGGER.info("Receiver initialized.")
break
except IOError as err:
if err.errno == errno.ENODEV:
LOGGER.error(
"Receiver initialization failed "
"(no such device). "
"Trying again in %d s", 10)
time.sleep(10)
else:
raise
else:
recv = _SimpleReceiver(port)
nameservers = ["localhost"]
self._is_running = True
with Publish("address_receiver",
self._port, ["addresses"],
nameservers=nameservers) as pub:
try:
while self._do_run:
try:
data, fromaddr = recv()
LOGGER.debug("data %s", data)
del fromaddr
except SocketTimeout:
if self._multicast_enabled:
LOGGER.debug("Multicast socket timed out on recv!")
continue
finally:
self._check_age(pub, min_interval=self._max_age / 20)
if self._do_heartbeat:
pub.heartbeat(min_interval=29)
msg = Message.decode(data)
name = msg.subject.split("/")[1]
if (msg.type == 'info'
and msg.subject.lower().startswith(self._subject)):
addr = msg.data["URI"]
msg.data['status'] = True
metadata = copy.copy(msg.data)
metadata["name"] = name
LOGGER.debug('receiving address %s %s %s', str(addr),
str(name), str(metadata))
if addr not in self._addresses:
LOGGER.info("nameserver: publish add '%s'",
str(msg))
pub.send(msg.encode())
self._add(addr, metadata)
finally:
self._is_running = False
recv.close()
def _run(self):
"""Run the receiver.
"""
port = broadcast_port
nameservers = []
if self._multicast_enabled:
recv = MulticastReceiver(port).settimeout(2.)
while True:
try:
recv = MulticastReceiver(port).settimeout(2.)
LOGGER.info("Receiver initialized.")
break
except IOError as err:
if err.errno == errno.ENODEV:
LOGGER.error("Receiver initialization failed "
"(no such device). "
"Trying again in %d s",
10)
time.sleep(10)
else:
raise
else:
recv = _SimpleReceiver(port)
nameservers = ["localhost"]
self._is_running = True
with Publish("address_receiver", self._port, ["addresses"],
nameservers=nameservers) as pub:
try:
while self._do_run:
try:
data, fromaddr = recv()
LOGGER.debug("data %s" % data)
del fromaddr
except SocketTimeout:
if self._multicast_enabled:
LOGGER.debug("Multicast socket timed out on recv!")
continue
finally:
self._check_age(pub, min_interval=self._max_age / 20)
if self._do_heartbeat:
pub.heartbeat(min_interval=29)
msg = Message.decode(data)
name = msg.subject.split("/")[1]
if(msg.type == 'info' and
msg.subject.lower().startswith(self._subject)):
addr = msg.data["URI"]
msg.data['status'] = True
metadata = copy.copy(msg.data)
metadata["name"] = name
LOGGER.debug('receiving address %s %s %s', str(addr),
str(name), str(metadata))
if addr not in self._addresses:
LOGGER.info("nameserver: publish add '%s'",
str(msg))
pub.send(msg.encode())
self._add(addr, metadata)
finally:
self._is_running = False
recv.close()
# a tuple should be returned...
for addr in gc.get_addresses():
if data_type in addr["type"]:
return Message("/oper/ns", "info", addr)
return Message("/oper/ns", "info", "")
if __name__ == '__main__':
GC = GenericConnections("")
GC.start()
port = 5555
try:
context = zmq.Context()
listener = context.socket(zmq.REP)
listener.bind("tcp://*:"+str(port))
while True:
m = listener.recv()
print m
msg = Message.decode(m)
listener.send_unicode(str(get_address(msg.data["type"], GC)))
except KeyboardInterrupt:
# this is needed for the reception to be interruptible
pass
finally:
print "terminating nameserver..."
GC.stop()
listener.close()
