def test_check_message_okay_message_ok(self): """Test for check if message is okay.""" input_msg = Message.decode(rawstr=TEST_INPUT_MSG) result = check_message_okay(input_msg) self.assertEqual(result, None) input_msg = Message.decode(rawstr=TEST_INPUT_MSG_NO_DATASET) result = check_message_okay(input_msg) self.assertEqual(result, None)
def test_prepare_posttroll_message(setup_comm, get_config, gethostname): """Test setup the posttroll message.""" get_config.return_value = CONFIG_EXAMPLE gethostname.return_value = "my.host.name" myconfigfile = "/my/config/file/path" myboarders_file = "/my/shape/file/with/country/boarders" mymask_file = "/my/shape/file/with/polygons/to/filter/out" afpp = ActiveFiresPostprocessing(myconfigfile, myboarders_file, mymask_file) test_filepath = "/my/geojson/file/path" input_msg = Message.decode(rawstr=TEST_MSG) res_msg = afpp._generate_output_message(test_filepath, input_msg) assert res_msg.data['platform_name'] == 'NOAA-20' assert res_msg.data['type'] == 'GEOJSON-filtered' assert res_msg.data['format'] == 'geojson' assert res_msg.data['product'] == 'afimg' assert res_msg.subject == '/VIIRS/L2/Fires/PP/National' assert res_msg.data['uri'] == 'ssh://my.host.name//my/geojson/file/path' input_msg = Message.decode(rawstr=TEST_MSG) fake_region_mask = { 'attributes': { 'Kod_omr': '9999', 'Testomr': 'Some area description' } } res_msg = afpp._generate_output_message(test_filepath, input_msg, region=fake_region_mask) assert res_msg.subject == '/VIIRS/L2/Fires/PP/Regional/9999' msg_str = 'No fire detections for this granule' result_messages = afpp._generate_no_fires_messages(input_msg, msg_str) for (nat_or_reg, res_msg) in zip(['National', 'Regional'], result_messages): assert res_msg.data['info'] == msg_str assert res_msg.subject == '/VIIRS/L2/Fires/PP/' + nat_or_reg assert 'type' not in res_msg.data assert 'format' not in res_msg.data assert 'product' not in res_msg.data assert 'uri' not in res_msg.data
def get_pub_address(name, timeout=10, nameserver="localhost"): """Get the address of the publisher for a given publisher *name* from the nameserver on *nameserver* (localhost by default). """ # Socket to talk to server socket = get_context().socket(REQ) try: socket.setsockopt(LINGER, timeout * 1000) socket.connect("tcp://" + nameserver + ":" + str(PORT)) logger.debug('Connecting to %s', "tcp://" + nameserver + ":" + str(PORT)) poller = Poller() poller.register(socket, POLLIN) message = Message("/oper/ns", "request", {"service": name}) socket.send_string(six.text_type(message)) # Get the reply. sock = poller.poll(timeout=timeout * 1000) if sock: if sock[0][0] == socket: message = Message.decode(socket.recv_string(NOBLOCK)) return message.data else: raise TimeoutError("Didn't get an address after %d seconds." % timeout) finally: socket.close()
def run(self): last_hb = datetime.now() minutes = 2 while self._loop: if datetime.now() - last_hb > timedelta(minutes=minutes): logger.error("No heartbeat from " + str(self._pubaddress)) last_hb = datetime.now() minutes = 1440 socks = dict(self._poller.poll(2000)) if (socks and self._subsocket in socks and socks[self._subsocket] == POLLIN): message = Message.decode(self._subsocket.recv()) else: continue if message.type == "have": sat = message.data["satellite"] key = strp_isoformat(message.data["timecode"]) elevation = message.data["elevation"] quality = message.data.get("quality", 100) data = _MirrorGetter(self._req, sat, key) self._holder.add(sat, key, elevation, quality, data) if message.type == "heartbeat": logger.debug("Got heartbeat from " + str(self._pubaddress) + ": " + str(message)) self._sched.mirror_next_pass = message.data["next_pass"] last_hb = datetime.now() minutes = 2
def get(self, timeout=None): """Return a published message or None if a timeout has happend. """ if timeout: timeout *= 1000. self.subscriber.connect(self.destination) poller = zmq.Poller() poller.register(self.subscriber, zmq.POLLIN) try: while (True): try: ret = poller.poll(timeout=timeout) if ret: if ret[0][0] == self.subscriber: msg = Message.decode( self.subscriber.recv(zmq.NOBLOCK)) yield msg else: logger.error("WHAT THE HECK") else: # timeout yield None except zmq.ZMQError: logger.exception('recv failed') finally: poller.unregister(self.subscriber) self.subscriber.close()
def get_address(data_type, timeout=2): warnings.warn( "nameclient.get_address shouldn't be used, " + "use posttroll.subscriber.get_address instead...", DeprecationWarning) context = zmq.Context() # Socket to talk to server socket = context.socket(zmq.REQ) try: socket.connect("tcp://localhost:5555") msg = Message("/oper/ns", "request", {"type": data_type}) socket.send(str(msg)) # Get the reply. poller = zmq.Poller() poller.register(socket, zmq.POLLIN) s = poller.poll(timeout=timeout * 1000) if s: if s[0][0] == socket: m = Message.decode(socket.recv(zmq.NOBLOCK)) return m.data else: raise RuntimeError("Unknown socket ?!?!?") else: raise TimeoutException("Didn't get an address after %d seconds." % timeout) print "Received reply to ", data_type, ": [", m, "]" finally: socket.close()
def get_pub_address(name, timeout=10, nameserver="localhost"): """Get the address of the publisher for a given publisher *name* from the nameserver on *nameserver* (localhost by default). """ # Socket to talk to server socket = context.socket(REQ) try: socket.setsockopt(LINGER, timeout * 1000) socket.connect("tcp://" + nameserver + ":" + str(PORT)) poller = Poller() poller.register(socket, POLLIN) message = Message("/oper/ns", "request", {"service": name}) socket.send(str(message)) # Get the reply. sock = poller.poll(timeout=timeout * 1000) if sock: if sock[0][0] == socket: message = Message.decode(socket.recv(NOBLOCK)) return message.data else: raise TimeoutError("Didn't get an address after %d seconds." % timeout) finally: socket.close()
def run(self): print("starting SDR subscriber loop") while True: msg_bytes = self.socket.recv() print("GOT SDR: {}".format(msg_bytes)) npnow = pd.to_datetime("now") message = Message.decode(msg_bytes) filename = os.path.basename(message.data["uri"]) file_time = datetime.strptime(filename[-69:-51], "_d%Y%m%d_t%H%M%S") npthen = pd.to_datetime(file_time) delay = npnow - npthen if self._sdrs.index.size > 0: gap = npnow - self._sdrs.index[-1] else: gap = pd.Timedelta("0 seconds") with self.lock: self._sdrs.at[npnow] = ( message.data["segment"], message.data["platform_name"], pd.to_datetime(message.data["start_time"]), pd.to_datetime(message.data["end_time"]), message.data["orbit_number"], pd.to_datetime(message.data["proctime"]), message.data["uid"], delay, gap, )
def _run(self): port = broadcast_port recv = MulticastReceiver(port).settimeout(2.0) self._is_running = True with Publish("address_receiver", ["addresses"], self._port) as pub: try: while self._do_run: try: data, fromaddr = recv() del fromaddr except SocketTimeout: continue msg = Message.decode(data) name = msg.subject.split("/")[1] if(msg.type == 'info' and msg.subject.lower().endswith(self._subject)): addr = msg.data["URI"] metadata = copy.copy(msg.data) metadata["name"] = name if debug: print 'receiving address', addr, name, metadata if addr not in self._addresses: pub.send(str(msg)) if debug: print 'publishing address', addr, name, metadata self._add(addr, metadata) finally: self._is_running = False recv.close()
def test_metadata(self): """Test metadata encoding/decoding. """ metadata = copy.copy(SOME_METADATA) msg = Message.decode(Message("/sat/polar/smb/level1", "file", data=metadata).encode()) self.assertTrue(msg.data == metadata, msg="Messaging, metadata decoding / encoding failed")
def test_create_publish_message(self, gethostname): """Test the creation of the publish message.""" gethostname.return_value = "my_local_server" my_fake_level1c_file = '/my/level1c/file/path/level1c.nc' input_msg = Message.decode(rawstr=TEST_INPUT_MSG) result = prepare_l1c_message(my_fake_level1c_file, input_msg.data, orbit=99999) expected = { 'data_type': 'MSG4', 'orig_platform_name': 'MSG4', 'start_time': datetime(2021, 5, 18, 14, 15), 'variant': '0DEG', 'series': 'MSG4', 'platform_name': 'Meteosat-11', 'channel': '', 'nominal_time': datetime(2021, 5, 18, 14, 15), 'compressed': '', 'origin': '172.18.0.248:9093', 'sensor': ['seviri'], 'uri': 'ssh://my_local_server/my/level1c/file/path/level1c.nc', 'uid': 'level1c.nc', 'format': 'PPS-L1C', 'type': 'NETCDF', 'data_processing_level': '1c' } self.assertDictEqual(result, expected)
def run(self): last_hb = datetime.now() minutes = 2 while self._loop: if datetime.now() - last_hb > timedelta(minutes=minutes): logger.error("No heartbeat from " + str(self._pubaddress)) last_hb = datetime.now() minutes = 1440 socks = dict(self._poller.poll(2000)) if (socks and self._subsocket in socks and socks[self._subsocket] == POLLIN): message = Message.decode(self._subsocket.recv()) else: continue if message.type == "have": sat = message.data["satellite"] key = strp_isoformat(message.data["timecode"]) elevation = message.data["elevation"] quality = message.data.get("quality", 100) data = _MirrorGetter(self._req, sat, key) self._holder.add(sat, key, elevation, quality, data) if message.type == "heartbeat": logger.debug("Got heartbeat from " + str(self._pubaddress) + ": " + str(message)) self._sched.mirror_next_pass = message.data["next_pass"] last_hb = datetime.now() minutes = 2
def get_address(data_type, timeout=2): warnings.warn( "nameclient.get_address shouldn't be used, " + "use posttroll.subscriber.get_address instead...", DeprecationWarning, ) context = zmq.Context() # Socket to talk to server socket = context.socket(zmq.REQ) try: socket.connect("tcp://localhost:5555") msg = Message("/oper/ns", "request", {"type": data_type}) socket.send(str(msg)) # Get the reply. poller = zmq.Poller() poller.register(socket, zmq.POLLIN) s = poller.poll(timeout=timeout * 1000) if s: if s[0][0] == socket: m = Message.decode(socket.recv(zmq.NOBLOCK)) return m.data else: raise RuntimeError("Unknown socket ?!?!?") else: raise TimeoutException("Didn't get an address after %d seconds." % timeout) print "Received reply to ", data_type, ": [", m, "]" finally: socket.close()
def get(self, timeout=None): """Return a published message or None if a timeout has happend. """ if timeout: timeout *= 1000. self.subscriber.connect(self.destination) poller = zmq.Poller() poller.register(self.subscriber, zmq.POLLIN) try: while(True): try: ret = poller.poll(timeout=timeout) if ret: if ret[0][0] == self.subscriber: msg = Message.decode( self.subscriber.recv(zmq.NOBLOCK)) yield msg else: logger.error("WHAT THE HECK") else: # timeout yield None except zmq.ZMQError: logger.exception('recv failed') finally: poller.unregister(self.subscriber) self.subscriber.close()
def run(self, *args): """Run the listener and answer to requests. """ del args arec = AddressReceiver(max_age=self._max_age) arec.start() port = PORT try: self.listener = context.socket(REP) self.listener.bind("tcp://*:" + str(port)) poller = Poller() poller.register(self.listener, POLLIN) while self.loop: socks = dict(poller.poll(1000)) if socks: if socks.get(self.listener) == POLLIN: msg = self.listener.recv() else: continue logger.debug("Replying to request: " + str(msg)) msg = Message.decode(msg) self.listener.send_unicode(str(get_active_address( msg.data["service"], arec))) except KeyboardInterrupt: # Needed to stop the nameserver. pass finally: arec.stop() self.listener.close()
def get_messages(file_list): messages = {} for message_file in file_list: print("reading {}".format(message_file)) with open(message_file, "r") as file: message = Message.decode(file.read()) messages[get_key(message)] = message return messages
def test_set_collection_area_id_not_in_message_and_not_in_config(self): """Test setting the collection area id.""" msg = Message.decode(self.message1) config = self.config_mandatory runconfig = DummyAappRunnerConfig(config, 'xl-band') set_collection_area_id(msg.data, runconfig) assert config['collection_area_id'] is None
def test_set_collection_area_id_present_in_config_but_not_in_message(self): """Test setting the collection area id.""" msg = Message.decode(self.message1) config = self.config_complete runconfig = DummyAappRunnerConfig(config, 'xl-band') set_collection_area_id(msg.data, runconfig) assert config['collection_area_id'] == 'euron1'
def test_get_options_regional_filtering(self, setup_comm, read_config, gethostname, netrc): secrets = MyNetrcMock() netrc.return_value = secrets gethostname.return_value = 'default' myconfigfile = "/my/config/file/path" regstream = io.StringIO(REG_CONFIG) read_config.return_value = yaml.load(regstream, Loader=yaml.UnsafeLoader) this = EndUserNotifierRegional(myconfigfile) expected = {'publish_topic': 'VIIRS/L2/MSB/Regional', 'subscribe_topics': ['VIIRS/L2/Fires/PP/Regional'], 'smtp_server': 'smtp.mydomain.xx', 'domain': 'mydomain.xx', 'sender': '*****@*****.**', 'recipients': { 'Area1-name': {'subject': 'My subject', 'Kod_omr': '0999', 'name': 'Name of my area 1', 'recipients': ['*****@*****.**'], 'recipients_attachment': ['*****@*****.**']}, 'Area2-name': {'subject': 'My subject', 'Kod_omr': '0114', 'name': 'Name of my area 2', 'recipients': ['*****@*****.**'], 'recipients_attachment': ['*****@*****.**']}}, 'max_number_of_fires_in_sms': 3, 'fire_data': ['power', 'observation_time'], 'unsubscribe': {'address': '*****@*****.**', 'text': 'Stop being bothered: Send a note to [email protected]'}, 'unsubscribe_address': '*****@*****.**', 'unsubscribe_text': 'Stop being bothered: Send a note to [email protected]'} self.assertDictEqual(expected, this.options) assert this.smtp_server == 'smtp.mydomain.xx' assert this.domain == 'mydomain.xx' assert this.sender == '*****@*****.**' # assert len(this.recipients) == 2 # recipients = this.recipients['Area1-name'] # expected = {'subject': 'My subject', 'knkod': '0999', 'name': 'Name of my area 1', # 'recipients': ['*****@*****.**'], # 'recipients_attachment': ['*****@*****.**']} # self.assertDictEqual(expected, recipients) # recipients = this.recipients['Area2-name'] # expected = {'subject': 'My subject', 'knkod': '0114', 'name': 'Name of my area 2', # 'recipients': ['*****@*****.**'], # 'recipients_attachment': ['*****@*****.**']} # self.assertDictEqual(expected, recipients) # assert this.subject is None # assert this.max_number_of_fires_in_sms == 3 # self.assertListEqual(this.fire_data, ['power', 'observation_time']) # assert this.unsubscribe_address == '*****@*****.**' # assert this.input_topic == 'VIIRS/L2/Fires/PP/Regional' # assert this.output_topic == 'VIIRS/L2/MSB/Regional' input_msg = Message.decode(rawstr=REGIONAL_TEST_MESSAGE)
def test_metadata(self): """Test metadata encoding/decoding. """ metadata = copy.copy(SOME_METADATA) msg = Message.decode( Message('/sat/polar/smb/level1', 'file', data=metadata).encode()) self.assertTrue(msg.data == metadata, msg='Messaging, metadata decoding / encoding failed')
def test_check_message_okay_message_has_no_start_time(self): """Test that message is not okay if it does not contain start_time.""" input_msg = Message.decode(rawstr=TEST_INPUT_MSG_NO_START_TIME) with pytest.raises(MessageContentMissing) as exec_info: _ = check_message_okay(input_msg) exception_raised = exec_info.value self.assertEqual("Message is lacking crucial fields: start_time", str(exception_raised))
def test_encode_decode(self): """Test the encoding/decoding of the message class. """ msg1 = Message("/test/whatup/doc", "info", data="not much to say") sender = "%s@%s" % (msg1.user, msg1.host) self.assertTrue(sender == msg1.sender, msg="Messaging, decoding user, host from sender failed") msg2 = Message.decode(msg1.encode()) self.assertTrue(str(msg2) == str(msg1), msg="Messaging, encoding, decoding failed")
def test_check_message_okay_message_has_no_dataset(self): """Test that message is not okay if it is not a dataset.""" input_msg = Message.decode(rawstr=TEST_INPUT_MSG_NO_DATASET_NO_FILE) with pytest.raises(MessageTypeNotSupported) as exec_info: _ = check_message_okay(input_msg) exception_raised = exec_info.value self.assertEqual("Not a dataset or a file, don't do anything...", str(exception_raised))
def test_decode(self): """Test the decoding of a message. """ rawstr = (_MAGICK + r'/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01' + r' text/ascii "what' + r"'" + r's up doc"') msg = Message.decode(rawstr) self.assertTrue(str(msg) == rawstr, msg='Messaging, decoding of message failed')
def test_decode(self): """Test the decoding of a message. """ rawstr = ( _MAGICK + r'/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01' + r' text/ascii "what' + r"'" + r's up doc"') msg = Message.decode(rawstr) self.assertTrue(str(msg) == rawstr, msg='Messaging, decoding of message failed')
def test_decode(self): """Test the decoding of a message. """ rawstr = ( _MAGICK + "/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01" + ' application/json "what\'s up doc"' ) msg = Message.decode(rawstr) self.assertTrue(str(msg) == rawstr, msg="Messaging, decoding of message failed")
def test_decode(self): """Test the decoding of a message. """ rawstr = ( _MAGICK + '/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01' + ' application/json "what\'s up doc"') msg = Message.decode(rawstr) self.assertTrue(str(msg) == rawstr, msg='Messaging, decoding of message failed')
def test_encode_decode(self): """Test the encoding/decoding of the message class. """ msg1 = Message('/test/whatup/doc', 'info', data='not much to say') sender = '%s@%s' % (msg1.user, msg1.host) self.assertTrue(sender == msg1.sender, msg='Messaging, decoding user, host from sender failed') msg2 = Message.decode(msg1.encode()) self.assertTrue(str(msg2) == str(msg1), msg='Messaging, encoding, decoding failed')
def test_encode_decode(self): """Test the encoding/decoding of the message class. """ msg1 = Message('/test/whatup/doc', 'info', data='not much to say') sender = '%s@%s' % (msg1.user, msg1.host) self.assertTrue( sender == msg1.sender, msg='Messaging, decoding user, host from sender failed') msg2 = Message.decode(msg1.encode()) self.assertTrue(str(msg2) == str(msg1), msg='Messaging, encoding, decoding failed')
def recv(self, timeout=None): """Receive, optionally with *timeout* in seconds. """ if timeout: timeout *= 1000. for sub in list(self.subscribers) + self._hooks: self.poller.register(sub, POLLIN) self._loop = True try: while self._loop: sleep(0) try: socks = dict(self.poller.poll(timeout=timeout)) if socks: for sub in self.subscribers: if sub in socks and socks[sub] == POLLIN: m__ = Message.decode(sub.recv_string(NOBLOCK)) if not self._filter or self._filter(m__): if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] m__.sender = ( m__.sender.split("@")[0] + "@" + host) yield m__ for sub in self._hooks: if sub in socks and socks[sub] == POLLIN: m__ = Message.decode(sub.recv_string(NOBLOCK)) self._hooks_cb[sub](m__) else: # timeout yield None except ZMQError as err: LOGGER.exception("Receive failed: %s", str(err)) finally: for sub in list(self.subscribers) + self._hooks: self.poller.unregister(sub)
def recv(self, timeout=None): """Receive, optionally with *timeout* in seconds. """ if timeout: timeout *= 1000. for sub in list(self.subscribers) + self._hooks: self.poller.register(sub, POLLIN) self._loop = True try: while self._loop: sleep(0) try: socks = dict(self.poller.poll(timeout=timeout)) if socks: for sub in self.subscribers: if sub in socks and socks[sub] == POLLIN: m__ = Message.decode(sub.recv_string(NOBLOCK)) if not self._filter or self._filter(m__): if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] m__.sender = (m__.sender.split("@")[0] + "@" + host) yield m__ for sub in self._hooks: if sub in socks and socks[sub] == POLLIN: m__ = Message.decode(sub.recv_string(NOBLOCK)) self._hooks_cb[sub](m__) else: # timeout yield None except ZMQError as err: LOGGER.exception("Receive failed: %s", str(err)) finally: for sub in list(self.subscribers) + self._hooks: self.poller.unregister(sub)
def test_orbit_number_from_msg_viirs(self, cpu_count, config): """Test use orbit number from message.""" cpu_count.return_value = 1 config.return_value = self.config_viirs_orbit_number_from_msg_ok input_msg = Message.decode(rawstr=TEST_INPUT_MESSAGE_VIIRS_MSG) with patch('nwcsafpps_runner.l1c_processing.ThreadPool'): with tempfile.NamedTemporaryFile() as myconfig_file: l1c_proc = L1cProcessor(myconfig_file.name, 'viirs-l1c') l1c_proc.run(input_msg) expected_orbit_number = TEST_VIIRS_MSG_DATA.get('orbit_number') self.assertEqual(l1c_proc.orbit_number, expected_orbit_number)
def process_message(msg_bytes): try: message = Message.decode(msg_bytes) publish_products(message) except MessageError: logger.exception("Message decode error.") except NotImplementedError: logger.exception("Crap. I accepted a message I can't process.") except ValueError: logger.exception("I got a message, but couldn't find the data") except KeyError: logger.exception("missing data, skipping") logger.debug("Whew, that was hard. Let rest for 10 seconds.") time.sleep(10)
def test_orbit_number_missing_in_msg_viirs(self, cpu_count, config): """Test use orbit number but missing in message.""" cpu_count.return_value = 1 config.return_value = self.config_viirs_orbit_number_from_msg_ok input_msg = Message.decode( rawstr=TEST_INPUT_MESSAGE_VIIRS_NO_ORBIT_MSG) with self.assertLogs('nwcsafpps_runner.l1c_processing', level='INFO') as cm: with patch('nwcsafpps_runner.l1c_processing.ThreadPool'): with tempfile.NamedTemporaryFile() as myconfig_file: l1c_proc = L1cProcessor(myconfig_file.name, 'viirs-l1c') l1c_proc.run(input_msg) expected_orbit_number = 99999 self.assertEqual(l1c_proc.orbit_number, expected_orbit_number) self.assertEqual( cm.output[2], 'WARNING:nwcsafpps_runner.l1c_processing:You asked for orbit_number ' 'from the message, but its not there. Keep init orbit.')
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) context = zmq.Context() socket = context.socket(zmq.SUB) socket.setsockopt(zmq.TCP_KEEPALIVE, 1) socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 60) socket.setsockopt(zmq.TCP_KEEPALIVE_CNT, 20) socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 60) socket.setsockopt_string(zmq.SUBSCRIBE, "pytroll://AVO/viirs/sdr") socket.connect(SDR_PUBLISHER) while True: msg_bytes = socket.recv() message = Message.decode(msg_bytes) filename = os.path.basename(message.data["uri"]) file_time = datetime.strptime(filename[-69:-51], "_d%Y%m%d_t%H%M%S") age = datetime.now() - file_time print("{} (retrieval took {})".format(filename, humanize.naturaldelta(age)))
def recv(self, timeout=None): if timeout: timeout *= 1000. for sub in self.subscribers: self.poller.register(sub, zmq.POLLIN) self._loop = True try: while(self._loop): try: s = dict(self.poller.poll(timeout=timeout)) if s: for sub in self.subscribers: if sub in s and s[sub] == zmq.POLLIN: m__ = Message.decode(sub.recv(zmq.NOBLOCK)) if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] m__.sender = (m__.sender.split("@")[0] + "@" + host) # Only accept pre-defined data types try: if (self._data_types and (m__.data['type'] not in self._data_types)): continue except (KeyError, TypeError): pass yield m__ else: # timeout yield None except zmq.ZMQError: print >>sys.stderr, 'receive failed' finally: for sub in self.subscribers: self.poller.unregister(sub)
def _run(self): """Run the receiver. """ port = broadcast_port recv = MulticastReceiver(port).settimeout(2.) self._is_running = True with Publish("address_receiver", self._port, ["addresses"]) as pub: try: while self._do_run: try: data, fromaddr = recv() del fromaddr except SocketTimeout: logger.debug("Multicast socket timed out on recv!") continue finally: self._check_age(pub, min_interval=self._max_age / 20) if self._do_heartbeat: pub.heartbeat(min_interval=29) msg = Message.decode(data) name = msg.subject.split("/")[1] if(msg.type == 'info' and msg.subject.lower().startswith(self._subject)): addr = msg.data["URI"] msg.data['status'] = True metadata = copy.copy(msg.data) metadata["name"] = name logger.debug('receiving address ' + str(addr) + " " + str(name) + " " + str(metadata)) if addr not in self._addresses: logger.info("nameserver: publish add '%s'", str(msg)) pub.send(msg.encode()) self._add(addr, metadata) finally: self._is_running = False recv.close()
def run(self, *args): """Run the listener and answer to requests. """ del args arec = AddressReceiver( max_age=self._max_age, multicast_enabled=self._multicast_enabled, restrict_to_localhost=self._restrict_to_localhost) arec.start() port = PORT try: with nslock: self.listener = get_context().socket(REP) self.listener.bind("tcp://*:" + str(port)) logger.debug('Listening on port %s', str(port)) poller = Poller() poller.register(self.listener, POLLIN) while self.loop: with nslock: socks = dict(poller.poll(1000)) if socks: if socks.get(self.listener) == POLLIN: msg = self.listener.recv_string() else: continue logger.debug("Replying to request: " + str(msg)) msg = Message.decode(msg) active_address = get_active_address( msg.data["service"], arec) self.listener.send_unicode(six.text_type(active_address)) except KeyboardInterrupt: # Needed to stop the nameserver. pass finally: arec.stop() self.stop()
def recv(self, timeout=None): """Receive a message, timeout in seconds. """ if timeout: timeout *= 1000 while self._loop: with self._lock: if not self._loop: break subs = dict(self._poller.poll(timeout=timeout)) if subs: for sub in self.subscribers: if sub in subs and subs[sub] == POLLIN: msg = Message.decode(sub.recv()) if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] msg.sender = msg.sender.split("@")[0] + "@" + host yield msg else: yield None
def recv(self, timeout=None): """Receive a message, timeout in seconds. """ if timeout: timeout *= 1000 while(self._loop): with self._lock: if not self._loop: break subs = dict(self._poller.poll(timeout=timeout)) if subs: for sub in self.subscribers: if sub in subs and subs[sub] == POLLIN: msg = Message.decode(sub.recv()) if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] msg.sender = (msg.sender.split("@")[0] + "@" + host) yield msg else: yield None
def setUp(self): self.config_complete = create_config_from_yaml(TEST_YAML_CONTENT_OK) self.input_msg_dset = Message.decode(rawstr=TEST_INPUT_MSG_DSET) self.input_msg_uri = Message.decode(rawstr=TEST_INPUT_MSG_URI)
def _run(self): """Run the receiver. """ port = broadcast_port nameservers = [] if self._multicast_enabled: recv = MulticastReceiver(port).settimeout(2.) while True: try: recv = MulticastReceiver(port).settimeout(2.) LOGGER.info("Receiver initialized.") break except IOError as err: if err.errno == errno.ENODEV: LOGGER.error( "Receiver initialization failed " "(no such device). " "Trying again in %d s", 10) time.sleep(10) else: raise else: recv = _SimpleReceiver(port) nameservers = ["localhost"] self._is_running = True with Publish("address_receiver", self._port, ["addresses"], nameservers=nameservers) as pub: try: while self._do_run: try: data, fromaddr = recv() LOGGER.debug("data %s", data) del fromaddr except SocketTimeout: if self._multicast_enabled: LOGGER.debug("Multicast socket timed out on recv!") continue finally: self._check_age(pub, min_interval=self._max_age / 20) if self._do_heartbeat: pub.heartbeat(min_interval=29) msg = Message.decode(data) name = msg.subject.split("/")[1] if (msg.type == 'info' and msg.subject.lower().startswith(self._subject)): addr = msg.data["URI"] msg.data['status'] = True metadata = copy.copy(msg.data) metadata["name"] = name LOGGER.debug('receiving address %s %s %s', str(addr), str(name), str(metadata)) if addr not in self._addresses: LOGGER.info("nameserver: publish add '%s'", str(msg)) pub.send(msg.encode()) self._add(addr, metadata) finally: self._is_running = False recv.close()
def _run(self): """Run the receiver. """ port = broadcast_port nameservers = [] if self._multicast_enabled: recv = MulticastReceiver(port).settimeout(2.) while True: try: recv = MulticastReceiver(port).settimeout(2.) LOGGER.info("Receiver initialized.") break except IOError as err: if err.errno == errno.ENODEV: LOGGER.error("Receiver initialization failed " "(no such device). " "Trying again in %d s", 10) time.sleep(10) else: raise else: recv = _SimpleReceiver(port) nameservers = ["localhost"] self._is_running = True with Publish("address_receiver", self._port, ["addresses"], nameservers=nameservers) as pub: try: while self._do_run: try: data, fromaddr = recv() LOGGER.debug("data %s" % data) del fromaddr except SocketTimeout: if self._multicast_enabled: LOGGER.debug("Multicast socket timed out on recv!") continue finally: self._check_age(pub, min_interval=self._max_age / 20) if self._do_heartbeat: pub.heartbeat(min_interval=29) msg = Message.decode(data) name = msg.subject.split("/")[1] if(msg.type == 'info' and msg.subject.lower().startswith(self._subject)): addr = msg.data["URI"] msg.data['status'] = True metadata = copy.copy(msg.data) metadata["name"] = name LOGGER.debug('receiving address %s %s %s', str(addr), str(name), str(metadata)) if addr not in self._addresses: LOGGER.info("nameserver: publish add '%s'", str(msg)) pub.send(msg.encode()) self._add(addr, metadata) finally: self._is_running = False recv.close()
# a tuple should be returned... for addr in gc.get_addresses(): if data_type in addr["type"]: return Message("/oper/ns", "info", addr) return Message("/oper/ns", "info", "") if __name__ == '__main__': GC = GenericConnections("") GC.start() port = 5555 try: context = zmq.Context() listener = context.socket(zmq.REP) listener.bind("tcp://*:"+str(port)) while True: m = listener.recv() print m msg = Message.decode(m) listener.send_unicode(str(get_address(msg.data["type"], GC))) except KeyboardInterrupt: # this is needed for the reception to be interruptible pass finally: print "terminating nameserver..." GC.stop() listener.close()