def get_pub_address(name, timeout=10, nameserver="localhost"): """Get the address of the publisher for a given publisher *name* from the nameserver on *nameserver* (localhost by default). """ # Socket to talk to server socket = get_context().socket(REQ) try: socket.setsockopt(LINGER, timeout * 1000) socket.connect("tcp://" + nameserver + ":" + str(PORT)) logger.debug('Connecting to %s', "tcp://" + nameserver + ":" + str(PORT)) poller = Poller() poller.register(socket, POLLIN) message = Message("/oper/ns", "request", {"service": name}) socket.send_string(six.text_type(message)) # Get the reply. sock = poller.poll(timeout=timeout * 1000) if sock: if sock[0][0] == socket: message = Message.decode(socket.recv_string(NOBLOCK)) return message.data else: raise TimeoutError("Didn't get an address after %d seconds." % timeout) finally: socket.close()
def _check_age(self, pub, min_interval=timedelta(seconds=0)): """Check the age of the receiver. """ now = datetime.utcnow() if (now - self._last_age_check) <= min_interval: return LOGGER.debug("%s - checking addresses", str(datetime.utcnow())) self._last_age_check = now to_del = [] with self._address_lock: for addr, metadata in self._addresses.items(): atime = metadata["receive_time"] if now - atime > self._max_age: mda = { 'status': False, 'URI': addr, 'service': metadata['service'] } msg = Message('/address/' + metadata['name'], 'info', mda) to_del.append(addr) LOGGER.info("publish remove '%s'", str(msg)) pub.send(msg.encode()) for addr in to_del: del self._addresses[addr]
def get_address(data_type, timeout=2): warnings.warn( "nameclient.get_address shouldn't be used, " + "use posttroll.subscriber.get_address instead...", DeprecationWarning) context = zmq.Context() # Socket to talk to server socket = context.socket(zmq.REQ) try: socket.connect("tcp://localhost:5555") msg = Message("/oper/ns", "request", {"type": data_type}) socket.send(str(msg)) # Get the reply. poller = zmq.Poller() poller.register(socket, zmq.POLLIN) s = poller.poll(timeout=timeout * 1000) if s: if s[0][0] == socket: m = Message.decode(socket.recv(zmq.NOBLOCK)) return m.data else: raise RuntimeError("Unknown socket ?!?!?") else: raise TimeoutException("Didn't get an address after %d seconds." % timeout) print "Received reply to ", data_type, ": [", m, "]" finally: socket.close()
def test_pub_unicode(self): message = Message("/pџтяöll", "info", 'hej') with Publish("a_service", 9000) as pub: try: pub.send(message.encode()) except UnicodeDecodeError: self.fail("Sending raised UnicodeDecodeError unexpectedly!")
def ack(self, message): """Reply with ack to a publication """ for url in gen_dict_extract(message.data, 'uri'): uri = urlparse(url) pathname = uri.path if 'origin' in self._attrs and not fnmatch.fnmatch( os.path.basename(pathname), os.path.basename(globify(self._attrs["origin"]))): LOGGER.warning('Client trying to get invalid file: %s', pathname) return Message(message.subject, "err", data="{0:s} not reacheable".format(pathname)) if (self._attrs.get('compression') or self._attrs.get( 'delete', 'False').lower() in ["1", "yes", "true", "on"]): self._deleter.add(pathname) new_msg = Message(message.subject, "ack", data=message.data.copy()) try: new_msg.data['destination'] = clean_url( new_msg.data['destination']) except KeyError: pass return new_msg
def request_push(msg, destinations, login): # TODO make this a bit more elaborate (like create a new message) #hostname = "localhost" with cache_lock: if msg.data["uid"] in file_cache: urlobj = urlparse(msg.data['uri']) if socket.gethostbyname(urlobj.netloc) in get_local_ips(): PUB.send(msg) return file_cache.append(msg.data["uid"]) hostname, port = msg.data["request_address"].split(":") requester = PushRequester(hostname, int(port)) req = Message(msg.subject, "push", data=msg.data.copy()) duris = [] for destination in destinations: duris.append(urlunparse(("ftp", login + "@" + socket.gethostname(), destination, "", "", ""))) req.data["destinations"] = duris LOGGER.info("Requesting: " + str(req)) response = requester.send_and_recv(req) if response.type == "ack": LOGGER.debug("Server ack") else: LOGGER.error("Received an erraneous response from server %s", str(response))
def run(self): while self._loop: try: socks = dict(self._poller.poll(timeout=2000)) except ZMQError: logger.info("Poller interrupted.") continue if self._socket in socks and socks[self._socket] == POLLIN: logger.debug("Received a request, waiting for the lock") with self._lock: message = Message(rawstr=self._socket.recv(NOBLOCK)) logger.debug("processing request: " + str(message)) reply = Message(subject, "error") try: if message.type == "ping": reply = self.pong() elif (message.type == "request" and message.data["type"] == "scanline"): reply = self.scanline(message) elif (message.type == "notice" and message.data["type"] == "scanline"): reply = self.notice(message) else: # unknown request reply = self.unknown(message) except: logger.exception("Something went wrong" " when processing the request:") finally: self.send(reply) logger.debug("Lock released from manager") else: # timeout pass
def setUp(self): from threading import Lock import six.moves.queue as queue from posttroll.message import Message self.loader = SceneLoader() self.prodlist = write_yaml(PRODUCT_LIST) self.prodlist_2 = write_yaml(PRODUCT_LIST_TWO_AREAS) self.prodlist_2_together = write_yaml(PRODUCT_LIST_TWO_AREAS_TOGETHER) self.lock = Lock() self.prev_lock = Lock() self.output_queue = queue.Queue() self.context = { 'lock': self.lock, 'prev_lock': self.prev_lock, 'output_queue': self.output_queue } self.topic = '/topic' self.file_msg = Message(self.topic, 'file', METADATA_FILE) self.dataset_msg = Message(self.topic, 'dataset', METADATA_DATASET) self.collection_msg = Message(self.topic, 'collection', METADATA_COLLECTION) self.collection_dataset_msg = Message(self.topic, 'collection', METADATA_COLLECTION_DATASET) self.dset1 = Mock(name='dset1') self.dset2 = Mock(name='dset2') self.scene = Mock(attrs=METADATA_FILE, datasets=[self.dset1, self.dset2])
def test_encode(self): """Test the encoding of a message. """ subject = "/test/whatup/doc" atype = "info" data = "not much to say" msg1 = Message(subject, atype, data=data) sender = "%s@%s" % (msg1.user, msg1.host) self.assertEquals( _MAGICK + subject + " " + atype + " " + sender + " " + str(msg1.time.isoformat()) + " " + msg1.version + " " + "text/ascii" + " " + data, msg1.encode(), )
def push(self, message): """Reply to push request """ uri = urlparse(message.data["uri"]) pathname = uri.path if not fnmatch.fnmatch( os.path.basename(pathname), os.path.basename(globify(self._attrs["origin"]))): LOGGER.warning('Client trying to get invalid file: %s', pathname) return Message(message.subject, "err", data="{0:s} not reachable".format(pathname)) try: move_it(message, self._attrs) except Exception as err: return Message(message.subject, "err", data=str(err)) else: if (self._attrs.get('compression') or self._attrs.get( 'delete', 'False').lower() in ["1", "yes", "true", "on"]): self._deleter.add(pathname) new_msg = Message(message.subject, "file", data=message.data.copy()) new_msg.data['destination'] = clean_url( new_msg.data['destination']) return new_msg
def get_active_address(name, arec): """Get the addresses of the active modules for a given publisher *name*. """ addrs = arec.get(name) if addrs: return Message("/oper/ns", "info", addrs) else: return Message("/oper/ns", "info", "")
def _get_cleaned_ack_message(message): new_msg = Message(message.subject, "ack", data=message.data.copy()) try: new_msg.data['destination'] = clean_url(new_msg.data['destination']) except KeyError: pass return new_msg
def test_check_message_okay_message_ok(self): """Test for check if message is okay.""" input_msg = Message.decode(rawstr=TEST_INPUT_MSG) result = check_message_okay(input_msg) self.assertEqual(result, None) input_msg = Message.decode(rawstr=TEST_INPUT_MSG_NO_DATASET) result = check_message_okay(input_msg) self.assertEqual(result, None)
def test_metadata(self): """Test metadata encoding/decoding. """ metadata = copy.copy(SOME_METADATA) msg = Message.decode( Message('/sat/polar/smb/level1', 'file', data=metadata).encode()) self.assertTrue(msg.data == metadata, msg='Messaging, metadata decoding / encoding failed')
def test_encode_decode(self): """Test the encoding/decoding of the message class. """ msg1 = Message("/test/whatup/doc", "info", data="not much to say") sender = "%s@%s" % (msg1.user, msg1.host) self.assertTrue(sender == msg1.sender, msg="Messaging, decoding user, host from sender failed") msg2 = Message.decode(msg1.encode()) self.assertTrue(str(msg2) == str(msg1), msg="Messaging, encoding, decoding failed")
def test_pub_unicode(self): from posttroll.message import Message from posttroll.publisher import Publish message = Message("/pџтяöll", "info", 'hej') with Publish("a_service", 9000) as pub: try: pub.send(message.encode()) except UnicodeDecodeError: self.fail("Sending raised UnicodeDecodeError unexpectedly!")
def _sanitize_message_destination(message): sanitized_message = Message(rawstr=str(message)) try: _ = urlparse(message.data['destination']) except (KeyError, TypeError): pass else: sanitized_message.data['destination'] = clean_url( message.data['destination']) return sanitized_message
def get_address(data_type, gc): """Get the address of the module for a given *data_type*. """ if data_type == "": return Message("/oper/ns", "info", gc.get_addresses()) # a tuple should be returned... for addr in gc.get_addresses(): if data_type in addr["type"]: return Message("/oper/ns", "info", addr) return Message("/oper/ns", "info", "")
def push(self, message): """Reply to push request.""" new_msg = self._move_files(message) if new_msg is None: new_msg = Message(message.subject, _get_push_message_type(message), data=message.data.copy()) new_msg.data['destination'] = clean_url( new_msg.data['destination']) return new_msg
def test_encode_decode(self): """Test the encoding/decoding of the message class. """ msg1 = Message('/test/whatup/doc', 'info', data='not much to say') sender = '%s@%s' % (msg1.user, msg1.host) self.assertTrue(sender == msg1.sender, msg='Messaging, decoding user, host from sender failed') msg2 = Message.decode(msg1.encode()) self.assertTrue(str(msg2) == str(msg1), msg='Messaging, encoding, decoding failed')
def scanline(self, message): """Reply to scanline request """ sat = message.data["satellite"] key = strp_isoformat(message.data["utctime"]) try: data = self._holder.get_data(sat, key) except KeyError: resp = Message(subject, "missing") else: resp = Message(subject, "scanline", data, binary=True) return resp
def fetch_files(message, destination): """Fetch files from the network""" new_message = Message(rawstr=str(message)) if new_message.type == "dataset": for dset in new_message.data["dataset"]: dset["uri"] = fetch_file(dset["uri"], destination) elif new_message.type == "collection": raise NotImplementedError else: new_message.data["uri"] = fetch_file(new_message.data["uri"], destination) return new_message
def test_encode(self): """Test the encoding of a message. """ subject = '/test/whatup/doc' atype = "info" data = 'not much to say' msg1 = Message(subject, atype, data=data) sender = '%s@%s' % (msg1.user, msg1.host) self.assertEquals( _MAGICK + subject + " " + atype + " " + sender + " " + str(msg1.time.isoformat()) + " " + msg1.version + " " + 'text/ascii' + " " + data, msg1.encode())
def test_encode_decode(self): """Test the encoding/decoding of the message class. """ msg1 = Message('/test/whatup/doc', 'info', data='not much to say') sender = '%s@%s' % (msg1.user, msg1.host) self.assertTrue( sender == msg1.sender, msg='Messaging, decoding user, host from sender failed') msg2 = Message.decode(msg1.encode()) self.assertTrue(str(msg2) == str(msg1), msg='Messaging, encoding, decoding failed')
def check_and_publish(datatype, rpc_metadata, publish, heartbeat): """Check for new files of type *datatype*, with the given *rpc_metadata* and publish them through *publish*. """ stamp_config = DatexLastStamp(datatype) def younger_than_stamp_files(): """Uses glob polling to get new files. """ fdir, fglob = datex_config.get_path(datatype) del fglob fstamp = stamp_config.get_last_stamp() for fname, ftime in _get_file_list(datatype, time_start=fstamp + TIME_EPSILON): if datex_config.distribute(datatype): yield os.path.join(fdir, fname) stamp_config.update_last_stamp(ftime) # Give the publisher a little time to initialize # (e.g reconnections from subscribers) time.sleep(1) logger.info('publisher starting for datatype %s' % datatype) if heartbeat: last_heartbeat = datetime.now() - timedelta(seconds=heartbeat + 1) try: while (True): if (heartbeat and (datetime.now() - last_heartbeat).seconds >= heartbeat): # Send a heartbeat msg = Message('/hearbeat/' + datatype, 'heartbeat', datetime.utcnow().isoformat()) logger.info('sending: ' + str(msg)) try: publish.send(str(msg)) last_heartbeat = datetime.now() except zmq.ZMQError: logger.exception('publish failed') for filedesc in younger_than_stamp_files(): # Publish new files data = copy.copy(rpc_metadata) data['uri'] += os.path.basename(filedesc) msg = Message('/' + datatype, 'file', data) logger.info('sending: ' + str(msg)) try: publish.send(str(msg)) except zmq.ZMQError: logger.exception('publish failed') time.sleep(TIME_WAKEUP) except (KeyboardInterrupt, SystemExit): pass finally: logger.info('publisher stopping') publish.close()
def test_prepare_posttroll_message(setup_comm, get_config, gethostname): """Test setup the posttroll message.""" get_config.return_value = CONFIG_EXAMPLE gethostname.return_value = "my.host.name" myconfigfile = "/my/config/file/path" myboarders_file = "/my/shape/file/with/country/boarders" mymask_file = "/my/shape/file/with/polygons/to/filter/out" afpp = ActiveFiresPostprocessing(myconfigfile, myboarders_file, mymask_file) test_filepath = "/my/geojson/file/path" input_msg = Message.decode(rawstr=TEST_MSG) res_msg = afpp._generate_output_message(test_filepath, input_msg) assert res_msg.data['platform_name'] == 'NOAA-20' assert res_msg.data['type'] == 'GEOJSON-filtered' assert res_msg.data['format'] == 'geojson' assert res_msg.data['product'] == 'afimg' assert res_msg.subject == '/VIIRS/L2/Fires/PP/National' assert res_msg.data['uri'] == 'ssh://my.host.name//my/geojson/file/path' input_msg = Message.decode(rawstr=TEST_MSG) fake_region_mask = { 'attributes': { 'Kod_omr': '9999', 'Testomr': 'Some area description' } } res_msg = afpp._generate_output_message(test_filepath, input_msg, region=fake_region_mask) assert res_msg.subject == '/VIIRS/L2/Fires/PP/Regional/9999' msg_str = 'No fire detections for this granule' result_messages = afpp._generate_no_fires_messages(input_msg, msg_str) for (nat_or_reg, res_msg) in zip(['National', 'Regional'], result_messages): assert res_msg.data['info'] == msg_str assert res_msg.subject == '/VIIRS/L2/Fires/PP/' + nat_or_reg assert 'type' not in res_msg.data assert 'format' not in res_msg.data assert 'product' not in res_msg.data assert 'uri' not in res_msg.data
def publish_message(self, mymessage): """Publish the message.""" posttroll_msg = Message(mymessage['header'], mymessage['type'], mymessage['content']) msg_to_publish = posttroll_msg.encode() manager = Manager() publisher_q = manager.Queue() pub_thread = PPSPublisher(publisher_q, self.nameservers) pub_thread.start() LOG.info("Sending: " + str(msg_to_publish)) publisher_q.put(msg_to_publish) pub_thread.stop()
def test_item_translator(self): """Test translating dict items.""" orig = ( 'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json ' '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.tar", "uri": "/home/user/bla.tar"}' ) dest_dir = '/tmp' expected = ( 'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json ' '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.png", "uri": "/tmp/bla.png"}' ) def dummy_cb(var, k): dirname, filename = os.path.split(var[k]) basename, ext = os.path.splitext(filename) if dirname: dest = os.path.join(dest_dir, basename + '.png') else: dest = basename + '.png' var[k] = dest return var expected_dict = Message(rawstr=expected).data res = translate_dict_item(Message(rawstr=orig).data, 'uri', dummy_cb) res = translate_dict_item(res, 'uid', dummy_cb) self.assertDictEqual(expected_dict, res) expected = ( 'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json ' '{"sensor": "viirs", "format": "SDR", "variant": "DR", "dataset": [{"uid": "bla1.png", "uri": "/tmp/bla1.png"}, {"uid": "bla2.png", "uri": "/tmp/bla2.png"}]}' ) # noqa def dummy_cb(var): if not var['uid'].endswith('.tar'): return var dirname, filename = os.path.split(var.pop('uri')) basename, ext = os.path.splitext(filename) new_names = [basename + str(i) + '.png' for i in range(1, 3)] var.pop('uid') var['dataset'] = [ dict(uid=nn, uri=os.path.join(dest_dir, nn)) for nn in new_names ] return var expected_dict = Message(rawstr=expected).data res = translate_dict( Message(rawstr=orig).data, ('uri', 'uid'), dummy_cb) self.assertDictEqual(expected_dict, res)
def publish_callback(msg, *args, **kwargs): # save to file_cache with cache_lock: if msg.data['uid'] in file_registery: file_registery[msg.data['uid']].append(msg) return file_registery[msg.data['uid']] = [msg] # transform message new_msg = Message(msg.subject, msg.type, msg.data.copy()) new_msg.data['request_address'] = request_address # send onwards LOGGER.debug('Sending %s', str(new_msg)) send(str(new_msg))
def test_metadata(self): """Test metadata encoding/decoding. """ metadata = copy.copy(SOME_METADATA) msg = Message.decode(Message("/sat/polar/smb/level1", "file", data=metadata).encode()) self.assertTrue(msg.data == metadata, msg="Messaging, metadata decoding / encoding failed")
def _run(self): port = broadcast_port recv = MulticastReceiver(port).settimeout(2.0) self._is_running = True with Publish("address_receiver", ["addresses"], self._port) as pub: try: while self._do_run: try: data, fromaddr = recv() del fromaddr except SocketTimeout: continue msg = Message.decode(data) name = msg.subject.split("/")[1] if(msg.type == 'info' and msg.subject.lower().endswith(self._subject)): addr = msg.data["URI"] metadata = copy.copy(msg.data) metadata["name"] = name if debug: print 'receiving address', addr, name, metadata if addr not in self._addresses: pub.send(str(msg)) if debug: print 'publishing address', addr, name, metadata self._add(addr, metadata) finally: self._is_running = False recv.close()
def receive_from_zmq(host, port, station, environment, excluded_platforms, days=1): """Receive 2met! messages from zeromq. """ # socket = Subscriber(["tcp://localhost:9331"], ["2met!"]) sock = GMCSubscriber(host, port) msg_rec = MessageReceiver(host, excluded_platforms) with Publish("receiver", 0, ["HRPT 0", "PDS", "RDR", "EPS 0"]) as pub: for rawmsg in sock.recv(): # TODO: # - Watch for idle time in order to detect a hangout LOGGER.debug("receive from 2met! %s", str(rawmsg)) string = TwoMetMessage(rawmsg) to_send = msg_rec.receive(string) if to_send is None: continue subject = "/".join( ("", to_send['format'], to_send['data_processing_level'], station, environment, "polar", "direct_readout")) msg = Message(subject, "file", to_send).encode() LOGGER.debug("publishing %s", str(msg)) pub.send(msg) if days: msg_rec.clean_passes(days)
def fun(orig_pathname): """Publish what we have.""" if not fnmatch.fnmatch(orig_pathname, pattern): return else: LOGGER.debug('We have a match: %s', orig_pathname) pathname = unpack(orig_pathname, **attrs) info = attrs.get("info", {}) if info: info = dict((elt.strip().split('=') for elt in info.split(";"))) for infokey, infoval in info.items(): if "," in infoval: info[infokey] = infoval.split(",") info.update(parse(attrs["origin"], orig_pathname)) info['uri'] = pathname info['uid'] = os.path.basename(pathname) info['request_address'] = attrs.get( "request_address", get_own_ip()) + ":" + attrs["request_port"] msg = Message(attrs["topic"], 'file', info) publisher.send(str(msg)) with file_cache_lock: file_cache.appendleft(attrs["topic"] + '/' + info["uid"]) LOGGER.debug("Message sent: " + str(msg))
def process(self, pathname): '''Process the event''' # New file created and closed logger.debug("processing %s", pathname) # parse information and create self.info dict{} metadata = self.config.copy() success = False for parser in self.parsers: try: metadata.update(parser.parse(pathname)) success = True break except ValueError: pass if not success: logger.warning("Could not find a matching pattern for %s", pathname) metadata['uri'] = pathname metadata['uid'] = os.path.basename(pathname) if self.tbus_orbit and "orbit_number" in metadata: logger.info("Changing orbit number by -1!") metadata["orbit_number"] -= 1 # replace values with corresponding aliases, if any are given if self.aliases: for key in metadata: if key in self.aliases: metadata[key] = self.aliases[key][str(metadata[key])] message = Message(self.topic, 'file', metadata) logger.info("Publishing message %s" % str(message)) self.pub.send(str(message))
def get(self, timeout=None): """Return a published message or None if a timeout has happend. """ if timeout: timeout *= 1000. self.subscriber.connect(self.destination) poller = zmq.Poller() poller.register(self.subscriber, zmq.POLLIN) try: while(True): try: ret = poller.poll(timeout=timeout) if ret: if ret[0][0] == self.subscriber: msg = Message.decode( self.subscriber.recv(zmq.NOBLOCK)) yield msg else: logger.error("WHAT THE HECK") else: # timeout yield None except zmq.ZMQError: logger.exception('recv failed') finally: poller.unregister(self.subscriber) self.subscriber.close()
def test_translate_dict_item_to_dataset(self): """Test translating dict items to dataset.""" orig = ( 'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json ' '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.tar", "uri": "/home/user/bla.tar"}' ) expected = ( 'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json ' '{"sensor": "viirs", "format": "SDR", "variant": "DR", "dataset": [{"uid": "bla1.png", "uri": "/tmp/bla1.png"}, {"uid": "bla2.png", "uri": "/tmp/bla2.png"}]}' ) # noqa expected_dict = Message(rawstr=expected).data res = translate_dict( Message(rawstr=orig).data, ('uri', 'uid'), dummy_callback_translate_dict_item_to_dataset) self.assertDictEqual(expected_dict, res)
def copy_hook(pathname, dest, val=val, pub=pub): fname = os.path.basename(pathname) destination = urlunparse((dest.scheme, dest.hostname, os.path.join(dest.path, fname), dest.params, dest.query, dest.fragment)) info = val.get("info", "") if info: info = dict( (elt.strip().split('=') for elt in info.split(";"))) for infokey, infoval in info.items(): if "," in infoval: info[infokey] = infoval.split(",") else: info = {} try: info.update( parse(os.path.basename(val["origin"]), os.path.basename(pathname))) except ValueError: info.update( parse( os.path.basename( os.path.splitext(val["origin"])[0]), os.path.basename(pathname))) info['uri'] = destination info['uid'] = fname msg = Message(val["topic"], 'file', info) pub.send(str(msg)) LOGGER.debug("Message sent: %s", str(msg))
def get_pub_address(name, timeout=10, nameserver="localhost"): """Get the address of the publisher for a given publisher *name* from the nameserver on *nameserver* (localhost by default). """ # Socket to talk to server socket = context.socket(REQ) try: socket.setsockopt(LINGER, timeout * 1000) socket.connect("tcp://" + nameserver + ":" + str(PORT)) poller = Poller() poller.register(socket, POLLIN) message = Message("/oper/ns", "request", {"service": name}) socket.send(str(message)) # Get the reply. sock = poller.poll(timeout=timeout * 1000) if sock: if sock[0][0] == socket: message = Message.decode(socket.recv(NOBLOCK)) return message.data else: raise TimeoutError("Didn't get an address after %d seconds." % timeout) finally: socket.close()
def get_address(data_type, timeout=2): warnings.warn( "nameclient.get_address shouldn't be used, " + "use posttroll.subscriber.get_address instead...", DeprecationWarning, ) context = zmq.Context() # Socket to talk to server socket = context.socket(zmq.REQ) try: socket.connect("tcp://localhost:5555") msg = Message("/oper/ns", "request", {"type": data_type}) socket.send(str(msg)) # Get the reply. poller = zmq.Poller() poller.register(socket, zmq.POLLIN) s = poller.poll(timeout=timeout * 1000) if s: if s[0][0] == socket: m = Message.decode(socket.recv(zmq.NOBLOCK)) return m.data else: raise RuntimeError("Unknown socket ?!?!?") else: raise TimeoutException("Didn't get an address after %d seconds." % timeout) print "Received reply to ", data_type, ": [", m, "]" finally: socket.close()
def run(self, *args): """Run the listener and answer to requests. """ del args arec = AddressReceiver(max_age=self._max_age) arec.start() port = PORT try: self.listener = context.socket(REP) self.listener.bind("tcp://*:" + str(port)) poller = Poller() poller.register(self.listener, POLLIN) while self.loop: socks = dict(poller.poll(1000)) if socks: if socks.get(self.listener) == POLLIN: msg = self.listener.recv() else: continue logger.debug("Replying to request: " + str(msg)) msg = Message.decode(msg) self.listener.send_unicode(str(get_active_address( msg.data["service"], arec))) except KeyboardInterrupt: # Needed to stop the nameserver. pass finally: arec.stop() self.listener.close()
def test_translate_dict_values_file_message(self): """Test translating message dictionary values.""" orig = ( 'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json ' '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.png", "uri": "/home/user/bla.png"}' ) expected = ( 'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json ' '{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.png", "uri": "/tmp/bla.png"}' ) expected_dict = Message(rawstr=expected).data res = translate_dict_value( Message(rawstr=orig).data, 'uri', dummy_callback_translate_dict_values) self.assertDictEqual(expected_dict, res)
def run(self): last_hb = datetime.now() minutes = 2 while self._loop: if datetime.now() - last_hb > timedelta(minutes=minutes): logger.error("No heartbeat from " + str(self._pubaddress)) last_hb = datetime.now() minutes = 1440 socks = dict(self._poller.poll(2000)) if (socks and self._subsocket in socks and socks[self._subsocket] == POLLIN): message = Message.decode(self._subsocket.recv()) else: continue if message.type == "have": sat = message.data["satellite"] key = strp_isoformat(message.data["timecode"]) elevation = message.data["elevation"] quality = message.data.get("quality", 100) data = _MirrorGetter(self._req, sat, key) self._holder.add(sat, key, elevation, quality, data) if message.type == "heartbeat": logger.debug("Got heartbeat from " + str(self._pubaddress) + ": " + str(message)) self._sched.mirror_next_pass = message.data["next_pass"] last_hb = datetime.now() minutes = 2
def test_encode(self): """Test the encoding of a message. """ subject = '/test/whatup/doc' atype = "info" data = 'not much to say' msg1 = Message(subject, atype, data=data) sender = '%s@%s' % (msg1.user, msg1.host) self.assertEqual(_MAGICK + subject + " " + atype + " " + sender + " " + str(msg1.time.isoformat()) + " " + msg1.version + " " + 'text/ascii' + " " + data, msg1.encode())
def test_decode(self): """Test the decoding of a message. """ rawstr = (_MAGICK + r'/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01' + r' text/ascii "what' + r"'" + r's up doc"') msg = Message.decode(rawstr) self.assertTrue(str(msg) == rawstr, msg='Messaging, decoding of message failed')
def test_decode(self): """Test the decoding of a message. """ rawstr = ( _MAGICK + "/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01" + ' application/json "what\'s up doc"' ) msg = Message.decode(rawstr) self.assertTrue(str(msg) == rawstr, msg="Messaging, decoding of message failed")
def _check_age(self, pub, min_interval=timedelta(seconds=0)): """Check the age of the receiver. """ now = datetime.utcnow() if (now - self._last_age_check) <= min_interval: return LOGGER.debug("%s - checking addresses", str(datetime.utcnow()) ) self._last_age_check = now with self._address_lock: for addr, metadata in self._addresses.items(): atime = metadata["receive_time"] if now - atime > self._max_age: mda = {'status': False, 'URI': addr, 'service': metadata['service']} msg = Message('/address/' + metadata['name'], 'info', mda) del self._addresses[addr] LOGGER.info("publish remove '%s'", str(msg)) pub.send(msg.encode())
def recv(self, timeout=None): """Receive, optionally with *timeout* in seconds. """ if timeout: timeout *= 1000. for sub in list(self.subscribers) + self._hooks: self.poller.register(sub, POLLIN) self._loop = True try: while self._loop: sleep(0) try: socks = dict(self.poller.poll(timeout=timeout)) if socks: for sub in self.subscribers: if sub in socks and socks[sub] == POLLIN: m__ = Message.decode(sub.recv_string(NOBLOCK)) if not self._filter or self._filter(m__): if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] m__.sender = (m__.sender.split("@")[0] + "@" + host) yield m__ for sub in self._hooks: if sub in socks and socks[sub] == POLLIN: m__ = Message.decode(sub.recv_string(NOBLOCK)) self._hooks_cb[sub](m__) else: # timeout yield None except ZMQError as err: LOGGER.exception("Receive failed: %s", str(err)) finally: for sub in list(self.subscribers) + self._hooks: self.poller.unregister(sub)
def _run(self): """Run the receiver. """ port = broadcast_port recv = MulticastReceiver(port).settimeout(2.) self._is_running = True with Publish("address_receiver", self._port, ["addresses"]) as pub: try: while self._do_run: try: data, fromaddr = recv() del fromaddr except SocketTimeout: logger.debug("Multicast socket timed out on recv!") continue finally: self._check_age(pub, min_interval=self._max_age / 20) if self._do_heartbeat: pub.heartbeat(min_interval=29) msg = Message.decode(data) name = msg.subject.split("/")[1] if(msg.type == 'info' and msg.subject.lower().startswith(self._subject)): addr = msg.data["URI"] msg.data['status'] = True metadata = copy.copy(msg.data) metadata["name"] = name logger.debug('receiving address ' + str(addr) + " " + str(name) + " " + str(metadata)) if addr not in self._addresses: logger.info("nameserver: publish add '%s'", str(msg)) pub.send(msg.encode()) self._add(addr, metadata) finally: self._is_running = False recv.close()
def recv(self, timeout=None): if timeout: timeout *= 1000. for sub in self.subscribers: self.poller.register(sub, zmq.POLLIN) self._loop = True try: while(self._loop): try: s = dict(self.poller.poll(timeout=timeout)) if s: for sub in self.subscribers: if sub in s and s[sub] == zmq.POLLIN: m__ = Message.decode(sub.recv(zmq.NOBLOCK)) if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] m__.sender = (m__.sender.split("@")[0] + "@" + host) # Only accept pre-defined data types try: if (self._data_types and (m__.data['type'] not in self._data_types)): continue except (KeyError, TypeError): pass yield m__ else: # timeout yield None except zmq.ZMQError: print >>sys.stderr, 'receive failed' finally: for sub in self.subscribers: self.poller.unregister(sub)
def recv(self, timeout=None): """Receive a message, timeout in seconds. """ if timeout: timeout *= 1000 while self._loop: with self._lock: if not self._loop: break subs = dict(self._poller.poll(timeout=timeout)) if subs: for sub in self.subscribers: if sub in subs and subs[sub] == POLLIN: msg = Message.decode(sub.recv()) if self._translate: url = urlsplit(self.sub_addr[sub]) host = url[1].split(":")[0] msg.sender = msg.sender.split("@")[0] + "@" + host yield msg else: yield None
# a tuple should be returned... for addr in gc.get_addresses(): if data_type in addr["type"]: return Message("/oper/ns", "info", addr) return Message("/oper/ns", "info", "") if __name__ == '__main__': GC = GenericConnections("") GC.start() port = 5555 try: context = zmq.Context() listener = context.socket(zmq.REP) listener.bind("tcp://*:"+str(port)) while True: m = listener.recv() print m msg = Message.decode(m) listener.send_unicode(str(get_address(msg.data["type"], GC))) except KeyboardInterrupt: # this is needed for the reception to be interruptible pass finally: print "terminating nameserver..." GC.stop() listener.close()
def _run(self): """Run the receiver. """ port = broadcast_port nameservers = [] if self._multicast_enabled: recv = MulticastReceiver(port).settimeout(2.) while True: try: recv = MulticastReceiver(port).settimeout(2.) LOGGER.info("Receiver initialized.") break except IOError as err: if err.errno == errno.ENODEV: LOGGER.error("Receiver initialization failed " "(no such device). " "Trying again in %d s", 10) time.sleep(10) else: raise else: recv = _SimpleReceiver(port) nameservers = ["localhost"] self._is_running = True with Publish("address_receiver", self._port, ["addresses"], nameservers=nameservers) as pub: try: while self._do_run: try: data, fromaddr = recv() LOGGER.debug("data %s" % data) del fromaddr except SocketTimeout: if self._multicast_enabled: LOGGER.debug("Multicast socket timed out on recv!") continue finally: self._check_age(pub, min_interval=self._max_age / 20) if self._do_heartbeat: pub.heartbeat(min_interval=29) msg = Message.decode(data) name = msg.subject.split("/")[1] if(msg.type == 'info' and msg.subject.lower().startswith(self._subject)): addr = msg.data["URI"] msg.data['status'] = True metadata = copy.copy(msg.data) metadata["name"] = name LOGGER.debug('receiving address %s %s %s', str(addr), str(name), str(metadata)) if addr not in self._addresses: LOGGER.info("nameserver: publish add '%s'", str(msg)) pub.send(msg.encode()) self._add(addr, metadata) finally: self._is_running = False recv.close()