class MirrorStreamer(Thread): """Act as a relay... """ def __init__(self, holder, configfile): Thread.__init__(self) self.scanlines = holder cfg = ConfigParser() cfg.read(configfile) host = cfg.get("local_reception", "mirror") hostname = cfg.get(host, "hostname") port = cfg.get(host, "pubport") rport = cfg.get(host, "reqport") address = "tcp://" + hostname + ":" + port self._sub = Subscriber([address], "hrpt 0") self._reqaddr = "tcp://" + hostname + ":" + rport def run(self): for message in self._sub.recv(1): if message is None: continue if(message.type == "have"): logger.debug("Relaying " + str(message.data["timecode"])) self.scanlines.add_scanline(message.data["satellite"], strp_isoformat(message.data["timecode"]), message.data["elevation"], None, self._reqaddr) def stop(self): """Stop streaming. """ self._sub.stop()
def create_subscriber(self): """Create a subscriber using specified addresses and message types.""" if self.subscriber is None: if self.topics: LOGGER.info("Subscribing to %s with topics %s", str(self.address), str(self.topics)) self.subscriber = Subscriber(self.address, self.topics) LOGGER.debug("Subscriber %s", str(self.subscriber))
def create_subscriber(self): '''Create a subscriber instance using specified addresses and message types. ''' if self.subscriber is None: if len(self.address_list) > 0: if len(self.msg_type_list) > 0: self.subscriber = Subscriber(self.address_list, *self.msg_type_list)
class Listener(Thread): '''PyTroll listener class for reading messages for Trollduction ''' def __init__(self, address, topics, callback, *args, **kwargs): '''Init Listener object ''' super(Listener, self).__init__() self.topics = topics self.callback = callback self.subscriber = None self.address = address self.create_subscriber() self.running = False self.cargs = args self.ckwargs = kwargs def create_subscriber(self): '''Create a subscriber instance using specified addresses and message types. ''' if self.subscriber is None: if self.topics: LOGGER.info("Subscribing to %s with topics %s", str(self.address), str(self.topics)) self.subscriber = Subscriber(self.address, self.topics) def run(self): '''Run listener ''' self.running = True for msg in self.subscriber(timeout=1): if msg is None: if self.running: continue else: break self.callback(msg, *self.cargs, **self.ckwargs) LOGGER.debug("exiting listener %s", str(self.address)) def stop(self): '''Stop subscriber and delete the instance ''' self.running = False time.sleep(1) if self.subscriber is not None: self.subscriber.close() self.subscriber = None
def __init__(self, holder, configfile): Thread.__init__(self) self.scanlines = holder cfg = ConfigParser() cfg.read(configfile) host = cfg.get("local_reception", "mirror") hostname = cfg.get(host, "hostname") port = cfg.get(host, "pubport") rport = cfg.get(host, "reqport") address = "tcp://" + hostname + ":" + port self._sub = Subscriber([address], "hrpt 0") self._reqaddr = "tcp://" + hostname + ":" + rport
def create_subscriber(self): '''Create a subscriber instance using specified addresses and message types. ''' if self.subscriber is None: if self.topics: LOGGER.info("Subscribing to %s with topics %s", str(self.address), str(self.topics)) self.subscriber = Subscriber(self.address, self.topics)
class Listener(object): '''PyTroll listener class for reading messages and adding them to deque ''' def __init__(self, ip=None, port=None, processes=2, pipe=None): ''' ''' self.address_list = [] self.type_list = [] self.add_address(ip, port) self.deque = deque() self.pipe = pipe self.create_subscriber() def add_address(self, ip, port): ''' ''' if ip is not None and port is not None: self.address_list.append('tcp://'+ip+':%05d' % port) def add_address_list(self, address_list): ''' ''' for address in address_list: self.address_list.append(address) def create_subscriber(self): ''' ''' if len(self.address_list) > 0: self.subscriber = Subscriber(self.address_list, self.type_list) def send_to_pipe(self, msg): ''' ''' self.pipe.send(msg) def start(self): ''' ''' print "start" for msg in self.subscriber.recv(): print "new msg" if self.pipe is None: self.deque.append(msg) else: while len(self.deque) > 0: self.send_to_pipe(self.deque.popleft()) self.send_to_pipe(msg)
def test_pub_suber(self): """Test publisher and subscriber. """ pub_address = "tcp://" + str(get_own_ip()) + ":0" pub = Publisher(pub_address) addr = pub_address[:-1] + str(pub.port_number) sub = Subscriber([addr], '/counter') tested = False for counter in range(5): message = Message("/counter", "info", str(counter)) pub.send(str(message)) time.sleep(1) msg = sub.recv(2).next() if msg is not None: self.assertEquals(str(msg), str(message)) tested = True self.assertTrue(tested) pub.stop()
class Listener(object): '''PyTroll listener class for reading messages and adding them to deque ''' def __init__(self, ip=None, port=None, processes=2, pipe=None): ''' ''' self.address_list = [] self.type_list = [] self.add_address(ip, port) self.deque = deque() self.pipe = pipe self.create_subscriber() def add_address(self, ip, port): ''' ''' if ip is not None and port is not None: self.address_list.append('tcp://' + ip + ':%05d' % port) def add_address_list(self, address_list): ''' ''' for address in address_list: self.address_list.append(address) def create_subscriber(self): ''' ''' if len(self.address_list) > 0: self.subscriber = Subscriber(self.address_list, self.type_list) def send_to_pipe(self, msg): ''' ''' self.pipe.send(msg) def start(self): ''' ''' print "start" for msg in self.subscriber.recv(): print "new msg" if self.pipe is None: self.deque.append(msg) else: while len(self.deque) > 0: self.send_to_pipe(self.deque.popleft()) self.send_to_pipe(msg)
def create_subscriber(cfgfile): """Create a new subscriber for all the remote hosts in cfgfile. """ cfg = ConfigParser() cfg.read(cfgfile) addrs = [] for host in cfg.get("local_reception", "remotehosts").split(): addrs.append("tcp://" + cfg.get(host, "hostname") + ":" + cfg.get(host, "pubport")) localhost = cfg.get("local_reception", "localhost") addrs.append("tcp://" + cfg.get(localhost, "hostname") + ":" + cfg.get(localhost, "pubport")) logger.debug("Subscribing to " + str(addrs)) return Subscriber(addrs, "hrpt 0", translate=True)
def create_subscriber(self): ''' ''' if len(self.address_list) > 0: self.subscriber = Subscriber(self.address_list, self.type_list)
class Listener(Thread): '''PyTroll listener class for reading messages for Trollduction ''' def __init__(self, address, topics, callback, *args, **kwargs): '''Init Listener object ''' super(Listener, self).__init__() self.topics = topics self.callback = callback self.subscriber = None self.address = address self.running = False self.cargs = args self.ckwargs = kwargs self.restart_event = Event() def create_subscriber(self): '''Create a subscriber instance using specified addresses and message types. ''' if self.subscriber is None: if self.topics: LOGGER.info("Subscribing to %s with topics %s", str(self.address), str(self.topics)) self.subscriber = Subscriber(self.address, self.topics) LOGGER.debug("Subscriber %s", str(self.subscriber)) def run(self): '''Run listener ''' with heartbeat_monitor.Monitor(self.restart_event, **self.ckwargs) as beat_monitor: self.running = True while self.running: # Loop for restart. LOGGER.debug("Starting listener %s", str(self.address)) self.create_subscriber() for msg in self.subscriber(timeout=1): if not self.running: break if self.restart_event.is_set(): self.restart_event.clear() self.stop() self.running = True break if msg is None: continue LOGGER.debug("Receiving (SUB) %s", str(msg)) beat_monitor(msg) if msg.type == "beat": continue self.callback(msg, *self.cargs, **self.ckwargs) LOGGER.debug("Exiting listener %s", str(self.address)) def stop(self): '''Stop subscriber and delete the instance ''' self.running = False time.sleep(1) if self.subscriber is not None: self.subscriber.close() self.subscriber = None
class Listener(object): '''PyTroll listener class for reading messages for Trollduction ''' def __init__(self, address_list=None, msg_type_list=None, ip=None, port=None, pipe=None): '''Init Listener object ''' self.address_list = [] self.add_address_list(address_list) self.add_address(ip, port) self.msg_type_list = [] if msg_type_list is not None: self.msg_type_list = msg_type_list self.deque = deque() self.pipe = pipe self.subscriber = None self.create_subscriber() self.running = False def add_address(self, ip, port): '''Add address that will be listened ''' if ip is not None and port is not None: self.address_list.append('tcp://' + ip + ':%04d' % port) def add_address_list(self, address_list): '''Add a list of addresses that will be listened ''' for address in address_list: self.address_list.append(address) def create_subscriber(self): '''Create a subscriber instance using specified addresses and message types. ''' if self.subscriber is None: if len(self.address_list) > 0: if len(self.msg_type_list) > 0: self.subscriber = Subscriber(self.address_list, *self.msg_type_list) def send_to_pipe(self, msg): '''Send message to parent via a Pipe() ''' self.pipe.send(msg) def run(self): '''Run listener ''' # TODO: add logging print "Starting Listener" self.running = True for msg in self.subscriber.recv(): print "New message received" if msg.subject == '/stop_listener': break if self.pipe is None: self.deque.append(msg) else: while len(self.deque) > 0: self.send_to_pipe(self.deque.popleft()) self.send_to_pipe(msg) def stop(self): '''Stop subscriber and delete the instance ''' # TODO: add logging self.subscriber.stop() self.subscriber.close() self.subscriber = None self.running = False def restart(self): '''Restart subscriber ''' self.stop() self.create_subscriber() self.run()
class Listener(Thread): """PyTroll listener class for reading messages for Trollduction.""" def __init__(self, address, topics, callback, *args, **kwargs): """Init Listener object.""" super(Listener, self).__init__() self.topics = topics self.callback = callback self.subscriber = None self.address = address self.running = False self.cargs = args self.ckwargs = kwargs self.restart_event = Event() def create_subscriber(self): """Create a subscriber using specified addresses and message types.""" if self.subscriber is None: if self.topics: LOGGER.info("Subscribing to %s with topics %s", str(self.address), str(self.topics)) self.subscriber = Subscriber(self.address, self.topics) LOGGER.debug("Subscriber %s", str(self.subscriber)) def run(self): """Run listener.""" with heartbeat_monitor.Monitor(self.restart_event, **self.ckwargs) as beat_monitor: self.running = True while self.running: # Loop for restart. LOGGER.debug("Starting listener %s", str(self.address)) self.create_subscriber() for msg in self.subscriber(timeout=1): if not self.running: break if self.restart_event.is_set(): self.restart_event.clear() self.stop() self.running = True break if msg is None: continue LOGGER.debug("Receiving (SUB) %s", str(msg)) beat_monitor(msg) if msg.type == "beat": continue # Handle public "push" messages as a hot spare client if msg.type == "push": # TODO: these need to be checked and acted if # the transfers are not finished on primary # client and are not cleared LOGGER.debug("Primary client published 'push'") add_to_ongoing(msg) # Handle public "ack" messages as a hot spare client if msg.type == "ack": LOGGER.debug("Primary client finished transfer") _ = add_to_file_cache(msg) _ = clean_ongoing_transfer(get_msg_uid(msg)) # If this is a hot spare client, wait for a while # for a public "push" message which will update # the ongoing transfers before starting processing here delay = self.ckwargs.get("processing_delay", False) if delay: add_timer(float(delay), self.callback, msg, *self.cargs, **self.ckwargs) else: self.callback(msg, *self.cargs, **self.ckwargs) LOGGER.debug("Exiting listener %s", str(self.address)) def stop(self): """Stop subscriber and delete the instance.""" self.running = False time.sleep(1) if self.subscriber is not None: self.subscriber.close() self.subscriber = None
class Listener(object): '''PyTroll listener class for reading messages for Trollduction ''' def __init__(self, address_list=None, msg_type_list=None, ip=None, port=None, pipe=None): '''Init Listener object ''' self.address_list = [] self.add_address_list(address_list) self.add_address(ip, port) self.msg_type_list = [] if msg_type_list is not None: self.msg_type_list = msg_type_list self.deque = deque() self.pipe = pipe self.subscriber = None self.create_subscriber() self.running = False def add_address(self, ip, port): '''Add address that will be listened ''' if ip is not None and port is not None: self.address_list.append('tcp://'+ip+':%04d' % port) def add_address_list(self, address_list): '''Add a list of addresses that will be listened ''' for address in address_list: self.address_list.append(address) def create_subscriber(self): '''Create a subscriber instance using specified addresses and message types. ''' if self.subscriber is None: if len(self.address_list) > 0: if len(self.msg_type_list) > 0: self.subscriber = Subscriber(self.address_list, *self.msg_type_list) def send_to_pipe(self, msg): '''Send message to parent via a Pipe() ''' self.pipe.send(msg) def run(self): '''Run listener ''' # TODO: add logging print "Starting Listener" self.running = True for msg in self.subscriber.recv(): print "New message received" if msg.subject == '/stop_listener': break if self.pipe is None: self.deque.append(msg) else: while len(self.deque) > 0: self.send_to_pipe(self.deque.popleft()) self.send_to_pipe(msg) def stop(self): '''Stop subscriber and delete the instance ''' # TODO: add logging self.subscriber.stop() self.subscriber.close() self.subscriber = None self.running = False def restart(self): '''Restart subscriber ''' self.stop() self.create_subscriber() self.run()
class Listener(Thread): """PyTroll listener class for reading messages for Trollduction.""" def __init__(self, address, topics, *args, die_event=None, **kwargs): """Init Listener object.""" super(Listener, self).__init__() self.topics = topics self.subscriber = None self.address = address self.running = False self.die_event = die_event self.cargs = args self.ckwargs = kwargs self.restart_event = Event() self.cause_of_death = None self.death_count = 0 def restart(self): """Restart the listener, returns a new running instance.""" self.stop() new_listener = self.__class__(self.address, self.topics, *self.cargs, die_event=self.die_event, **self.ckwargs) new_listener.death_count = self.death_count + 1 new_listener.start() return new_listener def create_subscriber(self): """Create a subscriber using specified addresses and message types.""" if self.subscriber is None: if self.topics: LOGGER.info("Subscribing to %s with topics %s", str(self.address), str(self.topics)) self.subscriber = Subscriber(self.address, self.topics) LOGGER.debug("Subscriber %s", str(self.subscriber)) def run(self): """Run listener.""" try: with heartbeat_monitor.Monitor(self.restart_event, **self.ckwargs) as beat_monitor: self.running = True while self.running: LOGGER.debug("Starting listener %s", str(self.address)) self.create_subscriber() self._get_messages(beat_monitor) except Exception as err: LOGGER.exception("Listener died.") self.cause_of_death = err with suppress(AttributeError): self.die_event.set() def _get_messages(self, beat_monitor): for msg in self.subscriber(timeout=1): if not self.running: break if not self._check_heartbeat(): break if msg is None: continue LOGGER.debug("Receiving (SUB) %s", str(msg)) beat_monitor(msg) if self._is_message_already_handled(msg): continue self._process_message(msg) LOGGER.debug("Exiting listener %s", str(self.address)) def _check_heartbeat(self): if self.restart_event.is_set(): LOGGER.warning( "Missing a heartbeat, restarting the subscriber to %s.", str(self.subscriber.addresses)) self.restart_event.clear() self.stop() self.running = True return False return True def _is_message_already_handled(self, msg): return (self._handle_beat_message(msg) or _handle_push_message(msg) or _handle_ack_message(msg) or _handle_message_from_another_client(msg)) def _handle_beat_message(self, msg): if msg.type == "beat": self.death_count = 0 return True return False def _process_message(self, msg): delay = self.ckwargs.get("processing_delay", False) if delay: # If this is a hot spare client, wait for a while # for a public "push" message which will update # the ongoing transfers before starting processing here add_request_push_timer(float(delay), msg, *self.cargs, **self.ckwargs) else: request_push(msg, *self.cargs, **self.ckwargs) def stop(self): """Stop subscriber and delete the instance.""" self.running = False time.sleep(1) if self.subscriber is not None: self.subscriber.close() self.subscriber = None