class MirrorStreamer(Thread):
"""Act as a relay...
"""
def __init__(self, holder, configfile):
Thread.__init__(self)
self.scanlines = holder
cfg = ConfigParser()
cfg.read(configfile)
host = cfg.get("local_reception", "mirror")
hostname = cfg.get(host, "hostname")
port = cfg.get(host, "pubport")
rport = cfg.get(host, "reqport")
address = "tcp://" + hostname + ":" + port
self._sub = Subscriber([address], "hrpt 0")
self._reqaddr = "tcp://" + hostname + ":" + rport
def run(self):
for message in self._sub.recv(1):
if message is None:
continue
if(message.type == "have"):
logger.debug("Relaying " + str(message.data["timecode"]))
self.scanlines.add_scanline(message.data["satellite"],
strp_isoformat(message.data["timecode"]),
message.data["elevation"],
None,
self._reqaddr)
def stop(self):
"""Stop streaming.
"""
self._sub.stop()
def create_subscriber(self):
"""Create a subscriber using specified addresses and message types."""
if self.subscriber is None:
if self.topics:
LOGGER.info("Subscribing to %s with topics %s",
str(self.address), str(self.topics))
self.subscriber = Subscriber(self.address, self.topics)
LOGGER.debug("Subscriber %s", str(self.subscriber))
def create_subscriber(self):
'''Create a subscriber instance using specified addresses and
message types.
'''
if self.subscriber is None:
if len(self.address_list) > 0:
if len(self.msg_type_list) > 0:
self.subscriber = Subscriber(self.address_list,
*self.msg_type_list)
class Listener(Thread):
'''PyTroll listener class for reading messages for Trollduction
'''
def __init__(self, address, topics, callback, *args, **kwargs):
'''Init Listener object
'''
super(Listener, self).__init__()
self.topics = topics
self.callback = callback
self.subscriber = None
self.address = address
self.create_subscriber()
self.running = False
self.cargs = args
self.ckwargs = kwargs
def create_subscriber(self):
'''Create a subscriber instance using specified addresses and
message types.
'''
if self.subscriber is None:
if self.topics:
LOGGER.info("Subscribing to %s with topics %s", str(self.address), str(self.topics))
self.subscriber = Subscriber(self.address, self.topics)
def run(self):
'''Run listener
'''
self.running = True
for msg in self.subscriber(timeout=1):
if msg is None:
if self.running:
continue
else:
break
self.callback(msg, *self.cargs, **self.ckwargs)
LOGGER.debug("exiting listener %s", str(self.address))
def stop(self):
'''Stop subscriber and delete the instance
'''
self.running = False
time.sleep(1)
if self.subscriber is not None:
self.subscriber.close()
self.subscriber = None
def __init__(self, holder, configfile):
Thread.__init__(self)
self.scanlines = holder
cfg = ConfigParser()
cfg.read(configfile)
host = cfg.get("local_reception", "mirror")
hostname = cfg.get(host, "hostname")
port = cfg.get(host, "pubport")
rport = cfg.get(host, "reqport")
address = "tcp://" + hostname + ":" + port
self._sub = Subscriber([address], "hrpt 0")
self._reqaddr = "tcp://" + hostname + ":" + rport
def create_subscriber(self):
'''Create a subscriber instance using specified addresses and
message types.
'''
if self.subscriber is None:
if self.topics:
LOGGER.info("Subscribing to %s with topics %s", str(self.address), str(self.topics))
self.subscriber = Subscriber(self.address, self.topics)
def create_subscriber(self):
'''Create a subscriber instance using specified addresses and
message types.
'''
if self.subscriber is None:
if len(self.address_list) > 0:
if len(self.msg_type_list) > 0:
self.subscriber = Subscriber(self.address_list,
*self.msg_type_list)
class Listener(object):
'''PyTroll listener class for reading messages and adding them to
deque
'''
def __init__(self, ip=None, port=None, processes=2, pipe=None):
'''
'''
self.address_list = []
self.type_list = []
self.add_address(ip, port)
self.deque = deque()
self.pipe = pipe
self.create_subscriber()
def add_address(self, ip, port):
'''
'''
if ip is not None and port is not None:
self.address_list.append('tcp://'+ip+':%05d' % port)
def add_address_list(self, address_list):
'''
'''
for address in address_list:
self.address_list.append(address)
def create_subscriber(self):
'''
'''
if len(self.address_list) > 0:
self.subscriber = Subscriber(self.address_list,
self.type_list)
def send_to_pipe(self, msg):
'''
'''
self.pipe.send(msg)
def start(self):
'''
'''
print "start"
for msg in self.subscriber.recv():
print "new msg"
if self.pipe is None:
self.deque.append(msg)
else:
while len(self.deque) > 0:
self.send_to_pipe(self.deque.popleft())
self.send_to_pipe(msg)
def test_pub_suber(self):
"""Test publisher and subscriber.
"""
pub_address = "tcp://" + str(get_own_ip()) + ":0"
pub = Publisher(pub_address)
addr = pub_address[:-1] + str(pub.port_number)
sub = Subscriber([addr], '/counter')
tested = False
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
pub.stop()
def test_pub_suber(self):
"""Test publisher and subscriber.
"""
pub_address = "tcp://" + str(get_own_ip()) + ":0"
pub = Publisher(pub_address)
addr = pub_address[:-1] + str(pub.port_number)
sub = Subscriber([addr], '/counter')
tested = False
for counter in range(5):
message = Message("/counter", "info", str(counter))
pub.send(str(message))
time.sleep(1)
msg = sub.recv(2).next()
if msg is not None:
self.assertEquals(str(msg), str(message))
tested = True
self.assertTrue(tested)
pub.stop()
class Listener(object):
'''PyTroll listener class for reading messages and adding them to
deque
'''
def __init__(self, ip=None, port=None, processes=2, pipe=None):
'''
'''
self.address_list = []
self.type_list = []
self.add_address(ip, port)
self.deque = deque()
self.pipe = pipe
self.create_subscriber()
def add_address(self, ip, port):
'''
'''
if ip is not None and port is not None:
self.address_list.append('tcp://' + ip + ':%05d' % port)
def add_address_list(self, address_list):
'''
'''
for address in address_list:
self.address_list.append(address)
def create_subscriber(self):
'''
'''
if len(self.address_list) > 0:
self.subscriber = Subscriber(self.address_list, self.type_list)
def send_to_pipe(self, msg):
'''
'''
self.pipe.send(msg)
def start(self):
'''
'''
print "start"
for msg in self.subscriber.recv():
print "new msg"
if self.pipe is None:
self.deque.append(msg)
else:
while len(self.deque) > 0:
self.send_to_pipe(self.deque.popleft())
self.send_to_pipe(msg)
def __init__(self, holder, configfile):
Thread.__init__(self)
self.scanlines = holder
cfg = ConfigParser()
cfg.read(configfile)
host = cfg.get("local_reception", "mirror")
hostname = cfg.get(host, "hostname")
port = cfg.get(host, "pubport")
rport = cfg.get(host, "reqport")
address = "tcp://" + hostname + ":" + port
self._sub = Subscriber([address], "hrpt 0")
self._reqaddr = "tcp://" + hostname + ":" + rport
def create_subscriber(cfgfile):
"""Create a new subscriber for all the remote hosts in cfgfile.
"""
cfg = ConfigParser()
cfg.read(cfgfile)
addrs = []
for host in cfg.get("local_reception", "remotehosts").split():
addrs.append("tcp://" + cfg.get(host, "hostname") + ":" +
cfg.get(host, "pubport"))
localhost = cfg.get("local_reception", "localhost")
addrs.append("tcp://" + cfg.get(localhost, "hostname") + ":" +
cfg.get(localhost, "pubport"))
logger.debug("Subscribing to " + str(addrs))
return Subscriber(addrs, "hrpt 0", translate=True)
def create_subscriber(self):
'''
'''
if len(self.address_list) > 0:
self.subscriber = Subscriber(self.address_list,
self.type_list)
class Listener(Thread):
'''PyTroll listener class for reading messages for Trollduction
'''
def __init__(self, address, topics, callback, *args, **kwargs):
'''Init Listener object
'''
super(Listener, self).__init__()
self.topics = topics
self.callback = callback
self.subscriber = None
self.address = address
self.running = False
self.cargs = args
self.ckwargs = kwargs
self.restart_event = Event()
def create_subscriber(self):
'''Create a subscriber instance using specified addresses and
message types.
'''
if self.subscriber is None:
if self.topics:
LOGGER.info("Subscribing to %s with topics %s",
str(self.address), str(self.topics))
self.subscriber = Subscriber(self.address, self.topics)
LOGGER.debug("Subscriber %s", str(self.subscriber))
def run(self):
'''Run listener
'''
with heartbeat_monitor.Monitor(self.restart_event,
**self.ckwargs) as beat_monitor:
self.running = True
while self.running:
# Loop for restart.
LOGGER.debug("Starting listener %s", str(self.address))
self.create_subscriber()
for msg in self.subscriber(timeout=1):
if not self.running:
break
if self.restart_event.is_set():
self.restart_event.clear()
self.stop()
self.running = True
break
if msg is None:
continue
LOGGER.debug("Receiving (SUB) %s", str(msg))
beat_monitor(msg)
if msg.type == "beat":
continue
self.callback(msg, *self.cargs, **self.ckwargs)
LOGGER.debug("Exiting listener %s", str(self.address))
def stop(self):
'''Stop subscriber and delete the instance
'''
self.running = False
time.sleep(1)
if self.subscriber is not None:
self.subscriber.close()
self.subscriber = None
class Listener(object):
'''PyTroll listener class for reading messages for Trollduction
'''
def __init__(self,
address_list=None,
msg_type_list=None,
ip=None,
port=None,
pipe=None):
'''Init Listener object
'''
self.address_list = []
self.add_address_list(address_list)
self.add_address(ip, port)
self.msg_type_list = []
if msg_type_list is not None:
self.msg_type_list = msg_type_list
self.deque = deque()
self.pipe = pipe
self.subscriber = None
self.create_subscriber()
self.running = False
def add_address(self, ip, port):
'''Add address that will be listened
'''
if ip is not None and port is not None:
self.address_list.append('tcp://' + ip + ':%04d' % port)
def add_address_list(self, address_list):
'''Add a list of addresses that will be listened
'''
for address in address_list:
self.address_list.append(address)
def create_subscriber(self):
'''Create a subscriber instance using specified addresses and
message types.
'''
if self.subscriber is None:
if len(self.address_list) > 0:
if len(self.msg_type_list) > 0:
self.subscriber = Subscriber(self.address_list,
*self.msg_type_list)
def send_to_pipe(self, msg):
'''Send message to parent via a Pipe()
'''
self.pipe.send(msg)
def run(self):
'''Run listener
'''
# TODO: add logging
print "Starting Listener"
self.running = True
for msg in self.subscriber.recv():
print "New message received"
if msg.subject == '/stop_listener':
break
if self.pipe is None:
self.deque.append(msg)
else:
while len(self.deque) > 0:
self.send_to_pipe(self.deque.popleft())
self.send_to_pipe(msg)
def stop(self):
'''Stop subscriber and delete the instance
'''
# TODO: add logging
self.subscriber.stop()
self.subscriber.close()
self.subscriber = None
self.running = False
def restart(self):
'''Restart subscriber
'''
self.stop()
self.create_subscriber()
self.run()
class Listener(Thread):
"""PyTroll listener class for reading messages for Trollduction."""
def __init__(self, address, topics, callback, *args, **kwargs):
"""Init Listener object."""
super(Listener, self).__init__()
self.topics = topics
self.callback = callback
self.subscriber = None
self.address = address
self.running = False
self.cargs = args
self.ckwargs = kwargs
self.restart_event = Event()
def create_subscriber(self):
"""Create a subscriber using specified addresses and message types."""
if self.subscriber is None:
if self.topics:
LOGGER.info("Subscribing to %s with topics %s",
str(self.address), str(self.topics))
self.subscriber = Subscriber(self.address, self.topics)
LOGGER.debug("Subscriber %s", str(self.subscriber))
def run(self):
"""Run listener."""
with heartbeat_monitor.Monitor(self.restart_event,
**self.ckwargs) as beat_monitor:
self.running = True
while self.running:
# Loop for restart.
LOGGER.debug("Starting listener %s", str(self.address))
self.create_subscriber()
for msg in self.subscriber(timeout=1):
if not self.running:
break
if self.restart_event.is_set():
self.restart_event.clear()
self.stop()
self.running = True
break
if msg is None:
continue
LOGGER.debug("Receiving (SUB) %s", str(msg))
beat_monitor(msg)
if msg.type == "beat":
continue
# Handle public "push" messages as a hot spare client
if msg.type == "push":
# TODO: these need to be checked and acted if
# the transfers are not finished on primary
# client and are not cleared
LOGGER.debug("Primary client published 'push'")
add_to_ongoing(msg)
# Handle public "ack" messages as a hot spare client
if msg.type == "ack":
LOGGER.debug("Primary client finished transfer")
_ = add_to_file_cache(msg)
_ = clean_ongoing_transfer(get_msg_uid(msg))
# If this is a hot spare client, wait for a while
# for a public "push" message which will update
# the ongoing transfers before starting processing here
delay = self.ckwargs.get("processing_delay", False)
if delay:
add_timer(float(delay), self.callback, msg,
*self.cargs, **self.ckwargs)
else:
self.callback(msg, *self.cargs, **self.ckwargs)
LOGGER.debug("Exiting listener %s", str(self.address))
def stop(self):
"""Stop subscriber and delete the instance."""
self.running = False
time.sleep(1)
if self.subscriber is not None:
self.subscriber.close()
self.subscriber = None
def create_subscriber(self):
'''
'''
if len(self.address_list) > 0:
self.subscriber = Subscriber(self.address_list, self.type_list)
class Listener(object):
'''PyTroll listener class for reading messages for Trollduction
'''
def __init__(self, address_list=None, msg_type_list=None,
ip=None, port=None, pipe=None):
'''Init Listener object
'''
self.address_list = []
self.add_address_list(address_list)
self.add_address(ip, port)
self.msg_type_list = []
if msg_type_list is not None:
self.msg_type_list = msg_type_list
self.deque = deque()
self.pipe = pipe
self.subscriber = None
self.create_subscriber()
self.running = False
def add_address(self, ip, port):
'''Add address that will be listened
'''
if ip is not None and port is not None:
self.address_list.append('tcp://'+ip+':%04d' % port)
def add_address_list(self, address_list):
'''Add a list of addresses that will be listened
'''
for address in address_list:
self.address_list.append(address)
def create_subscriber(self):
'''Create a subscriber instance using specified addresses and
message types.
'''
if self.subscriber is None:
if len(self.address_list) > 0:
if len(self.msg_type_list) > 0:
self.subscriber = Subscriber(self.address_list,
*self.msg_type_list)
def send_to_pipe(self, msg):
'''Send message to parent via a Pipe()
'''
self.pipe.send(msg)
def run(self):
'''Run listener
'''
# TODO: add logging
print "Starting Listener"
self.running = True
for msg in self.subscriber.recv():
print "New message received"
if msg.subject == '/stop_listener':
break
if self.pipe is None:
self.deque.append(msg)
else:
while len(self.deque) > 0:
self.send_to_pipe(self.deque.popleft())
self.send_to_pipe(msg)
def stop(self):
'''Stop subscriber and delete the instance
'''
# TODO: add logging
self.subscriber.stop()
self.subscriber.close()
self.subscriber = None
self.running = False
def restart(self):
'''Restart subscriber
'''
self.stop()
self.create_subscriber()
self.run()
class Listener(Thread):
"""PyTroll listener class for reading messages for Trollduction."""
def __init__(self, address, topics, *args, die_event=None, **kwargs):
"""Init Listener object."""
super(Listener, self).__init__()
self.topics = topics
self.subscriber = None
self.address = address
self.running = False
self.die_event = die_event
self.cargs = args
self.ckwargs = kwargs
self.restart_event = Event()
self.cause_of_death = None
self.death_count = 0
def restart(self):
"""Restart the listener, returns a new running instance."""
self.stop()
new_listener = self.__class__(self.address,
self.topics,
*self.cargs,
die_event=self.die_event,
**self.ckwargs)
new_listener.death_count = self.death_count + 1
new_listener.start()
return new_listener
def create_subscriber(self):
"""Create a subscriber using specified addresses and message types."""
if self.subscriber is None:
if self.topics:
LOGGER.info("Subscribing to %s with topics %s",
str(self.address), str(self.topics))
self.subscriber = Subscriber(self.address, self.topics)
LOGGER.debug("Subscriber %s", str(self.subscriber))
def run(self):
"""Run listener."""
try:
with heartbeat_monitor.Monitor(self.restart_event,
**self.ckwargs) as beat_monitor:
self.running = True
while self.running:
LOGGER.debug("Starting listener %s", str(self.address))
self.create_subscriber()
self._get_messages(beat_monitor)
except Exception as err:
LOGGER.exception("Listener died.")
self.cause_of_death = err
with suppress(AttributeError):
self.die_event.set()
def _get_messages(self, beat_monitor):
for msg in self.subscriber(timeout=1):
if not self.running:
break
if not self._check_heartbeat():
break
if msg is None:
continue
LOGGER.debug("Receiving (SUB) %s", str(msg))
beat_monitor(msg)
if self._is_message_already_handled(msg):
continue
self._process_message(msg)
LOGGER.debug("Exiting listener %s", str(self.address))
def _check_heartbeat(self):
if self.restart_event.is_set():
LOGGER.warning(
"Missing a heartbeat, restarting the subscriber to %s.",
str(self.subscriber.addresses))
self.restart_event.clear()
self.stop()
self.running = True
return False
return True
def _is_message_already_handled(self, msg):
return (self._handle_beat_message(msg) or _handle_push_message(msg)
or _handle_ack_message(msg)
or _handle_message_from_another_client(msg))
def _handle_beat_message(self, msg):
if msg.type == "beat":
self.death_count = 0
return True
return False
def _process_message(self, msg):
delay = self.ckwargs.get("processing_delay", False)
if delay:
# If this is a hot spare client, wait for a while
# for a public "push" message which will update
# the ongoing transfers before starting processing here
add_request_push_timer(float(delay), msg, *self.cargs,
**self.ckwargs)
else:
request_push(msg, *self.cargs, **self.ckwargs)
def stop(self):
"""Stop subscriber and delete the instance."""
self.running = False
time.sleep(1)
if self.subscriber is not None:
self.subscriber.close()
self.subscriber = None
