def __init__(self, port, attrs=None):
Thread.__init__(self)
self._loop = True
self.out_socket = get_context().socket(ROUTER)
self.out_socket.bind("tcp://*:" + str(port))
self.port = port
self.in_socket = get_context().socket(PULL)
self.in_socket.bind("inproc://replies" + str(port))
self._poller = Poller()
self._poller.register(self.out_socket, POLLIN)
self._poller.register(self.in_socket, POLLIN)
self._attrs = attrs
try:
# Checking the validity of the file pattern
_pattern = globify(attrs["origin"])
except ValueError as err:
raise ConfigError('Invalid file pattern: ' + str(err))
except KeyError:
if 'listen' not in attrs:
raise
self._deleter = Deleter()
try:
self._station = self._attrs["station"]
except (KeyError, TypeError):
LOGGER.warning("Station is not defined in config file")
self._station = "unknown"
LOGGER.debug("Station is '%s'" % self._station)
def get_pub_address(name, timeout=10, nameserver="localhost"):
"""Get the address of the publisher for a given publisher *name* from the
nameserver on *nameserver* (localhost by default).
"""
# Socket to talk to server
socket = get_context().socket(REQ)
try:
socket.setsockopt(LINGER, timeout * 1000)
socket.connect("tcp://" + nameserver + ":" + str(PORT))
logger.debug('Connecting to %s',
"tcp://" + nameserver + ":" + str(PORT))
poller = Poller()
poller.register(socket, POLLIN)
message = Message("/oper/ns", "request", {"service": name})
socket.send_string(six.text_type(message))
# Get the reply.
sock = poller.poll(timeout=timeout * 1000)
if sock:
if sock[0][0] == socket:
message = Message.decode(socket.recv_string(NOBLOCK))
return message.data
else:
raise TimeoutError("Didn't get an address after %d seconds." %
timeout)
finally:
socket.close()
def __init__(self, address, name=""):
"""Bind the publisher class to a port.
"""
# pylint: disable=E1103
self.name = name
self.destination = address
self.publish = get_context().socket(zmq.PUB)
# Check for port 0 (random port)
u__ = urlsplit(self.destination)
port = u__.port
if port == 0:
dest = urlunsplit((u__.scheme, u__.hostname,
u__.path, u__.query, u__.fragment))
self.port_number = self.publish.bind_to_random_port(dest)
netloc = u__.hostname + ":" + str(self.port_number)
self.destination = urlunsplit((u__.scheme, netloc, u__.path,
u__.query, u__.fragment))
else:
self.publish.bind(self.destination)
self.port_number = port
LOGGER.info("publisher started on port %s", str(self.port_number))
# Initialize no heartbeat
self._heartbeat = None
self._pub_lock = Lock()
def get_pub_address(name, timeout=10, nameserver="localhost"):
"""Get the address of the publisher for a given publisher *name* from the
nameserver on *nameserver* (localhost by default).
"""
# Socket to talk to server
socket = get_context().socket(REQ)
try:
socket.setsockopt(LINGER, timeout * 1000)
socket.connect("tcp://" + nameserver + ":" + str(PORT))
logger.debug('Connecting to %s',
"tcp://" + nameserver + ":" + str(PORT))
poller = Poller()
poller.register(socket, POLLIN)
message = Message("/oper/ns", "request", {"service": name})
socket.send_string(six.text_type(message))
# Get the reply.
sock = poller.poll(timeout=timeout * 1000)
if sock:
if sock[0][0] == socket:
message = Message.decode(socket.recv_string(NOBLOCK))
return message.data
else:
raise TimeoutError("Didn't get an address after %d seconds."
% timeout)
finally:
socket.close()
def __init__(self, address, name="", min_port=None, max_port=None):
"""Bind the publisher class to a port.
"""
# pylint: disable=E1103
self.name = name
self.destination = address
self.publish = get_context().socket(zmq.PUB)
# Limit port range or use the defaults when no port is defined
# by the user
min_port = min_port or int(
os.environ.get('POSTTROLL_PUB_MIN_PORT', 49152))
max_port = max_port or int(
os.environ.get('POSTTROLL_PUB_MAX_PORT', 65536))
# Check for port 0 (random port)
u__ = urlsplit(self.destination)
port = u__.port
if port == 0:
dest = urlunsplit(
(u__.scheme, u__.hostname, u__.path, u__.query, u__.fragment))
self.port_number = self.publish.bind_to_random_port(
dest, min_port=min_port, max_port=max_port)
netloc = u__.hostname + ":" + str(self.port_number)
self.destination = urlunsplit(
(u__.scheme, netloc, u__.path, u__.query, u__.fragment))
else:
self.publish.bind(self.destination)
self.port_number = port
LOGGER.info("publisher started on port %s", str(self.port_number))
# Initialize no heartbeat
self._heartbeat = None
self._pub_lock = Lock()
def run(self, *args):
"""Run the listener and answer to requests.
"""
del args
arec = AddressReceiver(max_age=self._max_age,
multicast_enabled=self._multicast_enabled)
arec.start()
port = PORT
try:
with nslock:
self.listener = get_context().socket(REP)
self.listener.bind("tcp://*:" + str(port))
logger.debug('Listening on port %s', str(port))
poller = Poller()
poller.register(self.listener, POLLIN)
while self.loop:
with nslock:
socks = dict(poller.poll(1000))
if socks:
if socks.get(self.listener) == POLLIN:
msg = self.listener.recv_string()
else:
continue
logger.debug("Replying to request: " + str(msg))
msg = Message.decode(msg)
self.listener.send_unicode(six.text_type(get_active_address(
msg.data["service"], arec)))
except KeyboardInterrupt:
# Needed to stop the nameserver.
pass
finally:
arec.stop()
self.stop()
def add_hook_pull(self, address, callback):
"""Same as above, but with a PULL socket.
(e.g good for pushed 'inproc' messages from another thread).
"""
LOGGER.info("Subscriber adding PULL hook %s", str(address))
socket = get_context().socket(PULL)
socket.connect(address)
self._add_hook(socket, callback)
def add_hook_pull(self, address, callback):
"""Same as above, but with a PULL socket.
(e.g good for pushed 'inproc' messages from another thread).
"""
LOGGER.info("Subscriber adding PULL hook %s", str(address))
socket = get_context().socket(PULL)
socket.connect(address)
self._add_hook(socket, callback)
def _send_multipart_reply(self, reply, address):
LOGGER.debug("Response: %s", str(reply))
in_socket = get_context().socket(PUSH)
in_socket.connect("inproc://replies" + str(self.port))
try:
in_socket.send_multipart([address, b'', str(reply)])
except TypeError:
in_socket.send_multipart(
[address, b'', bytes(str(reply), 'utf-8')])
def reset(self, addr):
with self._lock:
idx = self._addresses.index(addr)
self._poller.unregister(self.subscribers[idx])
self.subscribers[idx].setsockopt(LINGER, 0)
self.subscribers[idx].close()
self.subscribers[idx] = get_context().socket(SUB)
self.subscribers[idx].setsockopt(SUBSCRIBE, "pytroll")
self.subscribers[idx].connect(addr)
self._poller.register(self.subscribers[idx], POLLIN)
def __init__(self, holder, port, station):
Thread.__init__(self)
self._holder = holder
self._loop = True
self._port = port
self._station = station
self._lock = Lock()
self._socket = get_context().socket(REP)
self._socket.bind("tcp://*:" + str(self._port))
self._poller = Poller()
self._poller.register(self._socket, POLLIN)
def __init__(self, addresses, translate=False):
self._addresses = addresses
self._translate = translate
self.subscribers = []
self._poller = Poller()
for addr in self._addresses:
subscriber = get_context().socket(SUB)
subscriber.setsockopt(SUBSCRIBE, "pytroll")
subscriber.connect(addr)
self.subscribers.append(subscriber)
self._poller.register(subscriber)
self._lock = Lock()
self._loop = True
def add_hook_sub(self, address, topics, callback):
"""Specify a *callback* in the same stream (thread) as the main receive
loop. The callback will be called with the received messages from the
specified subscription.
Good for operations, which is required to be done in the same thread as
the main recieve loop (e.q operations on the underlying sockets).
"""
LOGGER.info("Subscriber adding SUB hook %s for topics %s",
str(address), str(topics))
socket = get_context().socket(SUB)
for t__ in self._magickfy_topics(topics):
socket.setsockopt_string(SUBSCRIBE, six.text_type(t__))
socket.connect(address)
self._add_hook(socket, callback)
def add_hook_sub(self, address, topics, callback):
"""Specify a *callback* in the same stream (thread) as the main receive
loop. The callback will be called with the received messages from the
specified subscription.
Good for operations, which is required to be done in the same thread as
the main recieve loop (e.q operations on the underlying sockets).
"""
LOGGER.info("Subscriber adding SUB hook %s for topics %s",
str(address), str(topics))
socket = get_context().socket(SUB)
for t__ in self._magickfy_topics(topics):
socket.setsockopt_string(SUBSCRIBE, six.text_type(t__))
socket.connect(address)
self._add_hook(socket, callback)
def __init__(self, holder, host, pubport, reqport, sched):
Thread.__init__(self)
self._holder = holder
self._pubaddress = "tcp://" + host + ":" + str(pubport)
self._reqaddress = "tcp://" + host + ":" + str(reqport)
self._req = SimpleRequester(host, reqport)
self._subsocket = get_context().socket(SUB)
self._subsocket.connect(self._pubaddress)
self._subsocket.setsockopt_string(SUBSCRIBE, "pytroll")
self._poller = Poller()
self._poller.register(self._subsocket, POLLIN)
self._lock = Lock()
self._loop = True
self._sched = sched
def reply_and_send(self, fun, address, message):
in_socket = get_context().socket(PUSH)
in_socket.connect("inproc://replies" + str(self.port))
reply = Message(message.subject, "error")
try:
reply = fun(message)
except Exception:
LOGGER.exception("Something went wrong"
" when processing the request: %s", str(message))
finally:
LOGGER.debug("Response: %s", str(reply))
try:
in_socket.send_multipart([address, b'', str(reply)])
except TypeError:
in_socket.send_multipart([address, b'', bytes(str(reply),
'utf-8')])
def _send_to_address(self, address, data, timeout=10):
"""Send data to *address* and *port* without verification of response."""
# Socket to talk to server
socket = get_context().socket(REQ)
try:
socket.setsockopt(LINGER, timeout * 1000)
if address.find(":") == -1:
socket.connect("tcp://%s:%d" % (address, self.default_port))
else:
socket.connect("tcp://%s" % address)
socket.send_string(data)
message = socket.recv_string()
if message != "ok":
LOGGER.warn("invalid acknowledge received: %s" % message)
finally:
socket.close()
def add(self, address, topics=None):
"""Add *address* to the subscribing list for *topics*.
It topics is None we will subscibe to already specified topics.
"""
with self._lock:
if address in self.addresses:
return False
topics = self._magickfy_topics(topics) or self._topics
LOGGER.info("Subscriber adding address %s with topics %s",
str(address), str(topics))
subscriber = get_context().socket(SUB)
for t__ in topics:
subscriber.setsockopt_string(SUBSCRIBE, six.text_type(t__))
subscriber.connect(address)
self.sub_addr[subscriber] = address
self.addr_sub[address] = subscriber
if self.poller:
self.poller.register(subscriber, POLLIN)
return True
def add(self, address, topics=None):
"""Add *address* to the subscribing list for *topics*.
It topics is None we will subscibe to already specified topics.
"""
with self._lock:
if address in self.addresses:
return False
topics = self._magickfy_topics(topics) or self._topics
LOGGER.info("Subscriber adding address %s with topics %s",
str(address), str(topics))
subscriber = get_context().socket(SUB)
for t__ in topics:
subscriber.setsockopt_string(SUBSCRIBE, six.text_type(t__))
subscriber.connect(address)
self.sub_addr[subscriber] = address
self.addr_sub[address] = subscriber
if self.poller:
self.poller.register(subscriber, POLLIN)
return True
def run(self, *args):
"""Run the listener and answer to requests.
"""
del args
arec = AddressReceiver(
max_age=self._max_age,
multicast_enabled=self._multicast_enabled,
restrict_to_localhost=self._restrict_to_localhost)
arec.start()
port = PORT
try:
with nslock:
self.listener = get_context().socket(REP)
self.listener.bind("tcp://*:" + str(port))
logger.debug('Listening on port %s', str(port))
poller = Poller()
poller.register(self.listener, POLLIN)
while self.loop:
with nslock:
socks = dict(poller.poll(1000))
if socks:
if socks.get(self.listener) == POLLIN:
msg = self.listener.recv_string()
else:
continue
logger.debug("Replying to request: " + str(msg))
msg = Message.decode(msg)
active_address = get_active_address(
msg.data["service"], arec)
self.listener.send_unicode(six.text_type(active_address))
except KeyboardInterrupt:
# Needed to stop the nameserver.
pass
finally:
arec.stop()
self.stop()
def __init__(self, port=None):
self._port = port or default_publish_port
self._socket = get_context().socket(REP)
self._socket.bind("tcp://*:" + str(port))
def __init__(self, port):
self._socket = get_context().socket(PUB)
self._socket.bind("tcp://*:" + str(port))
self._lock = Lock()
def serve(configfile):
"""Serve forever.
"""
try:
cfg = RawConfigParser()
cfg.read(configfile)
host = cfg.get("local_reception", "localhost")
# for messages
station = cfg.get("local_reception", "station")
set_subject(station)
# for elevation
global coords
coords = cfg.get("local_reception", "coordinates")
coords = [float(coord) for coord in coords.split()]
global tle_files
try:
tle_files = cfg.get("local_reception", "tle_files")
except NoOptionError:
tle_files = None
# publisher
pubport = cfg.getint(host, "pubport")
pub = Publisher(pubport)
# schedule reader
try:
sched = ScheduleReader(
cfg.get("local_reception", "schedule_file"),
cfg.get("local_reception", "schedule_format"))
sched.get_next_pass()
except NoOptionError:
logger.warning("No schedule file given")
sched = ScheduleReader(None, None)
# heart
hostname = cfg.get(host, "hostname")
pubaddress = hostname + ":" + str(pubport)
heart = Heart(pub, pubaddress, 30, sched)
heart.start()
# holder
holder = Holder(pub, pubaddress)
# cleaner
cleaner = Cleaner(holder, 1)
cleaner.start()
# watcher
#watcher = DummyWatcher(holder, 2)
path = cfg.get("local_reception", "data_dir")
watcher = None
if not os.path.exists(path):
logger.warning(path +
" doesn't exist, not getting data from files")
else:
pattern = cfg.get("local_reception", "file_pattern", raw=True)
watcher = FileWatcher(holder, os.path.join(path, pattern), sched)
watcher.start()
mirror_watcher = None
try:
mirror = cfg.get("local_reception", "mirror")
except NoOptionError:
pass
else:
pubport_m = cfg.getint(mirror, "pubport")
reqport_m = cfg.getint(mirror, "reqport")
host_m = cfg.get(mirror, "hostname")
mirror_watcher = MirrorWatcher(holder, host_m, pubport_m,
reqport_m, sched)
mirror_watcher.start()
# request manager
reqport = cfg.getint(host, "reqport")
reqman = RequestManager(holder, reqport, station)
reqman.start()
while True:
time.sleep(10000)
except KeyboardInterrupt:
pass
except:
logger.exception("There was an error!")
raise
finally:
try:
reqman.stop()
except UnboundLocalError:
pass
try:
if mirror_watcher is not None:
mirror_watcher.stop()
except UnboundLocalError:
pass
try:
if watcher is not None:
watcher.stop()
except UnboundLocalError:
pass
try:
cleaner.stop()
except UnboundLocalError:
pass
try:
heart.stop()
except UnboundLocalError:
pass
try:
pub.stop()
except UnboundLocalError:
pass
try:
get_context().term()
except ZMQError:
pass
def connect(self):
"""Connect to the server."""
self._socket = get_context().socket(REQ)
self._socket.connect(self._reqaddress)
self._poller.register(self._socket, POLLIN)
def connect(self):
"""Connect to the server."""
self._socket = get_context().socket(REQ)
# self._socket.setsockopt(zmq.RCVTIMEO, 500) # milliseconds
self._socket.connect(self._reqaddress)
self._poller.register(self._socket, POLLIN)
def _set_in_socket(self):
self.in_socket = get_context().socket(PULL)
self.in_socket.bind("inproc://replies" + str(self.port))
def _set_out_socket(self):
self.out_socket = get_context().socket(ROUTER)
self.out_socket.bind("tcp://*:" + str(self.port))
def __init__(self, port=None):
self._port = port or default_publish_port
self._socket = get_context().socket(REP)
self._socket.bind("tcp://*:" + str(port))
