class HomeBase(Thread):
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
def cleanup(self):
self.pull.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv()
msg += ", WORK RECEIVED "
print msg
if self._shutdown:
break
self.cleanup()
class Leatherneck(Thread):
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
def cleanup(self):
self.push.close()
self.pull.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv()
msg += " WORK COMPLETE, " + str(time())
self.push.send(msg)
if self._shutdown:
break
self.cleanup()
def _send_request(req_id, action, key, value=None):
""" Generate a request ID, push the request to the node and wait for the
result, filtering by the ID on the subscription.
This should be a request/response call to the node, but since many must be
possible at the same time, a PUSH/PULL AND PUB/SUB with an unique ID for
each request is used instead.
:return: the node response
:rtype: []
"""
# Create and connect the sockets.
context = current_app.config["context"]
subscriber = context.socket(SUB)
subscriber.setsockopt_string(SUBSCRIBE, req_id)
subscriber.connect(SUB_ENDPOINT)
request = context.socket(PUSH)
request.setsockopt(SNDTIMEO, _TIMEOUT)
request.connect(PUSH_ENDPOINT)
# Push the request.
request.send_json([req_id, action, key, value])
# Wait for the response from the publisher.
poller = Poller()
poller.register(subscriber, POLLIN)
sockets = dict(poller.poll(_TIMEOUT))
if subscriber not in sockets: # no response, time out
raise InternalServerError()
# Return the response, without the unique request ID
return subscriber.recv_multipart()[1:]
def get_pub_address(name, timeout=10, nameserver="localhost"):
"""Get the address of the publisher for a given publisher *name* from the
nameserver on *nameserver* (localhost by default).
"""
# Socket to talk to server
socket = context.socket(REQ)
try:
socket.setsockopt(LINGER, timeout * 1000)
socket.connect("tcp://" + nameserver + ":" + str(PORT))
poller = Poller()
poller.register(socket, POLLIN)
message = Message("/oper/ns", "request", {"service": name})
socket.send(str(message))
# Get the reply.
sock = poller.poll(timeout=timeout * 1000)
if sock:
if sock[0][0] == socket:
message = Message.decode(socket.recv(NOBLOCK))
return message.data
else:
raise TimeoutError("Didn't get an address after %d seconds."
% timeout)
finally:
socket.close()
def run(self, *args):
"""Run the listener and answer to requests.
"""
del args
arec = AddressReceiver(max_age=self._max_age)
arec.start()
port = PORT
try:
self.listener = context.socket(REP)
self.listener.bind("tcp://*:" + str(port))
poller = Poller()
poller.register(self.listener, POLLIN)
while self.loop:
socks = dict(poller.poll(1000))
if socks:
if socks.get(self.listener) == POLLIN:
msg = self.listener.recv()
else:
continue
logger.debug("Replying to request: " + str(msg))
msg = Message.decode(msg)
self.listener.send_unicode(str(get_active_address(
msg.data["service"], arec)))
except KeyboardInterrupt:
# Needed to stop the nameserver.
pass
finally:
arec.stop()
self.listener.close()
class Listener(Thread):
def __init__(self):
super(Listener, self).__init__(name="Listener")
self._shutdown = False
self.context = Context()
self.sub = self.context.socket(SUB)
self.sub.bind('tcp://*:7000')
self.sub.setsockopt(SUBSCRIBE, "")
self.poller = Poller()
self.poller.register(self.sub, POLLIN)
def cleanup(self):
self.sub.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.sub) == POLLIN:
msg = self.sub.recv(flags=NOBLOCK)
print msg
if self._shutdown:
break
self.cleanup()
class HomeBase(Thread):
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def cleanup(self):
print "Home exiting..."
self.pull.close()
self.context.term()
def run(self):
while True:
if not self.mainthread.is_alive():
self._shutdown = True
break
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv(flags=NOBLOCK)
msg += ", WORK RECEIVED "
print msg
if self._shutdown:
break
self.cleanup()
class Leatherneck(Thread):
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def cleanup(self):
print "Workers exiting..."
self.push.close()
self.pull.close()
self.context.term()
def run(self):
while True:
if not self.mainthread.is_alive():
self._shutdown = True
break
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv(flags=NOBLOCK)
msg += " WORK COMPLETE, " + str(time())
self.push.send(msg, flags=NOBLOCK)
if self._shutdown:
break
self.cleanup()
class Stethoscope(Thread):
def __init__(self, context, *args, **kw):
self.context = context
self.recv = self.context.socket(PAIR)
self.recv.connect("inproc://#1")
self.pub = self.context.socket(PUB)
self.pub.connect('tcp://localhost:7003')
self.pub.setsockopt(HWM, 1000)
self.poller = Poller()
self.poller.register(self.recv, POLLIN)
super(Stethoscope, self).__init__(*args, **kw)
def cleanup(self):
self.recv.close()
self.pub.close()
def run(self):
try:
while not shutdown.is_set():
socks = dict(self.poller.poll())
if socks.get(self.recv) == POLLIN:
msg = self.recv.recv()
self.pub.send(msg, flags=NOBLOCK)
if msg == "DIE":
raise KillThread
except KillThread:
print "%s exiting..." % self.name
finally:
self.cleanup()
class MirrorWatcher(Thread):
"""Watches a other server.
"""
def __init__(self, holder, host, pubport, reqport, sched):
Thread.__init__(self)
self._holder = holder
self._pubaddress = "tcp://" + host + ":" + str(pubport)
self._reqaddress = "tcp://" + host + ":" + str(reqport)
self._req = SimpleRequester(host, reqport)
self._subsocket = context.socket(SUB)
self._subsocket.connect(self._pubaddress)
self._subsocket.setsockopt(SUBSCRIBE, "pytroll")
self._poller = Poller()
self._poller.register(self._subsocket, POLLIN)
self._lock = Lock()
self._loop = True
self._sched = sched
def run(self):
last_hb = datetime.now()
minutes = 2
while self._loop:
if datetime.now() - last_hb > timedelta(minutes=minutes):
logger.error("No heartbeat from " + str(self._pubaddress))
last_hb = datetime.now()
minutes = 1440
socks = dict(self._poller.poll(2000))
if (socks and
self._subsocket in socks and
socks[self._subsocket] == POLLIN):
message = Message.decode(self._subsocket.recv())
else:
continue
if message.type == "have":
sat = message.data["satellite"]
key = strp_isoformat(message.data["timecode"])
elevation = message.data["elevation"]
quality = message.data.get("quality", 100)
data = _MirrorGetter(self._req, sat, key)
self._holder.add(sat, key, elevation, quality, data)
if message.type == "heartbeat":
logger.debug("Got heartbeat from " + str(self._pubaddress)
+ ": " + str(message))
self._sched.mirror_next_pass = message.data["next_pass"]
last_hb = datetime.now()
minutes = 2
def stop(self):
"""Stop the watcher
"""
self._loop = False
self._req.stop()
self._subsocket.setsockopt(LINGER, 0)
self._subsocket.close()
class ZMQPoller(object):
"""A poller that can be used in the tornado IOLoop.
This simply wraps a regular zmq.Poller, scaling the timeout
by 1000, so that it is in seconds rather than milliseconds.
"""
def __init__(self):
self._poller = Poller()
@staticmethod
def _map_events(events):
"""translate IOLoop.READ/WRITE/ERROR event masks into zmq.POLLIN/OUT/ERR"""
z_events = 0
if events & IOLoop.READ:
z_events |= POLLIN
if events & IOLoop.WRITE:
z_events |= POLLOUT
if events & IOLoop.ERROR:
z_events |= POLLERR
return z_events
@staticmethod
def _remap_events(z_events):
"""translate zmq.POLLIN/OUT/ERR event masks into IOLoop.READ/WRITE/ERROR"""
events = 0
if z_events & POLLIN:
events |= IOLoop.READ
if z_events & POLLOUT:
events |= IOLoop.WRITE
if z_events & POLLERR:
events |= IOLoop.ERROR
return events
def register(self, fd, events):
return self._poller.register(fd, self._map_events(events))
def modify(self, fd, events):
return self._poller.modify(fd, self._map_events(events))
def unregister(self, fd):
return self._poller.unregister(fd)
def poll(self, timeout):
"""poll in seconds rather than milliseconds.
Event masks will be IOLoop.READ/WRITE/ERROR
"""
z_events = self._poller.poll(1000*timeout)
return [ (fd,self._remap_events(evt)) for (fd,evt) in z_events ]
def close(self):
pass
class ZMQPoller(object):
"""A poller that can be used in the tornado IOLoop.
This simply wraps a regular zmq.Poller, scaling the timeout
by 1000, so that it is in seconds rather than milliseconds.
"""
def __init__(self):
self._poller = Poller()
@staticmethod
def _map_events(events):
"""translate IOLoop.READ/WRITE/ERROR event masks into zmq.POLLIN/OUT/ERR"""
z_events = 0
if events & IOLoop.READ:
z_events |= POLLIN
if events & IOLoop.WRITE:
z_events |= POLLOUT
if events & IOLoop.ERROR:
z_events |= POLLERR
return z_events
@staticmethod
def _remap_events(z_events):
"""translate zmq.POLLIN/OUT/ERR event masks into IOLoop.READ/WRITE/ERROR"""
events = 0
if z_events & POLLIN:
events |= IOLoop.READ
if z_events & POLLOUT:
events |= IOLoop.WRITE
if z_events & POLLERR:
events |= IOLoop.ERROR
return events
def register(self, fd, events):
return self._poller.register(fd, self._map_events(events))
def modify(self, fd, events):
return self._poller.modify(fd, self._map_events(events))
def unregister(self, fd):
return self._poller.unregister(fd)
def poll(self, timeout):
"""poll in seconds rather than milliseconds.
Event masks will be IOLoop.READ/WRITE/ERROR
"""
z_events = self._poller.poll(1000 * timeout)
return [(fd, self._remap_events(evt)) for (fd, evt) in z_events]
def close(self):
pass
def __init__(self, addresses, translate=False):
self._addresses = addresses
self._translate = translate
self.subscribers = []
self._poller = Poller()
for addr in self._addresses:
subscriber = context.socket(SUB)
subscriber.setsockopt(SUBSCRIBE, "pytroll")
subscriber.connect(addr)
self.subscribers.append(subscriber)
self._poller.register(subscriber)
self._lock = Lock()
self._loop = True
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def __init__(self, holder, host, pubport, reqport, sched):
Thread.__init__(self)
self._holder = holder
self._pubaddress = "tcp://" + host + ":" + str(pubport)
self._reqaddress = "tcp://" + host + ":" + str(reqport)
self._req = SimpleRequester(host, reqport)
self._subsocket = get_context().socket(SUB)
self._subsocket.connect(self._pubaddress)
self._subsocket.setsockopt_string(SUBSCRIBE, "pytroll")
self._poller = Poller()
self._poller.register(self._subsocket, POLLIN)
self._lock = Lock()
self._loop = True
self._sched = sched
def __init__(self, **kwargs):
self._input_sock = None
self._output_sock = None
self._control_sock = None
# determine if outgoing messages should enable NOBLOCK on send
# default behaviour is to block on a send call till receiver is present
self.no_block_send = False
# configure the interrupt handling
self._stop = True
signal.signal(signal.SIGINT, self._signal_interrupt_handler)
# a regular hearbeat interval must be set to the default.
self.heartbeat = 3 # seconds
# create the zmq context
self.zmq_ctx = Context()
# set the default input receive handler, if none has been assigned
if not hasattr(self, 'input_recv_handler'):
self.input_recv_handler = self._default_recv_handler
# set the default handler, if none has been assigned.
if not hasattr(self, '_command_handler'):
self._command_handler = self._default_command_handler
# construct the poller
self._poll = Poller()
# monitoring of message stream is off by default
self.monitor_stream = False
Scaffold.__init__(self, **kwargs)
def __init__(self, impl=None):
self._impl = impl or Poller()
if hasattr(self._impl, 'fileno'):
self._set_close_exec(self._impl.fileno())
self._handlers = {}
self._events = {}
self._callbacks = set()
self._timeouts = []
self._running = False
self._stopped = False
self._blocking_log_threshold = None
# Create a pipe that we send bogus data to when we want to wake
# the I/O loop when it is idle
if os.name != 'nt':
r, w = os.pipe()
self._set_nonblocking(r)
self._set_nonblocking(w)
self._set_close_exec(r)
self._set_close_exec(w)
self._waker_reader = os.fdopen(r, "r", 0)
self._waker_writer = os.fdopen(w, "w", 0)
else:
self._waker_reader = self._waker_writer = win32_support.Pipe()
r = self._waker_writer.reader_fd
self.add_handler(r, self._read_waker, self.READ)
def init_connexions(self):
"""
Initialise zmq sockets, poller and queues.
Because this class is s Process, This method must be call in the run
method to be hold by the correct processus.
"""
# Prepare our context and sockets
context = Context()
# Socket to talk to prev_stages
for name, connexions in self.connexions.items():
self.queue_limit[name] = connexions[2]
sock_router = context.socket(ROUTER)
try:
sock_router.bind('tcp://*:' + connexions[0])
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'.format(
e, connexions[0]))
return False
self.router_sockets[name] = sock_router
# Socket to talk to next_stages
sock_dealer = context.socket(ROUTER)
try:
sock_dealer.bind("tcp://*:" + connexions[1])
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'.format(
e, connexions[1]))
return False
self.dealer_sockets[name] = sock_dealer
self.next_available_stages[name] = list()
self.queue_jobs[name] = list()
# Use a ZMQ Pool to get multichannel message
self.poller = Poller()
# Register dealer socket to next_stage
for n, dealer in self.dealer_sockets.items():
self.poller.register(dealer, POLLIN)
for n, router in self.router_sockets.items():
self.poller.register(router, POLLIN)
# Register router socket to prev_stages or producer
self.socket_pub = context.socket(PUB)
if self.gui_address is not None:
try:
self.socket_pub.connect("tcp://" + self.gui_address)
except ZMQError as e:
self.log.error("".format(e, self.gui_address))
return False
# This flag stop this current processus
return True
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
def _request_quote(cls, request, ticker, context=None):
try:
socket = context.socket(REQ)
socket.connect("tcp://serum-server:6900")
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
socket.send(bytes(dumps({"endpoint": "quote", "marketAddress": ticker.get("id"), "program": ticker.get("symbol")}), encoding='utf8'))
responses = poller.poll(5000)
if len(responses) != 0:
response = socket.recv()
socket.close()
rawData = loads(response)
else:
socket.close()
return [{}, ""]
except:
return [{}, ""]
coinThumbnail = static_storage.icon if ticker.get("image") is None else ticker.get("image")
price = [float(rawData["price"])]
priceChange = 0
volume = 0
tokenImage = static_storage.icon
base = "USD" if ticker.get("base") in AbstractProvider.stableCoinTickers else ticker.get("base")
quote = "USD" if ticker.get("quote") in AbstractProvider.stableCoinTickers else ticker.get("quote")
payload = {
"quotePrice": "{:,.10f}".format(price[0]).rstrip('0').rstrip('.') + " " + quote,
"quoteVolume": "{:,.4f}".format(volume).rstrip('0').rstrip('.') + " " + base,
"title": ticker.get("name"),
"thumbnailUrl": coinThumbnail,
"messageColor": "amber" if priceChange == 0 else ("green" if priceChange > 0 else "red"),
"sourceText": "Data from Serum DEX",
"platform": "Serum",
"raw": {
"quotePrice": [price[0]],
"quoteVolume": [volume],
"timestamp": time()
}
}
return [payload, ""]
def __init__(self, timeout=-1):
self.controllerSocket = None
self.timecodeSocket = None
self.requestSocket = None
self.subscribers = {}
self.subscriberCallbacks = {}
self.subscriberTopicFilters = {}
self.periodicPublisher = None
self.commandCb = None
self.requestCb = None
self.timeout = timeout
self.poller = Poller()
self.running = False
self.pollCounter = 0
class Server(object):
def __init__(self, port):
self._ctx = Context()
self._socket = self._ctx.socket(DEALER)
self._socket.bind('tcp://*:%s' % port)
self._poller = Poller()
self._poller.register(self._socket, POLLIN)
self._clients = {}
self._actions = {'register': self._register_client}
def _register_client(self, message):
if message['pseudonym'] not in self._clients:
self._clients[message['pseudonym']] = message['cert']
message = {'type': 'Accepted'}
self._send_msg(message)
else:
message = {'type': 'Failed'}
self._send_msg(message)
def _send_msg(self, message):
pickled_message = dumps(message)
self._socket.send(pickled_message)
print('Sent --> ', message)
def _recv_msg(self):
pickled_message = self._socket.recv()
received_message = loads(pickled_message)
print("Received --> ", received_message)
return received_message
def execute(self):
while True:
sockets = dict(self._poller.poll())
if sockets.get(self._socket) == POLLIN:
received_message = self._recv_msg()
self._actions[received_message['type']](received_message)
def init_connections(self):
"""
Initialise zmq sockets.
Because this class is s Process, This method must be call in the run
method to be hold by the correct process.
"""
Connections.init_connections(self)
context = Context()
self.sock_for_me = context.socket(REQ)
self.sock_for_me.connect(self.sock_job_for_me_url)
# Use a ZMQ Pool to get multichannel message
self.poll = Poller()
# Register sockets
self.poll.register(self.sock_for_me, POLLIN)
# Send READY to next_router to inform about my capacity to compute new
# job
self.sock_for_me.send_pyobj("READY")
return True
def __init__(self, host, port):
self._socket = None
self._reqaddress = "tcp://" + host + ":" + str(port)
self._poller = Poller()
self._lock = Lock()
self.failures = 0
self.jammed = False
self.connect()
def __init__(self, host, port):
self._host = host
self._port = port
self._context = Context()
self._socket = self._context.socket(REQ)
self._socket.setsockopt(LINGER, 1)
self._socket.connect("tcp://"+host+":"+str(port))
self._poller = Poller()
self._poller.register(self._socket, POLLIN)
def _server(self, response):
""" Wait for a client request, record it and send the response. """
context = Context()
try:
socket = context.socket(REP)
try:
socket.set(LINGER, 0)
socket.bind("tcp://*:{}".format(_PORT))
poller = Poller()
poller.register(socket, POLLIN)
sockets = dict(poller.poll(_TIMEOUT))
if socket in sockets:
self._client_request = socket.recv_multipart()
if response:
socket.send_multipart(response)
finally:
socket.close()
finally:
context.destroy(linger=0)
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
def _server(self, response):
""" Wait for a client request, record it and send the response. """
context = Context()
try:
socket = context.socket(REP)
try:
socket.set(LINGER, 0)
socket.bind("tcp://*:{}".format(_PORT))
poller = Poller()
poller.register(socket, POLLIN)
sockets = dict(poller.poll(_TIMEOUT))
if socket in sockets:
self._client_request = socket.recv_multipart()
if response:
socket.send_multipart(response)
finally:
socket.close()
finally:
context.destroy(linger=0)
def execute_parser_request(endpoint, parameters, timeout=5):
socket = TickerParser.zmqContext.socket(REQ)
payload, responseText = None, None
socket.connect("tcp://parser:6900")
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
socket.send_multipart([endpoint] + parameters)
responses = poller.poll(timeout * 1000)
if len(responses) != 0:
response = socket.recv_multipart()
socket.close()
return response
else:
socket.close()
raise Exception("time out")
return None
def __init__(self, addresses, topics='', message_filter=None,
translate=False):
self._topics = self._magickfy_topics(topics)
self._filter = message_filter
self._translate = translate
self.sub_addr = {}
self.addr_sub = {}
self.poller = None
self._hooks = []
self._hooks_cb = {}
self.poller = Poller()
self._lock = Lock()
self.update(addresses)
self._loop = True
def __init__(self, context, req_address, pub_address):
self._context = context
self._poller = Poller()
self._nodes_sockets = {} # requests and service sockets
# FIFO request queue for nodes: [in_progress, data, callback(response)]
self._nodes_requests = defaultdict(list) # requests queues for nodes
self._cache = Cache() # the cache
node_id = uuid1().hex # unique node id, based on current time
_LOG.info("node id: %s", node_id)
self._nodes = Nodes(node_id, req_address, pub_address) # other nodes
def __init__(self, gui_port=None, statusBar=None):
"""
Parameters
----------
gui_port : str
port to connect for ZMQ communication
statusBar : QtGui.QStatusBar
MainWindow status bar to display information
"""
Thread.__init__(self)
QtCore.QObject.__init__(self)
if gui_port is not None:
self.context = Context()
# Socket to talk to pipeline kernel and pipeline steps and router
self.socket = self.context.socket(SUB)
gui_adress = "tcp://*:" + str(gui_port)
try:
self.socket.bind(gui_adress)
except ZMQError as e:
print("ERROR: ".format(str(e), gui_adress))
# Inform about connection in statusBar
if statusBar is not None:
self.statusBar = statusBar
self.statusBar.insertPermanentWidget(
0, QLabel("binded to " + gui_adress))
# Register socket in a poll and register topics
self.poll = Poller()
self.poll.register(self.socket, POLLIN)
self.socket.setsockopt_string(SUBSCRIBE, 'GUI_GRAPH')
self.socket.setsockopt_string(SUBSCRIBE, 'GUI_ROUTER_CHANGE')
self.socket.setsockopt_string(SUBSCRIBE, 'FINISH')
self.socket.setsockopt_string(SUBSCRIBE, 'MODE')
# self,stop flag is set by ficnish method to stop this thread
# properly when GUI is closed
# self.pipegui will receive new pipeline information
self.stop = False
self.steps = list()
self.config_time = 0
self.last_send_config = 0
self.nb_job_done = dict()
else:
self.stop = True
def start_poll(pull, push, sub, hash_checker):
global INPUT_DATA
seq_no = 0
poller = Poller()
poller.register(pull, zmq.POLLIN)
poller.register(sub, zmq.POLLIN)
while True:
try:
socks = dict(poller.poll())
except KeyboardInterrupt:
return
if sub in socks:
log.debug('recv -> {}'.format(sub.recv_multipart()))
try:
log.info('Concatting...')
concat_string = process()
concat_string_hash = HASHER(concat_string).digest()
# Send final hash
push.send_multipart([
MessageType.INPUT.value,
seq_no.to_bytes(2, byteorder='big'),
concat_string_hash,
])
log.debug('send -> {} | {} | {}', MessageType.INPUT.name,
seq_no, concat_string_hash.hex())
# Send original concatted string
push.send_multipart([
MessageType.COMMIT.value,
seq_no.to_bytes(2, byteorder='big'),
concat_string,
])
log.debug('send -> {} | {} | {}...', MessageType.COMMIT.name,
seq_no, concat_string[:20])
INPUT_DATA.clear()
log.info("Reset concat")
if seq_no == 0xFFFF:
seq_no = 0
else:
seq_no += 1
except Exception as e:
log.error("Could not concat inputs -> {}".format(e))
if pull in socks:
inp = pull.recv()
log.debug('recv -> {}'.format(inp.hex()))
try:
hash_checker.check(inp)
except:
log.warn('Bad hash received! -> {}', inp)
continue
INPUT_DATA.append(inp)
log.info('Input added | Elements: {}'.format(len(INPUT_DATA)))
def __init__(self):
super(Listener, self).__init__(name="Listener")
self._shutdown = False
self.context = Context()
self.sub = self.context.socket(SUB)
self.sub.bind('tcp://*:7000')
self.sub.setsockopt(SUBSCRIBE, "")
self.poller = Poller()
self.poller.register(self.sub, POLLIN)
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def __init__(self, holder, port, station):
Thread.__init__(self)
self._holder = holder
self._loop = True
self._port = port
self._station = station
self._lock = Lock()
self._socket = context.socket(REP)
self._socket.bind("tcp://*:" + str(self._port))
self._poller = Poller()
self._poller.register(self._socket, POLLIN)
def __init__(self, context, *args, **kw):
self.context = context
self.recv = self.context.socket(PAIR)
self.recv.connect("inproc://#1")
self.pub = self.context.socket(PUB)
self.pub.connect('tcp://localhost:7003')
self.pub.setsockopt(HWM, 1000)
self.poller = Poller()
self.poller.register(self.recv, POLLIN)
super(Stethoscope, self).__init__(*args, **kw)
def execute_database_request(endpoint, parameters, timeout=1):
socket = DatabaseConnector.zmqContext.socket(REQ)
payload, responseText = None, None
socket.connect("tcp://database:6900")
socket.setsockopt(LINGER, 0)
poller = Poller()
poller.register(socket, POLLIN)
socket.send_multipart([
endpoint,
bytes(str(int((time() + timeout) * 1000)), encoding='utf8'),
parameters
])
responses = poller.poll(timeout * 1000)
if len(responses) != 0:
[response] = socket.recv_multipart()
socket.close()
return loads(response)
else:
socket.close()
return None
def __init__(self):
super(Leatherneck, self).__init__(name="Leatherneck")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.connect("tcp://localhost:7000")
self.push = self.context.socket(PUSH)
self.push.connect("tcp://localhost:7001")
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
self._shutdown = False
for th in t_enum():
if th.name == "MainThread":
self.mainthread = th
def __init__(self, addresses, translate=False):
self._addresses = addresses
self._translate = translate
self.subscribers = []
self._poller = Poller()
for addr in self._addresses:
subscriber = context.socket(SUB)
subscriber.setsockopt(SUBSCRIBE, "pytroll")
subscriber.connect(addr)
self.subscribers.append(subscriber)
self._poller.register(subscriber)
self._lock = Lock()
self._loop = True
def directory_server(store: DirectoryServerStore, zmq_context: Context):
# pylint: disable=no-member # These zmq.ROUTER and zmq.PUB must be actually exists
print("Starting on libzmq {} with PyZMQ {}".format(zmq.zmq_version(),
zmq.pyzmq_version()))
entrypoint: Socket = zmq_context.socket(zmq.ROUTER)
entrypoint.bind("tcp://127.0.0.1:5350") # This is just a PROTOTYPE!
pub_file_changes: Socket = zmq_context.socket(zmq.PUB)
pub_file_changes.bind("tcp://127.0.0.1:5351")
poller = Poller()
poller.register(entrypoint, flags=zmq.POLLIN)
print(
"Directory server is started on 127.0.0.1:5350 (commands) and 127.0.0.1:5351 (file_changes_push)"
)
while True:
events: List[Tuple[Socket, int]] = poller.poll()
for socket, _ in events:
frames: List[Frame] = socket.recv_multipart(copy=False)
id_frame: Frame = frames.pop(0)
empty_frame: Frame = frames.pop(0)
assert (len(empty_frame.bytes) == 0)
command_frame: Frame = frames.pop(0)
command = str(command_frame.bytes, encoding='utf8')
if command == 'ping':
ping_handler(store, socket, frames, id_frame)
elif command == 'device.cast_address':
casting_address_handler(store, socket, frames, id_frame)
elif command == 'device.get_addresses':
get_addresses_handler(store, socket, frames, id_frame)
elif command == 'fs.list':
file_list_handler(store, socket, id_frame)
elif command == 'fs.declare':
file_declare_handler(store, socket, frames, id_frame,
pub_file_changes)
elif command == 'fs.disown':
file_disown_handler(store, socket, frames, id_frame,
pub_file_changes)
elif command == 'fs.get':
file_get_handler(store, socket, frames, id_frame)
def run(self, *args):
"""Run the listener and answer to requests.
"""
del args
arec = AddressReceiver(
max_age=self._max_age,
multicast_enabled=self._multicast_enabled,
restrict_to_localhost=self._restrict_to_localhost)
arec.start()
port = PORT
try:
with nslock:
self.listener = get_context().socket(REP)
self.listener.bind("tcp://*:" + str(port))
logger.debug('Listening on port %s', str(port))
poller = Poller()
poller.register(self.listener, POLLIN)
while self.loop:
with nslock:
socks = dict(poller.poll(1000))
if socks:
if socks.get(self.listener) == POLLIN:
msg = self.listener.recv_string()
else:
continue
logger.debug("Replying to request: " + str(msg))
msg = Message.decode(msg)
active_address = get_active_address(
msg.data["service"], arec)
self.listener.send_unicode(six.text_type(active_address))
except KeyboardInterrupt:
# Needed to stop the nameserver.
pass
finally:
arec.stop()
self.stop()
def _start_workers(context, sync_socket, count, timeout=None):
"""
:type context: zmq.Context
:type sync_socker: zmq.Socket
:type count: int
:param timeout: Timeout for waiting for worker messages, in milliseconds.
:type timeout: float
"""
_LOG.debug("Starting workers...")
worker_threads = [
Thread(
target=_handle_requests, name="worker {}".format(i),
args=(context, ))
for i in range(count)]
for thread in worker_threads:
thread.start()
_LOG.debug("Synchronizing workers...")
poller = Poller()
poller.register(sync_socket, POLLIN)
worker_identities = []
for _ in worker_threads:
sockets = dict(poller.poll(timeout=timeout))
if sync_socket in sockets:
worker_identities.append(sync_socket.recv_multipart()[0])
else:
raise RuntimeError("Worker did not respond in time.")
for worker_identity in worker_identities:
sync_socket.send_multipart([worker_identity, b"", b""])
for _ in worker_identities:
sockets = dict(poller.poll(timeout=timeout))
if sync_socket in sockets:
sync_socket.recv_multipart()
else:
raise RuntimeError("Worker did not respond in time.")
_LOG.debug("Workers synchronized.")
return worker_threads, worker_identities
class HomeBase(Thread):
def __init__(self):
super(HomeBase, self).__init__(name="HomeBase")
self.context = Context()
self.pull = self.context.socket(PULL)
self.pull.bind("tcp://*:7001")
self._shutdown = False
self.poller = Poller()
self.poller.register(self.pull, POLLIN)
def cleanup(self):
self.pull.close()
self.context.term()
def run(self):
while True:
socks = dict(self.poller.poll(timeout=1))
if socks.get(self.pull) == POLLIN:
msg = self.pull.recv()
msg += ", WORK RECEIVED "
print msg
if self._shutdown:
break
self.cleanup()
def __init__(self, holder, host, pubport, reqport, sched):
Thread.__init__(self)
self._holder = holder
self._pubaddress = "tcp://" + host + ":" + str(pubport)
self._reqaddress = "tcp://" + host + ":" + str(reqport)
self._req = SimpleRequester(host, reqport)
self._subsocket = context.socket(SUB)
self._subsocket.connect(self._pubaddress)
self._subsocket.setsockopt(SUBSCRIBE, "pytroll")
self._poller = Poller()
self._poller.register(self._subsocket, POLLIN)
self._lock = Lock()
self._loop = True
self._sched = sched
def init_connections(self):
"""
Initialise zmq sockets, poller and queues.
Because this class is s Process, This method must be call in the run
method to be hold by the correct process.
"""
# Prepare our context and sockets
context = Context()
# Socket to talk to prev_stages
for name,connections in self.connections.items():
self.queue_limit[name] = connections[2]
sock_router = context.socket(ROUTER)
try:
sock_router.bind('tcp://*:' + connections[0])
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'
.format(e, connections[0]))
return False
self.router_sockets[name] = sock_router
# Socket to talk to next_stages
sock_dealer = context.socket(ROUTER)
try:
sock_dealer.bind("tcp://*:" + connections[1] )
except ZMQError as e:
self.log.error('{} : tcp://localhost:{}'
.format(e, connections[1]))
return False
self.dealer_sockets[name] = sock_dealer
self.next_available_stages[name] = list()
self.queue_jobs[name] = list()
# Use a ZMQ Pool to get multichannel message
self.poller = Poller()
# Register dealer socket to next_stage
for n, dealer in self.dealer_sockets.items():
self.poller.register(dealer, POLLIN)
for n, router in self.router_sockets.items():
self.poller.register(router, POLLIN)
# Register router socket to prev_stages or producer
self.socket_pub = context.socket(PUB)
if self.gui_address is not None:
try:
self.socket_pub.connect("tcp://" + self.gui_address)
except ZMQError as e:
self.log.error("".format(e, self.gui_address))
return False
# This flag stop this current process
return True
def __init__(self, addresses, topics='', message_filter=None,
translate=False):
self._topics = self._magickfy_topics(topics)
self._filter = message_filter
self._translate = translate
self.sub_addr = {}
self.addr_sub = {}
self.poller = None
self._hooks = []
self._hooks_cb = {}
self.update(addresses)
self.poller = Poller()
self._loop = True
def listen(self, topics: Iterable[str] = ("log|", )):
"""
Listens to incoming log messages on the ZeroMQ socket(s).
Blocks until `Ctrl + C` is pressed,
or ``SIGINT`` or ``SIGTERM`` is received by the process.
Returns immediately if either a bind address nor at least
one connect address is set.
:param topics:
The topic to subscribe with.
Can be used for selecting log messages from specific source.
Defaults to ``["log|"]``.
:type topics: Iterable[str]
"""
if not self._bind_address and not self._connect_addresses:
return
poller = Poller()
if self._bind_address:
self._pull_socket = self._ctx.socket(PULL)
self._pull_socket.bind('tcp://' + self._bind_address)
poller.register(self._pull_socket, POLLIN)
if self._connect_addresses:
self._sub_socket = self._ctx.socket(SUB)
for topic in topics:
self._sub_socket.setsockopt(
SUBSCRIBE, topic.encode(encoding=_TOPIC_ENCODING))
for address in self._connect_addresses:
self._sub_socket.connect('tcp://' + address)
poller.register(self._sub_socket, POLLIN)
try:
while not is_stopped():
events = dict(poller.poll(self._interval * 1000))
if self._pull_socket in events \
and events[self._pull_socket] == POLLIN:
data = self._pull_socket.recv_multipart(flags=NOBLOCK)
self._handle_message(data)
if self._sub_socket in events \
and events[self._sub_socket] == POLLIN:
data = self._sub_socket.recv_multipart(flags=NOBLOCK)
self._handle_message(data)
finally:
if self._pull_socket:
self._pull_socket.close()
self._pull_socket = None
if self._sub_socket:
self._sub_socket.close()
self._sub_socket = None
def init_connexions(self):
"""
Initialise zmq sockets.
Because this class is s Process, This method must be call in the run
method to be hold by the correct processus.
"""
Connexions.init_connexions(self)
context = Context()
self.sock_for_me = context.socket(REQ)
self.sock_for_me.connect(self.sock_job_for_me_url)
# Use a ZMQ Pool to get multichannel message
self.poll = Poller()
# Register sockets
self.poll.register(self.sock_for_me, POLLIN)
# Send READY to next_router to inform about my capacity to compute new
# job
self.sock_for_me.send_pyobj("READY")
return True
def __init__(self, pipe, local_ep, local_uuid, config_svc):
Thread.__init__(self, name='dcamp.service.{}'.format(self))
self.ctx = Context.instance()
self.__control_pipe = pipe
assert isinstance(local_ep, EndpntSpec)
self.__endpoint = local_ep
assert isinstance(local_uuid, UUID)
self.__uuid = local_uuid
self.__cfgsvc = config_svc
self.logger = getLogger('dcamp.service.%s' % self)
self.poller = Poller()
self.poller_timer = None
self.poller.register(self.__control_pipe, POLLIN)
def __init__(self, impl=None):
self._impl = impl or Poller()
if hasattr(self._impl, 'fileno'):
set_close_exec(self._impl.fileno())
self._handlers = {}
self._events = {}
self._callbacks = []
self._callback_lock = threading.Lock()
self._timeouts = []
self._running = False
self._stopped = False
self._thread_ident = None
self._blocking_signal_threshold = None
# Create a pipe that we send bogus data to when we want to wake
# the I/O loop when it is idle
self._waker = Waker()
self.add_handler(self._waker.fileno(),
lambda fd, events: self._waker.consume(),
self.READ)
def start_poll(self, start_seq_no=None):
seq_no = start_seq_no if start_seq_no is not None else 0
last_process = time.time()
poller = Poller()
poller.register(self.pull, zmq.POLLIN)
poller.register(self.router, zmq.POLLIN)
while True:
try:
socks = dict(poller.poll(timeout=self.gather_time * 100))
except KeyboardInterrupt:
return
if self.pull in socks:
self.recv_input()
if self.router in socks:
self.recv_from_comp()
if self.worker_deque and self.ready_to_process() and time.time() - last_process >= self.gather_time:
compute_id = self.worker_deque.popleft()
last_process = time.time()
try:
self.process()
self.send_to_comp(compute_id, seq_no)
self.reset_inputs()
if seq_no == 0xFFFF:
seq_no = 0
else:
seq_no += 1
except Exception as e:
log.error("Could not process send and reset inputs -> {}".format(e))
else:
log.debug('time since last {}, deque size {}',
time.time() - last_process, len(self.worker_deque))
def __init__(self, timeout=-1, poolSize=4):
self.timeout = timeout
self.socket = None
self.numConnections = 0
self.poller = Poller()
def __init__(self):
self.pairSockets = []
self.poller = Poller()
class Listener:
def __init__(self, timeout=-1):
self.controllerSocket = None
self.timecodeSocket = None
self.requestSocket = None
self.subscribers = {}
self.subscriberCallbacks = {}
self.subscriberTopicFilters = {}
self.periodicPublisher = None
self.commandCb = None
self.requestCb = None
self.timeout = timeout
self.poller = Poller()
self.running = False
self.pollCounter = 0
def connectController(self, address, commandCb=None):
self.controllerSocket = zmq.Context().socket(zmq.PAIR)
self.controllerSocket.bind(address)
self.notifyConnected(address)
self.poller.register(self.controllerSocket, zmq.POLLIN)
self.commandCb = commandCb
def sendController(self, message):
wrappedMsg = IO.wrap(message)
self.controllerSocket.send(wrappedMsg)
def notifyConnected(self, address):
msg = {'status': 'connected', 'address': address}
self.sendController(msg)
def notifyReady(self, address):
msg = {'status': 'ready', 'address': address}
self.sendController(msg)
def listenToRequests(self, address, requestCb):
self.requestCb = requestCb
self.requestSocket = zmq.Context().socket(zmq.REP)
self.requestSocket.bind(address)
self.poller.register(self.requestSocket, zmq.POLLIN)
def subscribeTimecode(self, address, timecodeCb):
self.addSubscription(address,
timecodeCb,
label='_TC_',
filters=['TIMECODE'])
def addSubscription(self,
address,
subCb,
label='default',
filters=['default']):
if label not in self.subscribers:
self.subscribers[label] = zmq.Context().socket(zmq.SUB)
self.subscribers[label].set_hwm(1)
#self.subscribers[label].setsockopt(zmq.RCVHWM, 1)
self.subscriberCallbacks[label] = subCb
self.subscriberTopicFilters[label] = filters
self.poller.register(self.subscribers[label], zmq.POLLIN)
print "Subscribing", label, "to:", address
self.subscribers[label].connect(address)
if filters:
for filter in filters:
self.subscribers[label].setsockopt(zmq.SUBSCRIBE, filter)
else:
self.subscribers[label].setsockopt(zmq.SUBSCRIBE, '')
def setPeriodicPublisher(self, publisher, overrideTimeout=True):
self.periodicPublisher = publisher
if overrideTimeout: self.timeout = publisher.getTimeout()
def start(self):
self.running = True
self._main()
def stop(self):
self.running = False
def pollRegistered(self, expected=-1, timeout=-1):
if expected == -1: expected = len(self.poller.sockets)
self.pollCounter = expected
while self.pollCounter > 0:
self.poll(timeout)
def poll(self, timeout=-1):
if timeout == -1: timeout = self.timeout
socks = dict(self.poller.poll(timeout))
if self.controllerSocket in socks and socks[
self.controllerSocket] == zmq.POLLIN:
command = self.controllerSocket.recv()
command, _ = IO.unwrap(command)
self.pollCounter -= 1
# Allow the user of this module to react to this command and to modify its contents
if self.commandCb:
modifiedCommand = self.commandCb(command)
if modifiedCommand is not None: command = modifiedCommand
if 'action' not in command: return
action = command['action']
if action == 'exit':
self.running = False
if self.requestSocket in socks and socks[
self.requestSocket] == zmq.POLLIN:
self.pollCounter -= 1
message = self.requestSocket.recv()
message, _ = IO.unwrap(message)
replyMessage = self.requestCb(message)
if replyMessage is None: return
wrappedReply = IO.wrap(replyMessage)
self.requestSocket.send(wrappedReply)
if socks:
for label in self.subscribers.keys():
if self.subscribers[label] in socks and socks[
self.subscribers[label]] == zmq.POLLIN:
self.pollCounter -= 1
data = self.subscribers[label].recv()
# In case there is not space in the data, use find. Find returns -1 when the string isn't found
sepIndex = data.find(' ')
if sepIndex > -1 and not data[:4] == 'TIMS' and data[
0:sepIndex] in self.subscriberTopicFilters[label]:
topic, content = data[0:sepIndex], data[sepIndex + 1:]
else:
topic, content = '', data
content, _ = IO.unwrap(content)
if content is not None:
self.subscriberCallbacks[label](content)
if self.periodicPublisher:
self.periodicPublisher.publish()
def _main(self):
while self.running:
self.poll()
class PushRequester(object):
"""Base requester class.
"""
request_retries = 3
def __init__(self, host, port):
self._socket = None
self._reqaddress = "tcp://" + host + ":" + str(port)
self._poller = Poller()
self._lock = Lock()
self.failures = 0
self.jammed = False
self.running = True
self.connect()
def connect(self):
"""Connect to the server
"""
self._socket = context.socket(REQ)
self._socket.connect(self._reqaddress)
self._poller.register(self._socket, POLLIN)
def stop(self):
"""Close the connection to the server
"""
self.running = False
self._socket.setsockopt(LINGER, 0)
self._socket.close()
self._poller.unregister(self._socket)
def reset_connection(self):
"""Reset the socket
"""
self.stop()
self.connect()
def __del__(self, *args, **kwargs):
self.stop()
def send_and_recv(self, msg, timeout=DEFAULT_REQ_TIMEOUT):
with self._lock:
retries_left = self.request_retries
request = str(msg)
self._socket.send(request)
rep = None
small_timeout = 0.1
while retries_left and self.running:
now = time.time()
while time.time() < now + timeout:
if not self.running:
return rep
socks = dict(self._poller.poll(small_timeout))
if socks.get(self._socket) == POLLIN:
reply = self._socket.recv()
if not reply:
LOGGER.error("Empty reply!")
break
try:
rep = Message(rawstr=reply)
except MessageError as err:
LOGGER.error('Message error: %s', str(err))
break
LOGGER.debug("Receiving (REQ) %s", str(rep))
self.failures = 0
self.jammed = False
return rep
# During big file transfers, give some time to a friend.
time.sleep(0.1)
LOGGER.warning("Timeout from " + str(self._reqaddress) +
", retrying...")
# Socket is confused. Close and remove it.
self.stop()
retries_left -= 1
if retries_left <= 0:
LOGGER.error("Server doesn't answer, abandoning... " +
str(self._reqaddress))
self.connect()
self.failures += 1
if self.failures == 5:
LOGGER.critical("Server jammed ? %s", self._reqaddress)
self.jammed = True
break
LOGGER.info("Reconnecting and resending " + str(msg))
# Create new connection
self.connect()
self._socket.send(request)
return rep
class RequestManager(Thread):
"""Manage requests.
"""
def __init__(self, port, attrs=None):
Thread.__init__(self)
self._loop = True
self.out_socket = get_context().socket(ROUTER)
self.out_socket.bind("tcp://*:" + str(port))
self.port = port
self.in_socket = get_context().socket(PULL)
self.in_socket.bind("inproc://replies" + str(port))
self._poller = Poller()
self._poller.register(self.out_socket, POLLIN)
self._poller.register(self.in_socket, POLLIN)
self._attrs = attrs
try:
# Checking the validity of the file pattern
_pattern = globify(attrs["origin"])
except ValueError as err:
raise ConfigError('Invalid file pattern: ' + str(err))
except KeyError:
if 'listen' not in attrs:
raise
self._deleter = Deleter()
try:
self._station = self._attrs["station"]
except (KeyError, TypeError):
LOGGER.warning("Station is not defined in config file")
self._station = "unknown"
LOGGER.debug("Station is '%s'" % self._station)
def start(self):
self._deleter.start()
Thread.start(self)
def pong(self, message):
"""Reply to ping
"""
return Message(message.subject, "pong", {"station": self._station})
def push(self, message):
"""Reply to push request
"""
for the_dict in gen_dict_contains(message.data, 'uri'):
uri = urlparse(the_dict['uri'])
rel_path = the_dict.get('path', '')
pathname = uri.path
# FIXME: check against file_cache
if 'origin' in self._attrs and not fnmatch.fnmatch(
os.path.basename(pathname),
os.path.basename(globify(self._attrs["origin"]))):
LOGGER.warning('Client trying to get invalid file: %s',
pathname)
return Message(message.subject,
"err",
data="{0:s} not reachable".format(pathname))
try:
move_it(pathname,
message.data['destination'],
self._attrs,
rel_path=rel_path)
except Exception as err:
return Message(message.subject, "err", data=str(err))
else:
if (self._attrs.get('compression')
or self._attrs.get('delete', 'False').lower()
in ["1", "yes", "true", "on"]):
self._deleter.add(pathname)
if 'dataset' in message.data:
mtype = 'dataset'
elif 'collection' in message.data:
mtype = 'collection'
elif 'uid' in message.data:
mtype = 'file'
else:
raise KeyError('No known metadata in message.')
new_msg = Message(message.subject, mtype, data=message.data.copy())
new_msg.data['destination'] = clean_url(new_msg.data['destination'])
return new_msg
def ack(self, message):
"""Reply with ack to a publication
"""
for url in gen_dict_extract(message.data, 'uri'):
uri = urlparse(url)
pathname = uri.path
if 'origin' in self._attrs and not fnmatch.fnmatch(
os.path.basename(pathname),
os.path.basename(globify(self._attrs["origin"]))):
LOGGER.warning('Client trying to get invalid file: %s',
pathname)
return Message(message.subject,
"err",
data="{0:s} not reacheable".format(pathname))
if (self._attrs.get('compression') or self._attrs.get(
'delete', 'False').lower() in ["1", "yes", "true", "on"]):
self._deleter.add(pathname)
new_msg = Message(message.subject, "ack", data=message.data.copy())
try:
new_msg.data['destination'] = clean_url(
new_msg.data['destination'])
except KeyError:
pass
return new_msg
def info(self, message):
topic = message.subject
max_count = 2256 # Let's set a (close to arbitrary) limit on messages size.
try:
max_count = min(message.data.get("max_count", max_count),
max_count)
except AttributeError:
pass
uptime = datetime.datetime.utcnow() - START_TIME
files = []
with file_cache_lock:
for i in file_cache:
if i.startswith(topic):
files.append(i)
if len(files) == max_count:
break
return Message(message.subject,
"info",
data={
"files": files,
"max_count": max_count,
"uptime": str(uptime)
})
def unknown(self, message):
"""Reply to any unknown request.
"""
return Message(message.subject, "unknown")
def reply_and_send(self, fun, address, message):
in_socket = get_context().socket(PUSH)
in_socket.connect("inproc://replies" + str(self.port))
reply = Message(message.subject, "error")
try:
reply = fun(message)
except Exception:
LOGGER.exception(
"Something went wrong"
" when processing the request: %s", str(message))
finally:
LOGGER.debug("Response: " + str(reply))
try:
in_socket.send_multipart([address, b'', str(reply)])
except TypeError:
in_socket.send_multipart(
[address, b'', bytes(str(reply), 'utf-8')])
def run(self):
while self._loop:
try:
socks = dict(self._poller.poll(timeout=2000))
except ZMQError:
LOGGER.info("Poller interrupted.")
continue
if socks.get(self.out_socket) == POLLIN:
LOGGER.debug("Received a request")
address, empty, payload = self.out_socket.recv_multipart(
NOBLOCK)
message = Message(rawstr=payload)
fake_msg = Message(rawstr=str(message))
try:
urlparse(message.data['destination'])
except (KeyError, TypeError):
pass
else:
fake_msg.data['destination'] = clean_url(
message.data['destination'])
LOGGER.debug("processing request: " + str(fake_msg))
if message.type == "ping":
Thread(target=self.reply_and_send,
args=(self.pong, address, message)).start()
elif message.type == "push":
Thread(target=self.reply_and_send,
args=(self.push, address, message)).start()
elif message.type == "ack":
Thread(target=self.reply_and_send,
args=(self.ack, address, message)).start()
elif message.type == "info":
Thread(target=self.reply_and_send,
args=(self.info, address, message)).start()
else: # unknown request
Thread(target=self.reply_and_send,
args=(self.unknown, address, message)).start()
elif socks.get(self.in_socket) == POLLIN:
self.out_socket.send_multipart(
self.in_socket.recv_multipart(NOBLOCK))
else: # timeout
pass
def stop(self):
"""Stop the request manager."""
self._loop = False
self._deleter.stop()
self.out_socket.close(1)
self.in_socket.close(1)
#
# Weather Server
#
if __name__ == '__main__':
context = Context()
socket = context.socket(PUB)
socket.bind('tcp://*:6667')
# socket to receive control messages on
controller = context.socket(SUB)
controller.connect('tcp://localhost:6670')
controller.setsockopt(SUBSCRIBE, "")
# poller for handling receiver and controller messages
poller = Poller()
poller.register(socket, POLLIN)
poller.register(controller, POLLIN)
print 'Weather Update Server started ...'
while True:
socks = dict(poller.poll(timeout=1))
# exit the program if the kill command is executed
if socks.get(controller) == POLLIN:
controller.recv()
print 'Got kill command'
break
# if not killed, then tranmit the next weather update
zipcode = random.choice(zip_list)
# socket to receive messages on
receiver = context.socket(PULL)
receiver.connect("tcp://localhost:7779")
# socket to send messages to
sender = context.socket(PUSH)
sender.connect("tcp://localhost:7780")
# socket to receive control messages on
controller = context.socket(SUB)
controller.connect("tcp://localhost:6670")
controller.setsockopt(SUBSCRIBE, "")
# poler for handling receiver and controller messages
poller = Poller()
poller.register(receiver, POLLIN)
poller.register(controller, POLLIN)
print "Waiting for job ..."
# poller handling
while True:
socks = dict(poller.poll())
if socks.get(receiver) == POLLIN:
msg = receiver.recv()
# process task
workload_msec = int(msg)
time.sleep(workload_msec * 0.001)
def __init__(self):
self._poller = Poller()