def forwarder(frontend, backend):
"""Simple pub/sub forwarder
:param int frontend: fontend zeromq port
:param int backend: backend zeromq port
"""
try:
context = zmq.Context()
front_sub = context.socket(zmq.SUB)
front_sub.bind("tcp://*:%d" % frontend)
front_sub.setsockopt_string(zmq.SUBSCRIBE, "")
back_pub = context.socket(zmq.PUB)
back_pub.bind("tcp://*:%d" % backend)
print("forwarder started, backend on port : %d\tfrontend on port: %d" % (backend, frontend))
zmq.proxy(front_sub, back_pub)
except Exception as e:
print(e)
finally:
front_sub.close()
back_pub.close()
context.term()
async def start(self, handler):
async def _handler(message):
reply = await handler(message.data)
if reply:
await self.nats.publish(message.reply, reply)
async def _srv():
sock = self._ctx.socket(zmq.REP)
sock.connect("inproc://broker_%s" % id(self))
while True:
reply = None
try:
message = await sock.recv_pyobj()
reply = await handler(message)
finally:
await sock.send_pyobj(reply)
frontend = self.context.socket(zmq.ROUTER)
backend = self.context.socket(zmq.DEALER)
frontend.bind(self.bind_url)
backend.bind("inproc://broker_%s" % id(self))
zmq.proxy(frontend, backend)
for i in range(self.parallel_tasks):
self._awaits.append(_srv())
asyncio.gather(*self._awaits, loop=self.loop)
self.closed = False
def start(self):
# Start default writer and reader
CspZmqNode.start(self)
# Create sockets
xpub_out = self._context.socket(zmq.XPUB)
xsub_in = self._context.socket(zmq.XSUB)
xpub_out.bind('ipc:///tmp/zmqipcin')
xsub_in.bind('ipc:///tmp/zmqipcout')
s_mon = None
if self.monitor:
# Crate monitor socket
s_mon = self._context.socket(zmq.PUB)
s_mon.bind('tcp://*:{}'.format(self.mon_port_hub))
if self.console:
self.console_hub()
# Start ZMQ proxy (blocking)
try:
zmq.proxy(xsub_in, xpub_out, s_mon)
except KeyboardInterrupt as e:
print("Main:", e)
finally:
# print("Closing due to", e)
xsub_in.setsockopt(zmq.LINGER, 0)
xsub_in.close()
xpub_out.close()
if s_mon:
s_mon.close()
self.stop()
def proxy(url_worker, url_client, hwm=1000):
"""Server routine"""
try:
# pylint: disable=no-member
# Socket to talk to clients
clients = context.socket(zmq.ROUTER)
# number of *messages* in queue
clients.setsockopt(zmq.RCVHWM, hwm)
# clients.setsockopt(zmq.ZMQ_SWAP, swap)
clients.bind(url_client)
# Socket to talk to workers
worker = context.socket(zmq.DEALER)
worker.setsockopt(zmq.SNDHWM, hwm)
# workers.setsockopt(zmq.ZMQ_SWAP, swap)
worker.bind(url_worker)
zmq.proxy(clients, worker)
# We never get here but clean up anyhow
clients.close(linger=0)
workers.close(linger=0)
context.term()
except zmq.ZMQError as e:
# probably Address already in use
click.secho(f"broker: {e}", fg="red", bold=True, err=True)
# we're in a thread.. need this to exit
os._exit(1) # pylint: disable=protected-access
def __setupBroker(self):
frontendPort = config()["broker"]["frontendForwarder"]
backendPort = config()["broker"]["backendForwarder"]
try:
self.context = zmq.Context()
frontend = self.context.socket(zmq.XSUB)
frontend.bind(f"tcp://*:{frontendPort}")
backend = self.context.socket(zmq.XPUB)
backend.bind(f"tcp://*:{backendPort}")
print("Bound frontend and backend of Broker")
zmq.proxy(frontend, backend)
except Exception as e:
print(e)
finally:
pass
print("Shutting down broker")
frontend.close()
backend.close()
self.context.term()
def main_loop():
"""The main loop; sits and awaits for new connections."""
def start_workers():
"""Creates the worker threads and initiates them."""
from jomiel.subsys.broker.worker import worker_new
from threading import Thread
for worker_id in range(opts.broker_worker_threads):
worker_thread = Thread(
target=worker_new,
args=(worker_id + 1,),
)
worker_thread.start()
log("%d thread(s) active" % opts.broker_worker_threads)
start_workers()
try:
proxy(router, dealer)
except KeyboardInterrupt:
log("<sigint> signal interrupt")
except ZMQError as error:
log(error)
finally:
dealer.close()
router.close()
if auth:
auth.stop()
ctx.term()
log("shutdown")
def run(self):
try:
# Create a context
self.context = zmq.Context(1)
# Socket facing clients
self.frontend = self.context.socket(zmq.SUB)
self.frontend.bind("tcp://*:%d" % (ZMQ_SUBSCRIBE))
self.frontend.setsockopt_string(zmq.SUBSCRIBE, "")
# Socket facing services
self.backend = self.context.socket(zmq.PUB)
self.backend.bind("tcp://*:%d" % (ZMQ_PUBLISH))
# Start the proxy
try:
zmq.proxy(self.frontend, self.backend)
except Exception as e:
pass
except Exception as e:
raise Exception("\nException in Forwarder - terminating [%s][%s]" % (str(e), traceback.format_exc()))
finally:
if self.frontend != None: self.frontend.close()
if self.backend != None: self.backend.close()
def proxy_thread():
context = zmq.Context()
xpub = context.socket(zmq.XPUB)
xsub = context.socket(zmq.XSUB)
xsub.bind('tcp://*:{}'.format(self._port))
xpub.bind('tcp://*:{}'.format(self._port + 1))
zmq.proxy(xsub, xpub)
def run(self):
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.SUB)
self.pub_port.value = frontend.bind_to_random_port("tcp://*")
frontend.setsockopt(zmq.SUBSCRIBE, "")
# Socket facing services
backend = context.socket(zmq.PUB)
self.sub_port.value = backend.bind_to_random_port("tcp://*")
LOGGER.info("EventDevice Listen on pub_port=%d, sub_port=%d", self.pub_port.value, self.sub_port.value)
backend.setsockopt(zmq.LINGER, 0)
frontend.setsockopt(zmq.LINGER, 0)
self.ready_event.set()
zmq.proxy(frontend, backend)
except Exception as e:
LOGGER.exception("zmq device failed")
except (KeyboardInterrupt, SystemExit) as ex:
LOGGER.debug("EventsDevice was halted by %s", ex.__class__.__name__)
finally:
frontend.close()
backend.close()
context.term()
def send_batch_device():
ctx = zmq.Context()
frontend = ctx.socket(zmq.ROUTER)
frontend.bind("tcp://*:51003")
backend = ctx.socket(zmq.DEALER)
backend.bind("ipc:///tmp/5103.ipc")
zmq.proxy(frontend, backend)
def main(argv):
parser = argparse.ArgumentParser(description='Arguments of python worker')
parser.add_argument("--threads",
type=int,
default=1,
help='Number of threads for this worker.')
args = parser.parse_args()
workerInternalIdentifier = str(uuid.uuid4())
worker_url = "ipc:///tmp/workers" + workerInternalIdentifier + ".ipc"
# Start threads
for i in range(args.threads):
thread = threading.Thread(target=worker, args=(worker_url, ))
thread.start()
print("Thread %d created" % (i))
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
frontend.connect("tcp://router:5558")
backend = context.socket(zmq.DEALER)
backend.bind(worker_url)
zmq.proxy(frontend, backend)
frontend.close()
backend.close()
context.term()
def start(self, pool_size=2):
worker_port = 55999
server_port = self._port
if (sys.platform.lower().find("linux") >= 0):
setproctitle.setproctitle("server-server")
url_worker = "tcp://127.0.0.1:{0}".format(worker_port)
url_client = "tcp://*:{0}".format(server_port)
clients = self._context.socket(zmq.ROUTER)
try:
clients.bind(url_client)
except:
lib.debug.info (sys.exc_info())
self._context.term()
sys.exit(1)
# Socket to talk to workers
workers = self._context.socket(zmq.DEALER)
try:
workers.bind(url_worker)
except:
lib.debug.info (sys.exc_info())
self._context.term()
sys.exit(1)
# Launch pool of worker process
p = multiprocessing.Pool(processes=pool_size, initializer=self._worker, initargs=(url_worker,))
zmq.proxy(clients, workers)
# We never get here but clean up anyhow
clients.close()
workers.close()
self._context.term()
def run(self):
try:
zmq.proxy(self.xpub, self.xsub) # This blocks
except KeyboardInterrupt:
pass
finally:
ctx.destroy()
def run(self):
ZCONTEXT = zmq.Context()
xsub = ZCONTEXT.socket(zmq.XSUB)
xsub.bind(ZSUB)
xpub = ZCONTEXT.socket(zmq.XPUB)
xpub.bind(ZPUB)
zmq.proxy(xsub, xpub)
def main(ip, port):
logger = logging.getLogger('Drive-through')
logger.info('Setup ZMQ')
context = zmq.Context()
restaurant = DriveThrough()
# frontend for clients (socket type Router)
frontend = context.socket(zmq.ROUTER)
frontend.bind("tcp://{}:{}".format(ip, port))
# backend for workers (socket type Dealer)
backend = context.socket(zmq.DEALER)
backend.bind("inproc://backend")
# Setup workers
workers = []
for i in range(4):
# each worker is a different thread
worker = Worker(context, i, "inproc://backend", restaurant)
worker.start()
workers.append(worker)
# Setup proxy
# This device (already implemented in ZMQ) connects the backend with the frontend
zmq.proxy(frontend, backend)
# join all the workers
for w in workers:
w.join()
# close all the connections
frontend.close()
backend.close()
context.term()
def runPythonServer(port, handler, threads=1, args=()):
"""Server routine"""
url_worker = "inproc://workers"
url_client = "tcp://127.0.0.1:%s"%port
# Prepare our context and sockets
context = zmq.Context.instance()
# Socket to talk to clients
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
# Socket to talk to workers
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
# Launch pool of worker threads
for i in range(threads):
thread = threading.Thread(target=worker_routine, args=(url_worker,handler,args))
thread.start()
eprint("Spawned %d worker threads for server bound to %s"%(threads, url_client))
zmq.proxy(clients, workers)
# We never get here but clean up anyhow
clients.close()
workers.close()
context.term()
def proxy_forwarder(frontend_addr, backend_addr, capture_addr):
context = zmq.Context()
# facing publishers
#frontend = context.socket(zmq.XSUB)
frontend = context.socket(zmq.SUB)
frontend.setsockopt(zmq.SUBSCRIBE, b'')
frontend.connect(frontend_addr)
# facing services (sinks/subsribers)
backend = context.socket(zmq.XPUB)
backend.bind(backend_addr)
# infrom publishers of a new sink
#backend.setsockopt(ZMQ_XPUB_VERBOSE, 1)
logger.info(f"zmq pubsub proxy: {frontend_addr} -> {backend_addr}")
if capture_addr:
capture = context.socket(zmq.PUB)
capture.bind(capture_addr)
logger.info(f"zmq capture: {capture_addr}")
zmq.proxy(frontend, backend, capture)
else:
zmq.proxy(frontend, backend)
# we never get here
frontend.close()
backend.close()
if capture:
capture.close()
context.close()
def main():
context = zmq.Context()
xsub_socket = context.socket(zmq.XSUB)
xsub_socket.bind('tcp://*:6000')
xpub_socket = context.socket(zmq.XPUB)
xpub_socket.bind('tcp://*:6001')
poller = zmq.Poller()
poller.register(xpub_socket, zmq.POLLIN)
poller.register(xsub_socket, zmq.POLLIN)
if True:
while True:
#print('listen..')
try:
events = dict(poller.poll(1000))
except KeyboardInterrupt:
print('KeyboardInterrupt - send quit message')
#xpub_socket.send_multipart([b'quit'])
break
if xpub_socket in events:
message = xpub_socket.recv_multipart()
#print("[BROKER] subscription message: %r" % message[0])
xsub_socket.send_multipart(message)
if xsub_socket in events:
message = xsub_socket.recv_multipart()
#print("publishing message: %r" % message)
xpub_socket.send_multipart(message)
else:
pub = ctx.socket(zmq.PUB)
pub.bind('tcp://*:6002')
zmq.proxy(xpub_socket, xsub_socket, pub)
def main(port, data, threading_number):
"""Server routine"""
database = LevelDB(data or DATA)
url_worker = 'inproc://workers'
url_client = 'tcp://*:{}'.format(port or PORT)
# Prepare our context and sockets
context = zmq.Context.instance()
# Socket to talk to clients
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
# Socket to talk to workers
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
# Launch pool of worker threads
for _ in range(threading_number or THREADING_NUMBER):
thread = threading.Thread(target=worker_routine,
args=(url_worker, database))
thread.start()
zmq.proxy(clients, workers)
# We never get here but clean up anyhow
clients.close()
workers.close()
context.term()
def main():
context = zmq.Context.instance()
# socket facing clients
frontend = context.socket(zmq.XSUB)
frontend.bind("tcp://*:5559")
# socket facing services
backend = context.socket(zmq.XPUB)
backend.bind("tcp://*:5560")
# log socket
log_out = context.socket(zmq.PAIR)
log_out.bind("inproc://logging")
# start logging thread
stop_logging = threading.Event()
logger = Logger(stop_logging)
logger.start()
try:
zmq.proxy(frontend, backend, log_out)
except KeyboardInterrupt:
print("shutting down")
finally:
frontend.close()
backend.close()
stop_logging.set()
logger.join()
def run(self):
# setup zmq frontend and backend sockets
# create a shared ZMQ context
self.logger.info('Setup ZMQ')
context = zmq.Context()
# frontend for clients (socket type Router)
frontend = context.socket(zmq.ROUTER)
frontend.bind(self.frontend_addr)
# backend for workers (socket type Dealer)
backend = context.socket(zmq.DEALER)
backend.bind(self.backend_addr)
# Setup workers
workers = []
for i in range(self.n_workers):
# each worker is a different thread
worker = Worker(context, i, self.backend_addr)
worker.start()
workers.append(worker)
# Setup proxy
# This device (already implemented in ZMQ) connects the backend with the frontend
zmq.proxy(frontend, backend)
# join all the workers
for w in workers:
w.join()
# close all the connections
frontend.close()
backend.close()
context.term()
def run(self):
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
port = self.demandIntPort()
scheme = self.config.get('scheme', 'tcp')
endpoint = '%s://*:%s' % (scheme, str(port))
sys.stdout.write('endpoint: "%s" noisy=%s\n' % (endpoint, self.is_noisy))
frontend.bind(endpoint)
backend = context.socket(zmq.DEALER)
backend.bind('inproc://backend')
self.config['context'] = context
# Spawn some worker threads
for _ in range(5):
worker = ServerWorker(self.config)
worker.start()
self.workers.append(worker)
zmq.proxy(frontend, backend)
frontend.close()
backend.close()
context.term()
def main():
# Mail loop of the load balancer
# Context and sockets
context = zmq.Context.instance()
frontend = context.socket(zmq.PULL)
frontend.bind("tcp://127.0.0.1:5557")
backend = context.socket(zmq.PUSH)
backend.bind("tcp://127.0.0.1:5558")
# Start background tasks
def start(task, *args):
process = multiprocessing.Process(target=task, args=args)
process.daemon = True
process.start()
for i in range(N_DETECTORS):
start(detector_task, i)
for i in range(N_WORKERS):
start(worker_task, i)
start(collector_task)
# Use zmq.proxy() to handle load balancing
while True:
zmq.proxy(
frontend, backend
) #very powerful and convenient if no specific ending conditions needed, otherhwise should do it manually with zmq.POLLIN etc.
# Clean up
backend.close()
frontend.close()
sink.close()
context.term()
def main(ip, port):
logger = logging.getLogger('Drive-through')
logger.info('Setup ZMQ')
context = zmq.Context()
restaurant = DriveThrough()
frontend = context.socket(zmq.ROUTER)
frontend.bind("tcp://{}:{}".format(ip, port))
backend = context.socket(zmq.DEALER)
backend.bind("inproc://backend")
workers = []
for i in range(5):
worker = Worker(context, i, "inproc://backend", restaurant)
worker.start()
workers.append(worker)
zmq.proxy(frontend, backend)
for w in workers:
w.join()
frontend.close()
backend.close()
context.term()
def run(port):
""" Run a translations server at a specific port.
It always listens on all available network devices!
"""
context = Context(1)
sync_socket = context.socket(ROUTER)
sync_socket.bind(_SYNC_ENDPOINT)
frontend = context.socket(ROUTER)
frontend.bind("tcp://*:{}".format(port))
# Socket facing services
backend = context.socket(DEALER)
backend.bind(_REQUEST_ENDPOINT)
try:
worker_threads, worker_identities = _start_workers(
context, sync_socket, config.WORKERS, 1000)
_LOG.debug("Running device...")
try:
proxy(frontend, backend)
except KeyboardInterrupt:
print("\rShutting down...")
frontend.close()
frontend = None
_shut_down_workers(sync_socket, worker_threads, worker_identities, 5)
finally:
if frontend is not None:
frontend.close()
backend.close()
sync_socket.close()
_LOG.debug("Done")
def incoming_proxy(ctx):
with ctx.socket(zmq.PULL) as local, ctx.socket(zmq.PUSH) as remote:
# Block sends until connection completes
remote.set(zmq.IMMEDIATE, True)
connect(local, local_outgoing_port+1, remote, remote_outgoing_port+1)
zmq.proxy(local, remote)
def run(self):
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
port = self.demandIntPort()
scheme = self.config.get('scheme', 'tcp')
endpoint = '%s://*:%s' % (scheme, str(port))
sys.stdout.write('endpoint: "%s" noisy=%s\n' %
(endpoint, self.is_noisy))
frontend.bind(endpoint)
backend = context.socket(zmq.DEALER)
backend.bind('inproc://backend')
self.config['context'] = context
# Spawn some worker threads
for _ in range(5):
worker = ServerWorker(self.config)
worker.start()
self.workers.append(worker)
zmq.proxy(frontend, backend)
frontend.close()
backend.close()
context.term()
def run(self):
self._logger.info("Async Controller Server thread {0} started".format(
self.name))
try:
workers = []
for _ in range(MAX_CONTROLLER_INCOMING_WORKERS):
worker = IncomingAsyncControllerWorker(self._logger,
self._context,
self._incoming_queue,
self._stop_event)
workers.append(worker)
worker.start()
for _ in range(MAX_CONTROLLER_OUTGOING_WORKERS):
worker = OutgoingAsyncControllerWorker(self._logger,
self._context,
self._outgoing_queue,
self._stop_event)
workers.append(worker)
worker.start()
self._logger.info("Starting Proxy Device...")
zmq.proxy(self._frontend, self._backend)
except zmq.ZMQError as zmq_error:
self._logger.exception(zmq_error)
self._stop_event.set()
raise zmq_error
except Exception as generic_error:
self._logger.exception(
"Unhandled exception {0}".format(generic_error))
self._stop_event.set()
raise generic_error
finally:
self._logger.info("Closing sockets...")
self._context.close()
self._backend.close()
self._context.term()
def main():
logging.basicConfig(format='%(asctime)s: (%(levelname)s) %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S%Z',
level=logging.INFO)
url_worker = "tcp://*:22222"
url_client = "tcp://*:29999"
# Prepare our context and sockets
context = zmq.Context.instance()
# Socket to talk to clients
logging.info('Listening for client requests on %s', url_client)
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
# Socket to talk to workers
logging.info('Listening for worker offers on %s', url_worker)
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
# Proxy requests
logging.info('Now proxying requests')
zmq.proxy(clients, workers)
# We never get here but clean up anyhow
clients.close()
workers.close()
context.term()
def run(self):
ZCONTEXT = zmq.Context()
xsub = ZCONTEXT.socket(zmq.XSUB)
xsub.bind(ZSUB)
xpub = ZCONTEXT.socket(zmq.XPUB)
xpub.bind(ZPUB)
zmq.proxy(xsub, xpub)
def event_router(_, pidx, args):
# run as extra_procs by aiotools
ctx = zmq.Context()
ctx.linger = 50
in_sock = ctx.socket(zmq.PULL)
in_sock.bind("tcp://{0.host}:{0.port}".format(
args[0]['manager']['event-listen-addr']))
out_sock = ctx.socket(zmq.PUSH)
ipc_base_path.mkdir(parents=True, exist_ok=True)
try:
out_sock.bind(EVENT_IPC_ADDR)
zmq.proxy(in_sock, out_sock)
except zmq.error.ZMQError:
log.error('Cannot bind the event router socket to {}', EVENT_IPC_ADDR)
except (KeyboardInterrupt, SystemExit):
pass
except:
log.exception('unexpected error')
# raven.captureException()
finally:
in_sock.close()
out_sock.close()
ctx.term()
if ipc_events_sockpath.exists():
ipc_events_sockpath.unlink()
def _server_proxy_worker(self, zmq_context, ports):
assert len(ports) == 2
frontend = zmq_context.socket(zmq.PULL)
frontend.bind("tcp://*:%s" % ports[0])
backend = self._zmq_context.socket(zmq.PUSH)
backend.bind("tcp://*:%s" % ports[1])
zmq.proxy(frontend, backend)
def run_broker(frontend: str, backend: str) -> None:
with zmq.Context() as context:
with context.socket(zmq.ROUTER) as router:
router.bind("tcp://*:5559")
with context.socket(zmq.DEALER) as dealer:
dealer.bind("tcp://*:5560")
zmq.proxy(router, dealer)
def main():
"""Server routine"""
url_worker = "inproc://workers"
url_client = "tcp://*:5555"
# Prepare our context and sockets
context = zmq.Context.instance()
# Socket to talk to clients
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
# Socket to talk to workers
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
# Launch pool of worker threads
for i in range(5):
thread = threading.Thread(target=worker_routine, args=(url_worker,))
thread.daemon = True
thread.start()
zmq.proxy(clients, workers)
# We never get here but clean up anyhow
clients.close()
workers.close()
context.term()
def run(self):
try:
Logger.info(FeedDaemon, "Starting Feed Daemon...")
# Socket to talk to clients
clients = self.context.socket(zmq.ROUTER)
clients.bind(self.addr)
# Socket to talk to workers
workers = self.context.socket(zmq.DEALER)
workers.bind("inproc://workers.inproc")
# Launch pool of worker threads
for i in range(self.n_workers):
thread = threading.Thread(target=self.worker, args=())
thread.start()
Logger.info(FeedDaemon.run,
"Feed Daemon running. Serving on %s" % self.addr)
zmq.proxy(clients, workers)
except KeyboardInterrupt:
clients.close()
workers.close()
self.context.term()
def run(self):
context = zmq.Context()
try:
# Socket facing clients
frontend = context.socket(zmq.XREP)
frontend.bind("tcp://{host}:{port}".format(
host=self.frontend_host, port=self.frontend_port))
except Exception as err:
self.reporter.report(
'ERROR',
'Frontend failed to bind to tcp://{host}:{port}\n[{e}]'.format(
host=self.frontend_host, port=self.frontend_port, e=err))
try:
# Socket facing servers
backend = context.socket(zmq.XREQ)
backend.bind("tcp://{host}:{port}".format(host=self.backend_host,
port=self.backend_port))
except Exception as e:
self.reporter.report(
'ERROR',
'Backend failed to bind to tcp://{host}:{port}\n[{e}'.format(
host=self.backend_host, port=self.backend_port, e=err))
try:
zmq.proxy(frontend, backend)
except Exception as err:
self.reporter.report(
'ERROR', 'Proxy failed to initialize.\n[{e}'.format(e=err))
def main():
"""Server routine"""
url_worker = "inproc://workers"
url_client = "tcp://*:5555"
# Prepare our context and sockets
context = zmq.Context.instance()
# Socket to talk to clients
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
# Socket to talk to workers
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
# Launch pool of worker threads
for i in range(5):
thread = threading.Thread(target=worker_routine, args=(url_worker, ))
thread.start()
zmq.proxy(clients, workers)
# We never get here but clean up anyhow
clients.close()
workers.close()
context.term()
def serve_proxy(options):
if not (options.front_connect or options.front_bind):
print("No frontend socket address specified!", file=sys.stderr)
sys.exit(1)
if not (options.back_connect or options.back_bind):
print("No backend socket address specified!", file=sys.stderr)
sys.exit(1)
ctx = zmq.Context().instance()
front_type, back_type = options.sock_types
front = ctx.socket(front_type)
back = ctx.socket(back_type)
if options.monitor_bind or options.monitor_connect:
monitor = ctx.socket(zmq.PUB)
bind_connect(monitor, options.monitor_bind, options.monitor_connect)
else:
monitor = None
bind_connect(front, options.front_bind, options.front_connect)
bind_connect(back, options.back_bind, options.back_connect)
try:
if monitor:
zmq.proxy(front, back, monitor)
else:
zmq.proxy(front, back)
except:
return
finally:
front.close()
back.close()
def run(self):
# pylint: disable=no-member; disable `no-member' messages because of zmq.
context = frontend = backend = None
try:
context = zmq.Context(1)
# Socket facing clients.
frontend = context.socket(zmq.SUB)
self._pub_port.value = frontend.bind_to_random_port("tcp://*")
frontend.setsockopt(zmq.SUBSCRIBE, b"")
frontend.setsockopt(zmq.LINGER, 0)
# Socket facing services.
backend = context.socket(zmq.PUB)
self._sub_port.value = backend.bind_to_random_port("tcp://*")
backend.setsockopt(zmq.LINGER, 0)
self._ready.set()
LOGGER.info("EventsDevice listen on pub_port=%d, sub_port=%d", self._pub_port.value, self._sub_port.value)
zmq.proxy(frontend, backend)
except Exception: # pylint: disable=broad-except
LOGGER.exception("zmq device failed")
except (KeyboardInterrupt, SystemExit) as ex:
LOGGER.debug("EventsDevice was halted by %s", ex.__class__.__name__)
finally:
if frontend:
frontend.close()
if backend:
backend.close()
if context:
context.term()
def _run(config: typing.Dict):
"""
This is the target function that will run as its own process.
:param config: OPQ Mauka config file
:return: The process object.
"""
import logging
import signal
import os
import zmq
_logger = logging.getLogger("app")
logging.basicConfig(
format=
"[%(levelname)s][%(asctime)s][{} %(filename)s:%(lineno)s - %(funcName)s() ] %(message)s"
.format(os.getpid()))
signal.signal(signal.SIGINT, signal.SIG_IGN)
zmq_pub_interface = config["zmq.mauka.broker.pub.interface"]
zmq_sub_interface = config["zmq.mauka.broker.sub.interface"]
zmq_context = zmq.Context()
zmq_pub_socket = zmq_context.socket(zmq.PUB)
zmq_sub_socket = zmq_context.socket(zmq.SUB)
zmq_pub_socket.bind(zmq_pub_interface)
zmq_sub_socket.bind(zmq_sub_interface)
zmq_sub_socket.setsockopt(zmq.SUBSCRIBE, b"")
_logger.info("Starting Mauka pub/sub broker")
zmq.proxy(zmq_sub_socket, zmq_pub_socket)
_logger.info("Exiting Mauka pub/sub broker")
def run(self):
context = self.ctx
pub_public, pub_secret = zmq.auth.load_certificate(self.kp["be_key"])
sub_public, sub_secret = zmq.auth.load_certificate(self.kp["app_key"])
TCP_PXY_RTR = f"tcp://{self.pxy_ip}:{self.proxy_base_port+0}"
TCP_BE_DLR = f"tcp://{self.be_ip}:{self.be_base_port+0}"
frontend = context.socket(zmq.ROUTER)
frontend.curve_publickey = pub_public
frontend.curve_secretkey = pub_secret
frontend.curve_server = True
tprint(f"FAC--PXY: RouterProxy binding as Router to {TCP_PXY_RTR}")
frontend.bind(TCP_PXY_RTR)
backend = context.socket(zmq.DEALER)
backend.curve_publickey = sub_public
backend.curve_secretkey = sub_secret
backend.curve_serverkey = pub_public
tprint(f"PXY--BE: RouterProxy connecting as Dealer to {TCP_BE_DLR}")
backend.connect(TCP_BE_DLR)
try:
zmq.proxy(frontend, backend) # blocks
except zmq.error.ContextTerminated:
print("RouterProxy shutting down...")
frontend.close()
backend.close()
def run(self):
context = self.ctx
pub_public, pub_secret = zmq.auth.load_certificate(self.kp["be_key"])
sub_public, sub_secret = zmq.auth.load_certificate(self.kp["app_key"])
TCP_PXY_PUB = f"tcp://{self.pxy_ip}:{self.proxy_base_port+1}"
TCP_BE_SUB = f"tcp://{self.be_ip}:{self.be_base_port+1}"
frontend = context.socket(zmq.PUB)
frontend.curve_publickey = pub_public
frontend.curve_secretkey = pub_secret
frontend.curve_server = True
tprint(f"FAC--PXY: PubProxy binding as Publisher to {TCP_PXY_PUB}")
frontend.bind(TCP_PXY_PUB)
backend = context.socket(zmq.SUB)
backend.curve_publickey = sub_public
backend.curve_secretkey = sub_secret
backend.curve_serverkey = pub_public
backend.setsockopt(zmq.SUBSCRIBE, b"")
tprint(f"PXY--BE: PubProxy connecting as Subscriber to {TCP_BE_SUB}")
backend.connect(TCP_BE_SUB)
try:
zmq.proxy(frontend, backend) # blocks
except zmq.error.ContextTerminated:
print("PubProxy shutting down...")
frontend.close()
backend.close()
def main_proxy_info():
ctx = zmq.Context()
frontend = ctx.socket(zmq.XSUB)
for proxy in proxies:
queue = "tcp://localhost:" + proxy
frontend.connect(queue)
backend = ctx.socket(zmq.XPUB)
backend.bind("tcp://*:5550")
zmq.proxy(frontend,backend)
def run():
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
frontend.bind('tcp://*:10000')
backend = context.socket(zmq.DEALER)
# backend.bind('ipc://backend')
zmq.proxy(frontend, backend)
def proxy_zeromq(): ctx = zmq.asyncio.Context() sock_in = ctx.socket(zmq.PULL) sock_in.bind(url_in) sock_out = ctx.socket(zmq.PUSH) sock_out.bind(url_out) zmq.proxy(sock_in, sock_out)
def outgoing_proxy(ctx):
with ctx.socket(zmq.XPUB) as local, ctx.socket(zmq.XSUB) as remote:
# No limit on number of outstanding outgoing messages
local.set(zmq.SNDHWM, 0)
# No limit on number of outstanding incoming messages
remote.set(zmq.RCVHWM, 0)
connect(local, local_outgoing_port, remote, remote_outgoing_port)
zmq.proxy(remote, local)
def ipc_backbone(in_url, out_url):
ctx = zmq.Context.instance()
xsub = ctx.socket(zmq.XSUB)
xsub.bind(in_url)
xpub = ctx.socket(zmq.XPUB)
xpub.bind(out_url)
try:
zmq.proxy(xsub, xpub)
except zmq.ContextTerminated:
xsub.close()
xpub.close()
def main():
logger.info('Starting')
context = zmq.Context()
server_socket = context.socket(zmq.ROUTER)
server_socket.bind('tcp://*:{}'.format(config.SERVER_ZMQ_PORT))
services_socket = context.socket(zmq.DEALER)
services_socket.bind('tcp://*:{}'.format(config.SERVICES_ZMQ_PORT))
zmq.proxy(server_socket, services_socket)
def start(self):
context = zmq.Context()
subscriber = context.socket(zmq.SUB) # @UndefinedVariable
subscriber.setsockopt(zmq.SUBSCRIBE, b"") # @UndefinedVariable
subscriber.bind(self.frontend)
pub = context.socket(zmq.PUB) # @UndefinedVariable
pub.bind(self.backend)
zmq.proxy(subscriber, pub) # @UndefinedVariable
def proxy():
sub_core = context.socket(zmq.SUB)
for addr in sub_core_addrs:
sub_core.connect(addr)
sub_core.setsockopt(zmq.SUBSCRIBE, "")
pub_fringe = context.socket(zmq.PUB)
pub_fringe_addr = "tcp://*:5564"
pub_fringe.bind(pub_fringe_addr)
zmq.proxy(sub_core, pub_fringe)
def proxy(in_url, out_url):
ctx = zmq.Context.instance()
in_s = ctx.socket(zmq.PULL)
in_s.bind(in_url)
out_s = ctx.socket(zmq.PUSH)
out_s.bind(out_url)
try:
zmq.proxy(in_s, out_s)
except zmq.ContextTerminated:
print("proxy terminated")
in_s.close()
out_s.close()
def proxy_zeromq(): ctx = zmq.asyncio.Context() sock_in = ctx.socket(zmq.PULL) sock_in.bind(url_in) # while True: # msg = yield from sock_in.recv() # print(msg) sock_out = ctx.socket(zmq.PUB) sock_out.bind(url_out) zmq.proxy(sock_in, sock_out)
def sub_set():
# listen to messages the set is
# publishing on any/all channels
sub_set = context.socket(zmq.SUB)
for addr in set_addrs:
sub_set.connect( addr )
sub_set.setsockopt(zmq.SUBSCRIBE, "")
# publish those messages locally (in-process)
# to be demultiplexed and sent to clients
pub_inproc = context.socket(zmq.PUB)
pub_inproc_addr = "tcp://*:5555"
pub_inproc.bind(pub_inproc_addr)
zmq.proxy(sub_set, pub_inproc)
def busThreadFunction(self):
print("Bus thread created")
publish = self.context.socket(zmq.XPUB)
publish.bind("tcp://127.0.0.1:5556")
subscribe = self.context.socket(zmq.XSUB)
subscribe.bind("tcp://127.0.0.1:5555")
# Start proxy
try:
zmq.proxy(subscribe, publish);
except Exception as e:
print("Bus terminated")
def message_queue():
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
frontend.bind("tcp://0.0.0.0:5559")
backend = context.socket(zmq.DEALER)
backend.bind("tcp://0.0.0.0:5560")
zmq.proxy(frontend, backend)
frontend.close()
backend.close()
context.term()
def sub_self_loop():
sub_self = context.socket(zmq.SUB)
if sys.argv[1] == "core":
sub_self_addr = "tcp://127.0.0.1:5564"
else:
sub_self_addr = "tcp://" + proxy_host + ":5564"
sub_self.connect(sub_self_addr)
sub_self.setsockopt(zmq.SUBSCRIBE, "")
pub_self = context.socket(zmq.PUB)
pub_self_addr = "tcp://127.0.0.1:5555"
pub_self.bind(pub_self_addr)
zmq.proxy(sub_self, pub_self)
def run(self):
try:
nowtime = datetime.now()
if nowtime.hour >= 6 and nowtime.hour < 16:
self.xareal_SC0.AdviseRealData()
self.xareal_SC1.AdviseRealData()
self.xareal_C01.AdviseRealData()
elif nowtime.hour >= 16 or nowtime.hour < 6:
self.xareal_EU1.AdviseRealData()
zmq.proxy(self.socket_clients, self.socket_workers)
except zmq.ContextTerminated:
self.clients.close()
self.workers.close()
self.context.term()
def handle(self, *args, **options):
if options['level'] == 'ERROR':
self.logger.setLevel(logging.ERROR)
elif options['level'] == 'WARN':
self.logger.setLevel(logging.WARN)
elif options['level'] == 'INFO':
self.logger.setLevel(logging.INFO)
else:
self.logger.setLevel(logging.DEBUG)
if not settings.EVENT_NOTIFICATION:
self.logger.error("'EVENT_NOTIFICATION' is set to False, LAVA won't generated any events")
self.logger.info("Dropping priviledges")
if not self.drop_priviledges(options['user'], options['group']):
self.logger.error("Unable to drop priviledges")
return
self.logger.info("Creating the ZMQ proxy")
context = zmq.Context.instance()
pull = context.socket(zmq.PULL)
pull.bind(settings.INTERNAL_EVENT_SOCKET)
pub = context.socket(zmq.PUB)
pub.bind(settings.EVENT_SOCKET)
# Create the monitoring thread only in DEBUG
monitor_in = None
monitor = None
if options['level'] == 'DEBUG':
monitor_in = context.socket(zmq.PUSH)
monitor_in.bind('inproc:///monitor')
self.logger.debug("Starting the monitor")
stop_monitor = threading.Event()
monitor = Monitor(stop_monitor)
monitor.start()
self.logger.info("Starting the Proxy")
try:
zmq.proxy(pull, pub, monitor_in)
except KeyboardInterrupt:
self.logger.info("Received Ctrl+C, leaving")
if monitor_in is not None:
# Stop the monitor thread
stop_monitor.set()
# and send a message to unlock the socket
monitor_in.send('Finishing the monitor')
monitor.join()
def forward():
""" BACKGROUND THREAD """
# LISTEN TO CLUSTER ON 5563 TO FORWARD MESSAGES TO 5564
# subscribe to all channels (prefix-matching)
core_subscriber = context.socket(zmq.SUB)
core_subscriber.setsockopt(zmq.RCVHWM, max_buffer)
core_subscriber.connect("tcp://127.0.0.1:5563")
core_subscriber.connect("tcp://10.4.106.210:5563")
core_subscriber.connect("tcp://127.0.0.1:5565")
core_subscriber.setsockopt(zmq.SUBSCRIBE, "")
# PUBLISH TO CLUSTER ON 5564 AFTER RECEIVING ON 5563
cluster_publisher = context.socket(zmq.PUB)
cluster_publisher.bind("tcp://*:5564")
zmq.proxy(core_subscriber, cluster_publisher)
