def main(req_port=None, res_port=None, use_security=False):
'''main of queue
:param req_port: port for clients
:param res_port: port for servers
'''
if req_port is None:
req_port = env.get_req_port()
if res_port is None:
res_port = env.get_res_port()
auth = None
try:
context = zmq.Context()
frontend_service = context.socket(zmq.XREP)
backend_service = context.socket(zmq.XREQ)
if use_security:
if not os.path.exists(env.get_server_public_key_dir()):
create_certificates(env.get_server_public_key_dir())
auth = Authenticator.instance(env.get_server_public_key_dir())
auth.set_server_key(
frontend_service, env.get_server_secret_key_path())
auth.set_client_key(backend_service, env.get_client_secret_key_path(),
env.get_server_public_key_path())
frontend_service.bind('tcp://*:{req_port}'.format(req_port=req_port))
backend_service.bind('tcp://*:{res_port}'.format(res_port=res_port))
zmq.device(zmq.QUEUE, frontend_service, backend_service)
except KeyboardInterrupt:
pass
finally:
frontend_service.close()
backend_service.close()
context.term()
if use_security and auth is not None:
auth.stop()
def main(pub_port=None, sub_port=None):
'''main of forwarder
:param sub_port: port for subscribers
:param pub_port: port for publishers
'''
try:
if sub_port is None:
sub_port = get_sub_port()
if pub_port is None:
pub_port = get_pub_port()
context = zmq.Context(1)
frontend = context.socket(zmq.SUB)
backend = context.socket(zmq.PUB)
frontend.bind('tcp://*:{pub_port}'.format(pub_port=pub_port))
frontend.setsockopt(zmq.SUBSCRIBE, b'')
backend.bind('tcp://*:{sub_port}'.format(sub_port=sub_port))
zmq.device(zmq.FORWARDER, frontend, backend)
except KeyboardInterrupt:
pass
finally:
frontend.close()
backend.close()
context.term()
def main():
""" server routine """
url_worker = "inproc://workers"
url_client = "tcp://*:5555"
# Prepare our context and sockets
context = zmq.Context(1)
# Socket to talk to clients
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
# Socket to talk to workers
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
# Launch pool of worker threads
for i in range(5):
thread = threading.Thread(target=worker_routine, args=(url_worker, context, ))
thread.start()
zmq.device(zmq.QUEUE, clients, workers)
# We never get here but clean up anyhow
clients.close()
workers.close()
context.term()
def LaunchBroker(config):
"""Start a set of intermediate devices for the cluster on this machine.
This method does not return.
:param ConfigParser config: Configuration information for cluster.
"""
cmd_frontend_url = config.get('server', 'command_frontend_url')
cmd_backend_url = config.get('server', 'command_backend_url')
log_frontend_url = config.get('server', 'log_frontend_url')
log_backend_url = config.get('server', 'log_backend_url')
def thread_target():
context = zmq.Context()
log_be_socket = MakeSocket(context, url = log_backend_url, type = zmq.PUB,
bind = True)
log_fe_socket = MakeSocket(context, url = log_frontend_url, type = zmq.SUB,
bind = True, options = {zmq.SUBSCRIBE : ""})
zmq.device(zmq.FORWARDER, log_fe_socket, log_be_socket)
thread = threading.Thread(target = thread_target)
thread.daemon = True
logging.info("Launching logging forwarder -- clients should SUB to "
"%s and PUB to %s" % (log_backend_url, log_frontend_url))
thread.start()
context = zmq.Context()
cmd_be_socket = MakeSocket(context, url = cmd_backend_url, type = zmq.PUB,
bind = True)
cmd_fe_socket = MakeSocket(context, url = cmd_frontend_url, type = zmq.SUB,
bind = True, options = {zmq.SUBSCRIBE : ""})
logging.info("Launching command forwarder -- clients should SUB to "
"%s and PUB to %s" % (cmd_backend_url, cmd_frontend_url))
zmq.device(zmq.FORWARDER, cmd_fe_socket, cmd_be_socket)
def main():
cfg = Loader('mq').load()
cfg_messaging = dict(cfg[1])
log = logger.Logger('mq_event_fwd').get_logger()
try:
context = zmq.Context(1)
# Socket facing emitters
frontend = context.socket(zmq.SUB)
# Forwarder subscribes to the emitter *pub* port
sub_addr = "tcp://*:%s" % cfg_messaging['event_pub_port']
frontend.bind(sub_addr)
log.debug("Waiting for messages on %s" % sub_addr)
# We want to get all messages from emitters
frontend.setsockopt(zmq.SUBSCRIBE, "")
# Socket facing receivers
backend = context.socket(zmq.PUB)
# Forwarder publishes to the receiver *sub* port
pub_addr = "tcp://*:%s" % cfg_messaging['event_sub_port']
backend.bind(pub_addr)
log.debug("Sending messages to %s" % pub_addr)
log.debug("Forwarding messages...")
zmq.device(zmq.FORWARDER, frontend, backend)
except Exception as e:
log.error(e)
log.error("Bringing down ZMQ device")
raise Exception("Error with forwarder device")
finally:
pass
frontend.close()
backend.close()
context.term()
def run_forwarder():
sub_addr = 'tcp://*:5759'
pub_addr = 'tcp://*:5760'
print "starting forwarder device. sub: '%s', pub: '%s' " % (sub_addr, pub_addr)
print os.getpid()
backend = None
frontend = None
context = None
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.SUB)
frontend.bind(sub_addr)
#print "started forwarder with subs bound at '%s'" % sub_addr
frontend.setsockopt(zmq.SUBSCRIBE, "")
# Socket facing services
backend = context.socket(zmq.PUB)
backend.bind(pub_addr)
#print "started forwarder with pub bound at '%s'" % pub_addr
zmq.device(zmq.FORWARDER, frontend, backend)
except Exception, e:
print e
print "bringing down zmq device"
def q(listen,worker_target):
self.workers = self.context.socket(zmq.XREQ)
self.workers.bind(worker_target);
self.clients = self.context.socket(zmq.XREP)
self.clients.bind(listen)
zmq.device(zmq.QUEUE, self.clients, self.workers)
def forward_modules_to_clients_raw( self ):
self.log.info("Starting SUB->PUB forward from socket %d to %d " % (self.port_module_sub, self.port_client_pub ))
#import pdb; pdb.set_trace()
frontend = None
backend = None
try:
# Socket facing clients
context = zmq.Context(1)
frontend = context.socket(zmq.SUB)
frontend.bind("%s:%d" % ( self.protocol, self.port_module_sub ) )
# No filtering of messages please
frontend.setsockopt(zmq.SUBSCRIBE, "")
# Socket facing services
backend = context.socket(zmq.PUB)
backend.bind("%s:%d" % ( self.protocol, self.port_client_pub ) )
zmq.device( zmq.FORWARDER, frontend, backend )
print("FAIL")
except Exception as e:
self.log.error("Exception while running forward: " + str(e))
raise Exception("Forwarder failed")
finally:
pass
if frontend:
frontend.close()
if backend:
backend.close()
def run(self):
print("[*] Starting proxy")
backend = None
frontend = None
context = None
try:
context = zmq.Context()
# Socket facing clients
frontend = context.socket(zmq.PULL)
frontend.bind(self.listen_in)
# Socket facing services
backend = context.socket(zmq.PUSH)
backend.bind(self.listen_out)
zmq.device(zmq.FORWARDER, frontend, backend)
except Exception as e:
print("bringing down zmq device")
print(e)
finally:
frontend.close()
backend.close()
context.term()
def thread_target():
context = zmq.Context()
log_be_socket = MakeSocket(context, url = log_backend_url, type = zmq.PUB,
bind = True)
log_fe_socket = MakeSocket(context, url = log_frontend_url, type = zmq.SUB,
bind = True, options = {zmq.SUBSCRIBE : ""})
zmq.device(zmq.FORWARDER, log_fe_socket, log_be_socket)
def run(self):
self.socket = self.context.socket(zmq.REP)
if self.port == 0:
self.port = self.socket.bind_to_random_port('tcp://%s' % self.ip)
else:
self.socket.bind('tcp://%s:%i' % self.addr)
zmq.device(zmq.FORWARDER, self.socket, self.socket)
def main():
"""Server routine"""
url_worker = "inproc://workers"
url_client = "tcp://*:5555"
setup_buffers()
# Prepare our context and sockets
context = zmq.Context.instance()
# Socket to talk to clients
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
# Socket to talk to workers
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
# Launch pool of worker threads
for i in range(WORKER_NUM):
thread = threading.Thread(target=worker_routine, args=(url_worker,))
thread.start()
try:
zmq.device(zmq.QUEUE, clients, workers)
except KeyboardInterrupt:
pass
# We never get here but clean up anyhow
clients.close()
workers.close()
context.term()
def __init__(self, fan_in_port, fan_out_port):
ctx = zmq.Context()
front = ctx.socket(zmq.XREP)
back = ctx.socket(zmq.XREQ)
front.bind("tcp://*:{port}".format(port=int(fan_in_port)))
back.bind("tcp://*:{port}".format(port=int(fan_out_port)))
zmq.device(zmq.QUEUE, front, back)
def forward(front_port, backend_port):
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.SUB)
frontend.bind("tcp://*:%s" % front_port)
frontend.setsockopt(zmq.SUBSCRIBE, b"")
# Socket facing services
backend = context.socket(zmq.PUB)
backend.bind("tcp://*:%s" % backend_port)
try:
log.logging.info("Start forwarding from %s to %s", front_port, backend_port)
zmq.device(zmq.FORWARDER, frontend, backend)
except Exception as e:
log.logging.error(e)
print("[bringing down zmq device]")
finally:
frontend.close()
backend.close()
context.term()
def run(self):
'''
Start the ret port binding
'''
self.context = zmq.Context(self.opts['worker_threads'])
self.uri = 'tcp://{interface}:{ret_port}'.format(**self.opts)
log.info('ZMQ Ret port binding to {0}'.format(self.uri))
self.clients = self.context.socket(zmq.ROUTER)
if self.opts['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'):
# IPv6 sockets work for both IPv6 and IPv4 addresses
self.clients.setsockopt(zmq.IPV4ONLY, 0)
try:
self.clients.setsockopt(zmq.HWM, self.opts['rep_hwm'])
except AttributeError:
self.clients.setsockopt(zmq.SNDHWM, self.opts['rep_hwm'])
self.clients.setsockopt(zmq.RCVHWM, self.opts['rep_hwm'])
self.workers = self.context.socket(zmq.DEALER)
self.w_uri = 'ipc://{0}'.format(
os.path.join(self.opts['sock_dir'], 'workers.ipc')
)
log.info('Setting up the master communication server')
self.clients.bind(self.uri)
self.workers.bind(self.w_uri)
while True:
try:
zmq.device(zmq.QUEUE, self.clients, self.workers)
except zmq.ZMQError as exc:
if exc.errno == errno.EINTR:
continue
raise exc
def run(self):
logger.debug('Starting {} device'.format(self._name))
try:
self.frontend, self.backend = self.get_sockets()
zmq.device(zmq.QUEUE, self.frontend, self.backend)
except KeyboardInterrupt:
pass
def __bind(self):
'''
Binds the reply server
'''
log.info('Setting up the master communication server')
self.clients.bind(self.uri)
self.work_procs = []
for ind in range(int(self.opts['worker_threads'])):
self.work_procs.append(MWorker(self.opts,
self.master_key,
self.key,
self.crypticle))
for ind, proc in enumerate(self.work_procs):
log.info('Starting Salt worker process {0}'.format(ind))
proc.start()
self.workers.bind(self.w_uri)
while True:
try:
zmq.device(zmq.QUEUE, self.clients, self.workers)
except zmq.ZMQError as exc:
if exc.errno == errno.EINTR:
continue
raise exc
def start(self):
"""
Start the server.
Note: this is a blocking call.
"""
while True:
try:
print "Starting ZRPC server"
self.ctx = zmq.Context()
self.backend = self.ctx.socket(zmq.XREQ)
print "Listening for server workers on %s" % self._get_server_addr()
self.backend.bind(self._get_server_addr())
self.frontend = self.ctx.socket(zmq.XREP)
print "Listening for clients on %s" % self._get_client_addr()
self.frontend.bind(self._get_client_addr())
zmq.device(zmq.QUEUE, self.frontend, self.backend)
except Exception as e:
print 'Exception in ZRPC server: %s' % str(e)
print 'Stopping ZRPC server'
self.frontend.close()
self.backend.close()
self.ctx.term()
time.sleep(3)
def run_device(self):
"""The runner method.
Do not call me directly, instead call ``self.start()``, just like a Thread.
"""
ins,outs = self._setup_sockets()
device(self.device_type, ins, outs)
def start(self):
"""
Main execution.
Instantiate workers, Accept client connections,
distribute computation requests among workers and route computed results back to clients.
"""
print("Starting server...")
# Front facing socket to accept client connections.
socket_front = self.zmq_context.socket(zmq.ROUTER)
socket_front.bind('tcp://127.0.0.1:5010')
# Backend socket to distribute work.
socket_back = self.zmq_context.socket(zmq.DEALER)
socket_back.bind('inproc://backend')
# Start three workers.
for i in range(1,2):
worker = Worker(self.zmq_context, i, self._stree)
worker.start()
print("Threaded worker %d started..." % i)
# Use built in queue device to distribute requests among workers.
# What queue device does internally is,
# 1. Read a client's socket ID and request.
# 2. Send socket ID and request to a worker.
# 3. Read a client's socket ID and result from a worker.
# 4. Route result back to the client using socket ID.
zmq.device(zmq.QUEUE, socket_front, socket_back)
def main():
logging.basicConfig(format = "%(levelname)s:%(message)s", level = logging.INFO)
logging.info("Start HiveServer Router")
config = HiveServerRouterConfig("hsr.conf")
router_port = config.router_port()
if router_port == None:
logging.error("No router port")
sys.exit(1)
url_worker = "inproc://workers"
url_client = "tcp://*:" + router_port
context = zmq.Context()
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
ports = config.hive_servers_ports()
if ports == None:
logging.error("No hive server ports")
sys.exit(1)
for port in ports:
thread = threading.Thread(target = worker_routine, args = (url_worker, context, port))
thread.start()
zmq.device(zmq.QUEUE, clients, workers)
clients.close()
workers.close()
context.term()
def run_server(threads=10):
""" server routine """
signal.signal(signal.SIGINT, signal_handler)
global clients
global workers
global context
url_worker = "inproc://workers"
url_client = "tcp://*:2112"
context = zmq.Context(1)
clients = context.socket(zmq.ROUTER)
clients.bind(url_client)
# Worker sockets
workers = context.socket(zmq.DEALER)
workers.bind(url_worker)
for i in range(threads):
thread = threading.Thread(target=worker_routine, args=(url_worker, context, ))
thread.setDaemon(True)
thread.start()
zmq.device(zmq.QUEUE, clients, workers)
def LaunchStreamerDevice(context, frontend_connect, backend_connect):
frontend = frontend_connect.MakeSocket(context, type = zmq.PULL, bind = True)
backend = backend_connect.MakeSocket(context, type = zmq.PUSH, bind = True)
logging.info("LaunchStreamerDevice: starting streamer on pid %d" % \
os.getpid())
logging.info("LaunchStreamerDevice: frontend: %s" % (frontend_connect,))
logging.info("LaunchStreamerDevice: backend: %s" % (backend_connect,))
zmq.device(zmq.STREAMER, frontend, backend)
def main(front_addr, end_addr):
ctx = zmq.Context().instance()
front = ctx.socket(zmq.XREP)
end = ctx.socket(zmq.XREQ)
front.bind(front_addr)
end.bind(end_addr)
print 'listening on ', front_addr
print 'listening on ', end_addr
zmq.device(zmq.QUEUE, front, end)
def ventilator():
context = zmq.Context(1)
frontend = context.socket(zmq.PULL)
frontend.bind("tcp://*:5559")
backend = context.socket(zmq.PUSH)
backend.bind("tcp://*:5560")
zmq.device(zmq.STREAMER, frontend, backend)
def main(self):
try:
zmq.device(zmq.FORWARDER, self.frontend, self.backend)
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
self.frontend.close()
self.backend.close()
else:
raise
def start_forwarder():
print("Starting the forwarding")
context = zmq.Context()
frontend = context.socket(zmq.SUB)
frontend.setsockopt(zmq.SUBSCRIBE, '')
frontend.bind(FRONTEND_ADDR)
backend = context.socket(zmq.PUB)
backend.bind(BACKEND_ADDR)
zmq.device(zmq.FORWARDER, frontend, backend)
def LaunchForwarderDevice(context, frontend_connect, backend_connect):
frontend = frontend_connect.MakeSocket(context, type = zmq.SUB, bind = True,
options = {zmq.SUBSCRIBE : ""})
backend = backend_connect.MakeSocket(context, type = zmq.PUB, bind = True)
logging.info("LaunchForwarderDevice: starting forwarder on pid %d" % \
os.getpid())
logging.info("LaunchForwarderDevice: frontend: %s" % (frontend_connect,))
logging.info("LaunchForwarderDevice: backend: %s" % (backend_connect,))
zmq.device(zmq.FORWARDER, frontend, backend)
def main():
context = zmq.Context(1)
frontend = context.socket(zmq.SUB)
frontend.bind("tcp://*:%d" % (frontend_port,))
frontend.setsockopt(zmq.SUBSCRIBE, "")
backend = context.socket(zmq.PUB)
backend.bind("tcp://*:%d" % (backend_port,))
zmq.device(zmq.FORWARDER, frontend, backend)
def forwarder():
ctx = zmq.Context.instance()
rcv = ctx.socket(zmq.SUB)
rcv.bind('inproc://fwdrcv')
rcv.setsockopt(zmq.SUBSCRIBE,b'')
snd = ctx.socket(zmq.PUB)
snd.bind('tcp://127.0.0.1:6000')
zmq.device(zmq.FORWARDER,rcv,snd)
def create_forwarder(self, sub_port, pub_port):
self.logger.debug("Starting message forward device")
self.subscriber = self.create_subscriber(sub_port, bind=True)
self.publisher = self.create_publisher(pub_port, bind=True)
try:
zmq.device(zmq.FORWARDER, self.subscriber, self.publisher)
except Exception as e:
self.logger.warning(e)
self.logger.warning("bringing down zmq device")
finally:
self.publisher.close()
self.subscriber.close()
self.context.term()
def queue(self):
try:
frontend = self.context.socket(XREP)
frontend.bind("tcp://*:6900")
backend = self.context.socket(XREQ)
backend.bind("tcp://*:6969")
device(QUEUE, frontend, backend)
except Exception:
print(format_exc())
if environ["PRODUCTION_MODE"]: self.logging.report_exception()
finally:
pass
frontend.close()
backend.close()
def main():
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.XREP)
frontend.bind("tcp://*:5550")
# Socket facing services
backend = context.socket(zmq.XREQ)
backend.bind("tcp://*:5560")
zmq.device(zmq.QUEUE, frontend, backend)
except Exception, e:
print e
print "bringing down zmq device"
def __setup__(self):
# TODO - Refactor to Send to new Logger Command - With the 'logging_label'
logger_info_message = 'Starting Arg Proxy Queue listening on: ' + self.location1 + " returning to: " + self.location2
print("ARGProxyQueue.__setup__: (self.logging_label): " +
str(self.logging_label) + ": " + str(logger_info_message))
#self.logger.info('Starting Arg Proxy Queue listening on: '+self.location1+" returning to: "+self.location2)
context = zmq.Context()
inputsocket = context.socket(zmq.PULL)
inputsocket.bind(self.location1)
outputsocket = context.socket(zmq.PUSH)
outputsocket.bind(self.location2)
zmq.device(zmq.QUEUE, inputsocket, outputsocket)
def heartbeat_loop(self):
"""Run heartbeat"""
self.dprint(2, "Starting heartbeat loop...")
while not self.exiting:
self.dprint(3, ".", end="")
try:
zmq.device(zmq.FORWARDER, self.heartbeat_socket,
self.heartbeat_socket)
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
else:
raise
else:
break
def __bind(self):
'''
Binds the reply server
'''
log.info('Setting up the master communication server')
self.clients.bind(self.uri)
for ind in range(int(self.opts['worker_threads'])):
log.info('Starting Salt worker process {0}'.format(ind))
MWorker(self.opts, self.master_key, self.key, self.crypticle,
self.aes_funcs, self.clear_funcs).start()
self.workers.bind(self.w_uri)
zmq.device(zmq.QUEUE, self.clients, self.workers)
def start(self, name, clie):
try:
context = zmq.Context(1)
# Client side
clientside = context.socket(zmq.XREP)
clientside.bind("tcp://0.0.0.0:%s" % str(clie))
# Server side
serverside = context.socket(zmq.XREQ)
serverside.bind("tcp://0.0.0.0:%s" % str(name))
logging.info("Starting Queue...")
zmq.device(zmq.QUEUE, clientside, serverside)
except Exception, e:
logging.info("[QUEUE] Failed reason: %s", e)
logging.info("[QUEUE] Going down...")
def loop(self):
threading.Thread(target=self.stater).start()
for i in range(self.worker_num):
child = Process(target=self.worker,
args=(
i,
self.url_worker,
self.context,
self.worker_counts,
))
child.start()
zmq.device(zmq.QUEUE, self.router, self.workers)
self.router.close()
self.workers.close()
self.context.term()
def zmq_data_forwarder(pub_proxy_port=ZMQ_PUB_PROXY_PORT,
sub_proxy_port=ZMQ_SUB_PROXY_PORT):
''' forwarder for the publish/subscribe messaging pattern
pubsub proxy
'''
_context = zmq.Context()
frontend = _context.socket(zmq.PUB)
frontend.bind("tcp://*:%d" % pub_proxy_port)
backend = _context.socket(zmq.SUB)
backend.setsockopt(zmq.SUBSCRIBE, "")
#backend.setsockopt(zmq.HWM, 1)
backend.bind("tcp://*:%d" % sub_proxy_port)
print "Setup Proxy Sub:%d --> Pub:%d" % (sub_proxy_port, pub_proxy_port)
zmq.device(zmq.FORWARDER, frontend, backend)
print 'exit ... Proxy'
def message_bus(pub_port=5556, sub_port=5555):
bus_in = get_socket(zmq.XSUB)
bus_out = get_socket(zmq.XPUB)
bus_in.bind('tcp://*:{port}'.format(port=sub_port))
bus_out.bind('tcp://*:{port}'.format(port=pub_port))
print '∅MQ multi-publisher message bus {sub_port}→ {pub_port}' \
.format(sub_port=sub_port, pub_port=pub_port)
zmq.device(zmq.FORWARDER, bus_in, bus_out)
# shouldn't get here
bus_in.close()
bus_out.close()
def run(self, pub_port, sub_port):
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.SUB)
frontend.bind("tcp://*:{}".format(sub_port))
frontend.setsockopt(zmq.SUBSCRIBE, "".encode('utf-8'))
# Socket facing services
backend = context.socket(zmq.PUB)
backend.bind("tcp://*:{}".format(pub_port))
zmq.device(zmq.FORWARDER, frontend, backend)
except Exception:
self.to_parent.put(traceback.format_exc())
def worker():
context = zmq.Context(1)
frontend = context.socket(zmq.SUB)
frontend.bind("ipc:///tmp/keylime.verifier.ipc")
frontend.setsockopt(zmq.SUBSCRIBE, b'')
# Socket facing services
backend = context.socket(zmq.PUB)
backend.bind(
"tcp://%s:%s" %
(config.get('cloud_verifier', 'revocation_notifier_ip'),
config.getint('cloud_verifier', 'revocation_notifier_port')))
zmq.device(zmq.FORWARDER, frontend, backend)
def create_queue(xrep_port, xreq_port):
"""Vytvoření fronty."""
context = zmq.Context()
frontend = context.socket(zmq.XREP)
address = "tcp://*:{port}".format(port=xrep_port)
frontend.bind(address)
print("Bound to {a} on port {p}".format(a=address, p=xrep_port))
backend = context.socket(zmq.XREQ)
address = "tcp://*:{port}".format(port=xreq_port)
backend.bind(address)
print("Bound to {a} on port {p}".format(a=address, p=xreq_port))
zmq.device(zmq.QUEUE, frontend, backend)
def zmq_device(self):
'''
Multiprocessing target for the zmq queue device
'''
self.__setup_signals()
salt.utils.appendproctitle('MWorkerQueue')
self.context = zmq.Context(self.opts['worker_threads'])
# Prepare the zeromq sockets
self.uri = 'tcp://{interface}:{ret_port}'.format(**self.opts)
self.clients = self.context.socket(zmq.ROUTER)
if self.opts['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'):
# IPv6 sockets work for both IPv6 and IPv4 addresses
self.clients.setsockopt(zmq.IPV4ONLY, 0)
self.clients.setsockopt(zmq.BACKLOG, self.opts.get('zmq_backlog', 1000))
if HAS_ZMQ_MONITOR and self.opts['zmq_monitor']:
# Socket monitor shall be used the only for debug purposes so using threading doesn't look too bad here
import threading
self._monitor = ZeroMQSocketMonitor(self.clients)
t = threading.Thread(target=self._monitor.start_poll)
t.start()
self.workers = self.context.socket(zmq.DEALER)
if self.opts.get('ipc_mode', '') == 'tcp':
self.w_uri = 'tcp://127.0.0.1:{0}'.format(
self.opts.get('tcp_master_workers', 4515)
)
else:
self.w_uri = 'ipc://{0}'.format(
os.path.join(self.opts['sock_dir'], 'workers.ipc')
)
log.info('Setting up the master communication server')
self.clients.bind(self.uri)
self.workers.bind(self.w_uri)
while True:
if self.clients.closed or self.workers.closed:
break
try:
zmq.device(zmq.QUEUE, self.clients, self.workers)
except zmq.ZMQError as exc:
if exc.errno == errno.EINTR:
continue
raise exc
except (KeyboardInterrupt, SystemExit):
break
def start(inPort, outPort):
"""Start forwarding messages.
This function never returns.
= INPUTS
- inPort int: Input XSUB subscriber port number to use.
- outPort int: Output XPUB publisher port number to use.
"""
log = tHome.util.log.get("msgHub")
ctx = zmq.Context()
intSock, outSock = None, None
try:
# Inbound message port.
log.info("Starting inbound subscribe socket at port %d" % inPort)
inSock = ctx.socket(zmq.XSUB)
# Use * to bind on all interfaces. Otherwise the address has to
# be an exact match (127.0.0.1 != IP).
inSock.bind("tcp://*:%d" % inPort)
# Outbound message port.
log.info("Starting outbound publish socket at port %d" % outPort)
outSock = ctx.socket(zmq.XPUB)
outSock.bind("tcp://*:%d" % outPort)
# Use ZMP to handle all the forwarding. We could add logging
# here but it's easier just to add a new subscriber to read and
# log any messages.
#
# NOTE: this never returns.
log.info("Starting forwarding")
zmq.device(zmq.FORWARDER, inSock, outSock)
except (Exception, KeyboardInterrupt) as e:
log.critical("Exception thrown", exc_info=True)
raise
finally:
if inSock:
inSock.close()
if outSock:
outSock.close()
ctx.term()
def run(self):
self.dispatcher_insock = self.zmq_ctx.socket(zmq.PULL)
self.dispatcher_insock.bind("tcp://*:%s" %
self.config['general']['dispatcher_inport'])
logger.info('Listening watcher message, on port: %s' %
str(self.config['general']['dispatcher_inport']))
self.dispatcher_outsock = self.zmq_ctx.socket(zmq.PUSH)
self.dispatcher_outsock.bind(
"tcp://*:%s" % self.config['general']['dispatcher_outport'])
logger.info('Listening worker connect, on port: %s' %
str(self.config['general']['dispatcher_outport']))
logger.info('Starting job dispatcher')
zmq.device(zmq.STREAMER, self.dispatcher_insock,
self.dispatcher_outsock)
def run(self):
""" Not to be called directly. Use start() """
logger.debug("Starting RPC thread loop w/ %d worker(s)" %
self._num_threads)
self.receiver = self._context.socket(zmq.ROUTER)
self.receiver.bind(self._address)
logger.debug("Listening @ %s" % self._address)
self.dealer = self._context.socket(zmq.DEALER)
self.dealer.bind(self._worker_url)
for i in xrange(self._num_threads):
thread = Thread(target=self._worker_routine,
name="RPC-Worker-%d" % (i + 1))
thread.daemon = True
thread.start()
try:
# blocking
ret = zmq.device(zmq.QUEUE, self.receiver, self.dealer)
except ZMQError, e:
# stop() generates a valid ETERM, otherwise its unexpected
if not (e.errno == zmq.ETERM and self.exit_request):
raise
def main():
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.PULL)
frontend.bind("tcp://*:5559")
# Socket facing services
backend = context.socket(zmq.PUSH)
backend.bind("tcp://*:5560")
zmq.device(zmq.STREAMER, frontend, backend)
except Exception, e:
print e
print "bringing down zmq device"
def _start(self, num_threads=1):
log.debug("main thread %s starting" % threading.currentThread().ident)
for unused in range(num_threads):
thread = threading.Thread(target=self.thread_func)
thread.start()
self.worker_threads.append(thread)
try:
zmq.device(zmq.QUEUE, self.clients, self.workers)
except zmq.core.error.ZMQError, e:
if e.errno == zmq.ETERM:
# server was shut down
pass
else:
raise
def run(self):
self.forwarder_insock = self.zmq_ctx.socket(zmq.SUB)
self.forwarder_insock.bind("tcp://*:%s" %
self.config['general']['forwarder_inport'])
self.forwarder_insock.setsockopt(zmq.SUBSCRIBE, "")
logger.info('Listening worker connect, on port: %s' %
str(self.config['general']['forwarder_inport']))
self.forwarder_outsock = self.zmq_ctx.socket(zmq.PUB)
self.forwarder_outsock.bind("tcp://*:%s" %
self.config['general']['forwarder_outport'])
logger.info('Listening exporter connect, on port: %s' %
str(self.config['general']['forwarder_outport']))
logger.info('Starting job forwarder')
zmq.device(zmq.FORWARDER, self.forwarder_insock, self.forwarder_outsock)
def main():
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.SUB)
frontend.bind("tcp://*:5700")
frontend.setsockopt(zmq.SUBSCRIBE, "")
# Socket facing services
backend = context.socket(zmq.PUB)
backend.bind("tcp://*:5701")
zmq.device(zmq.FORWARDER, frontend, backend)
except Exception, e:
print e
print "bringing down zmq device"
def Forwarder(port_in=5555, port_out=5558):
try:
context = zmq.Context(1)
frontend = context.socket(zmq.PUB)
frontend.bind("tcp://*:{}".format(port_out))
backend = context.socket(zmq.SUB)
backend.bind("tcp://*:{}".format(port_in))
zmq.device(zmq.FORWARDER, frontend, backend)
except Exception as e:
print(e)
print("bringing down zmq device")
finally:
pass
frontend.close()
backend.close()
context.term()
def Queue(port_in=5555, port_out=5558):
try:
context = zmq.Context(1)
frontend = context.socket(zmq.XREP)
frontend.bind("tcp://*:{}".format(port_out))
backend = context.socket(zmq.XREQ)
backend.bind("tcp://*:{}".format(port_in))
zmq.device(zmq.QUEUE, frontend, backend)
except Exception as e:
print(e)
print("bringing down zmq device")
finally:
pass
frontend.close()
backend.close()
context.term()
def main():
try:
context = zmq.Context(1)
#sockets that face clients
frontend = context.socket(zmq.XREP) #XREP = response deivce
frontend.bind("tcp://*:5559")
#sockets that face services
backend = context.socket(zmq.XREQ)
backend.bind("tcp://*:5560")
zmq.device(zmq.QUEUE, frontend, backend)
except Exception, e:
print e
print "destroying zmq queue device"
def main():
try:
context = zmq.Context(1)
frontend = context.socket(zmq.SUB)
frontend.bind("tcp://0.0.0.0:%s" % 5559)
frontend.setsockopt(zmq.SUBSCRIBE, "")
backend = context.socket(zmq.PUB)
backend.bind("tcp://0.0.0.0:%s" % 5560)
zmq.device(zmq.FORWARDER, frontend, backend)
print "Starting ZeroMQ forwarder on port %s and %s" % (pub_port,
sub_port)
except Exception, e:
print e
print "Bringing down ZeroMQ forwarder"
def main(publisher_Port, subscriber_Port):
try:
context = zmq.Context(1)
# socket facing publisher
frontend = context.socket(zmq.SUB)
addr1 = "tcp://*:" + publisher_Port
frontend.bind(addr1)
#frontend.connect("tcp://127.0.0.1:5556")
frontend.setsockopt_string(zmq.SUBSCRIBE, "")
# socket facing suscriber
backend = context.socket(zmq.PUB)
addr2 = "tcp://*:" + subscriber_Port
backend.bind(addr2)
print("Broker is already connected...... ")
events = zmq.device(zmq.FORWARDER, frontend, backend)
except Exception as e:
print(e)
print("bring down zmq device")
finally:
pass
frontend.close()
backend.close()
context.term()
def run(self):
self.socket = self.context.socket(zmq.ROUTER)
self.socket.linger = 1000
c = ':' if self.transport == 'tcp' else '-'
self.socket.bind('%s://%s' % (self.transport, self.ip) + c +
str(self.port))
while True:
try:
zmq.device(zmq.QUEUE, self.socket, self.socket)
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
else:
raise
else:
break
def main():
print "\nServer for ProBot is running..."
try:
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.SUB)
frontend.bind("tcp://*:5579")
frontend.setsockopt(zmq.SUBSCRIBE, "")
# Socket facing services
backend = context.socket(zmq.PUB)
backend.bind("tcp://*:5580")
zmq.device(zmq.FORWARDER, frontend, backend)
except Exception, e:
print e
print "bringing down zmq device"
def main():
context = zmq.Context(1)
# Socket facing clients
frontend = context.socket(zmq.XREP)
frontend.bind("tcp://*:%d" % front_port)
# Socket facing services
backend = context.socket(zmq.XREQ)
backend.bind("tcp://*:%d" % back_port)
try:
zmq.device(zmq.QUEUE, frontend, backend)
except Exception:
print("bringing down zmq device")
finally:
pass
frontend.close()
backend.close()
context.term()
def run(self):
context = zmq.Context.instance()
try:
insock = context.socket(zmq.SUB)
insock.bind(self.in_address)
insock.setsockopt(zmq.SUBSCRIBE, '')
outsock = context.socket(zmq.PUB)
outsock.bind(self.out_address)
zmq.device(zmq.FORWARDER, insock, outsock)
except zmq.ZMQError as ex:
# errno 48 means failed to bind.
# Probably means the forwarder is already running.
if not ex.errno == 48:
raise ex
