def run_queue(): context = Context(1) frontend = context.socket(zmq.ROUTER) # ROUTER backend = context.socket(zmq.ROUTER) # ROUTER frontend.bind("tcp://*:5555") # For clients backend.bind("tcp://*:5556") # For workers poll_workers = Poller() poll_workers.register(backend, zmq.POLLIN) poll_both = Poller() poll_both.register(frontend, zmq.POLLIN) poll_both.register(backend, zmq.POLLIN) workers = [] while True: if workers: socks = yield poll_both.poll() else: socks = yield poll_workers.poll() socks = dict(socks) # Handle worker activity on backend if socks.get(backend) == zmq.POLLIN: # Use worker address for LRU routing msg = yield backend.recv_multipart() if not msg: break print('I: received msg: {}'.format(msg)) address = msg[0] workers.append(address) # Everything after the second (delimiter) frame is reply reply = msg[2:] # Forward message to client if it's not a READY if reply[0] != LRU_READY: print('I: sending -- reply: {}'.format(reply)) yield frontend.send_multipart(reply) else: print('I: received ready -- address: {}'.format(address)) if socks.get(frontend) == zmq.POLLIN: # Get client request, route to first available worker msg = yield frontend.recv_multipart() worker = workers.pop(0) request = [worker, b''] + msg print('I: sending -- worker: {} msg: {}'.format(worker, msg)) yield backend.send_multipart(request)
def socket(): """Context manager to give the bound socket.""" ctx = Context() sock = ctx.socket(zmq.PAIR) sock.bind("tcp://*:8889") yield sock sock.close()
def run_worker(): context = Context(1) worker = context.socket(zmq.REQ) identity = "%04X-%04X" % (randint(0, 0x10000), randint(0, 0x10000)) worker.setsockopt_string(zmq.IDENTITY, identity) worker.connect("tcp://localhost:5556") print("I: (%s) worker ready" % identity) yield worker.send_string(LRU_READY) cycles = 0 while True: msg = yield worker.recv_multipart() if not msg: break cycles += 1 if cycles > 3 and randint(0, 5) == 0: print("I: (%s) simulating a crash" % identity) break elif cycles > 3 and randint(0, 5) == 0: print("I: (%s) simulating CPU overload" % identity) yield gen.sleep(3) print("I: (%s) normal reply" % identity) # Do some heavy work yield gen.sleep(1) yield worker.send_multipart(msg)
async def open(self): context = Context() sock = context.socket(zmq.SUB) sock.setsockopt(zmq.CONFLATE, 1) sock.connect("tcp://localhost:{}".format(settings.ZMQ_PORT)) sock.subscribe(b'') self.sock = sock self._stopped = False logger.debug("client connected: %s", self.request.remote_ip) tornado.ioloop.IOLoop.current().spawn_callback(self.pipe_message)
class PyTestEngine(object): def __init__(self, opts): self.opts = opts self.id = opts["id"] self.role = opts["__role"] self.returner_address = opts["pytest-{}".format(self.role)]["returner_address"] def start(self): log.info("Starting Pytest Event Forwarder Engine(forwarding to %s)", self.returner_address) self.io_loop = ioloop.IOLoop() self.io_loop.make_current() self.io_loop.add_callback(self._start) atexit.register(self.stop) self.io_loop.start() @gen.coroutine def _start(self): self.context = Context() self.push = self.context.socket(zmq.PUSH) log.debug("Connecting PUSH socket to %s", self.returner_address) self.push.connect(self.returner_address) minion_opts = self.opts.copy() minion_opts["file_client"] = "local" self.event = salt.utils.event.get_event( "master", opts=minion_opts, io_loop=self.io_loop, listen=True ) self.event.subscribe("") self.event.set_event_handler(self.handle_event) event_tag = "salt/master/{}/start".format(self.id) log.info("Firing event on engine start. Tag: %s", event_tag) load = {"id": self.id, "tag": event_tag, "data": {}} self.event.fire_event(load, event_tag) def stop(self): push = self.push context = self.context event = self.event self.push = self.context = self.event = None if event: event.unsubscribe("") event.destroy() if push and context: push.close(1000) context.term() self.io_loop.add_callback(self.io_loop.stop) @gen.coroutine def handle_event(self, payload): tag, data = salt.utils.event.SaltEvent.unpack(payload) log.debug("Received Event; TAG: %r DATA: %r", tag, data) forward = salt.utils.msgpack.dumps((self.id, tag, data), use_bin_type=True) yield self.push.send(forward)
def publisher(port=8135): context = Context() pub_uuid = str(uuid.uuid4()) pub = context.socket(zmq.PUB) pub.connect("tcp://localhost:%s" % port) poller = Poller() poller.register(pub, zmq.POLLOUT) while True: topic = 'heartbeat' utc = arrow.utcnow() raw = json.dumps({"timestamp":utc.timestamp, "uuid": pub_uuid}) message = '{0} {1}'.format(topic, raw) yield pub.send(message) yield gen.sleep(1)
def subscriber(port=8135): ''' Bind Subscriber ''' logging.warning("Binding SUB socket on port: {0}".format(port)) context = Context() sub = context.socket(zmq.SUB) sub.bind("tcp://*:%s" % port) sub.setsockopt(zmq.SUBSCRIBE, " ") sub.setsockopt(zmq.SUBSCRIBE, "heartbeat") sub.setsockopt(zmq.SUBSCRIBE, "asterisk") sub.setsockopt(zmq.SUBSCRIBE, "logging") sub.setsockopt(zmq.SUBSCRIBE, "upload") sub.setsockopt(zmq.SUBSCRIBE, "beam") poller = Poller() poller.register(sub, zmq.POLLIN) http_client = _http_client.AsyncHTTPClient() while True: events = yield poller.poll(timeout=1000) if sub in dict(events): # receive raw msg from sub msg = yield sub.recv() # get topic and message topic = msg.split(' ')[0] message = ' '.join(msg.split(' ')[1:]) # this make more sense directly on the beam # that why we're moving, still this sub_bind.py # clear some ideas directly. if topic.startswith('heartbeat'): print(topic, message) elif topic.startswith('asterisk'): print(topic, message) elif topic.startswith('logging'): print(topic, message) elif topic.startswith('upload'): print(topic, message) elif topic.startswith('beam'): print(topic, message) else: # let it crash logging.warning('let it crash') print(msg) else: #logging.warning('nothing receive') pass
def run_server(): context = Context() server = context.socket(zmq.REP) server.bind(SERVER_ADDR) cycles = 0 while True: request = yield server.recv() cycles += 1 # Simulate various problems, after a few cycles if cycles > 3 and randint(0, 3) == 0: print("I: Simulating a crash") server.unbind(SERVER_ADDR) # Delay for a bit, else we get "Address already in use" error. # Note that to really simulate a crash, we should probably kill # this process and start another. yield gen.sleep(2) break elif cycles > 3 and randint(0, 3) == 0: print("I: Simulating CPU overload") yield gen.sleep(2) print("I: Normal request (%s)" % request) yield gen.sleep(1) # Do some heavy work yield server.send(request) raise gen.Return((context, server))
def run_broker(loop): """ main broker method """ url_worker = "inproc://workers" url_client = "inproc://clients" client_nbr = NBR_CLIENTS * 3 # Prepare our context and sockets context = Context() frontend = context.socket(zmq.ROUTER) frontend.bind(url_client) backend = context.socket(zmq.ROUTER) backend.bind(url_worker) # create workers and clients threads # worker_tasks = [] for idx in range(NBR_WORKERS): loop.add_callback(partial(run_worker, url_worker, context, idx)) # worker_tasks.append(task) # client_tasks = [] for idx in range(NBR_CLIENTS): loop.add_callback(partial(run_client, url_client, context, idx)) # client_tasks.append(task) # Logic of LRU loop # - Poll backend always, frontend only if 1+ worker ready # - If worker replies, queue worker as ready and forward reply # to client if necessary # - If client requests, pop next worker and send request to it # Queue of available workers available_workers = 0 workers_list = [] all_workers = set() # init poller poller = Poller() # Always poll for worker activity on backend poller.register(backend, zmq.POLLIN) # Poll front-end only if we have available workers poller.register(frontend, zmq.POLLIN) while True: socks = yield poller.poll() socks = dict(socks) # Handle worker activity on backend if backend in socks and socks[backend] == zmq.POLLIN: # Queue worker address for LRU routing message = yield backend.recv_multipart() assert available_workers < NBR_WORKERS worker_addr = message[0] # add worker back to the list of workers available_workers += 1 workers_list.append(worker_addr) all_workers.add(worker_addr) # Second frame is empty empty = message[1] assert empty == b"" # Third frame is READY or else a client reply address client_addr = message[2] # If client reply, send rest back to frontend if client_addr != b"READY": # Following frame is empty empty = message[3] assert empty == b"" reply = message[4] yield frontend.send_multipart([client_addr, b"", reply]) printdbg('(run_broker) to frontend -- reply: "{}"'.format(reply)) client_nbr -= 1 if client_nbr == 0: printdbg("(run_broker) exiting") break # Exit after N messages # poll on frontend only if workers are available if available_workers > 0: if frontend in socks and socks[frontend] == zmq.POLLIN: # Now get next client request, route to LRU worker # Client request is [address][empty][request] response = yield frontend.recv_multipart() [client_addr, empty, request] = response assert empty == b"" # Dequeue and drop the next worker address available_workers += -1 worker_id = workers_list.pop() yield backend.send_multipart([worker_id, b"", client_addr, b"", request]) printdbg('(run_broker) to backend -- request: "{}"'.format(request)) # out of infinite loop: do some housekeeping printdbg("(run_broker) finishing") for worker_id in workers_list: yield backend.send_multipart([worker_id, b"", b"", b"", b"Stop"]) printdbg("(run_broker) workers cancelled") yield gen.sleep(1) frontend.close() backend.close() # context.term() # Caution: calling term() blocks. printdbg("(run_broker) returning") result = "finished ok" raise gen.Return(result)
def run_broker(loop): """ main broker method """ url_worker = "inproc://workers" url_client = "inproc://clients" client_nbr = NBR_CLIENTS * 3 # Prepare our context and sockets context = Context() frontend = context.socket(zmq.ROUTER) frontend.bind(url_client) backend = context.socket(zmq.ROUTER) backend.bind(url_worker) # create workers and clients threads #worker_tasks = [] for idx in range(NBR_WORKERS): loop.add_callback(partial(run_worker, url_worker, context, idx)) #worker_tasks.append(task) #client_tasks = [] for idx in range(NBR_CLIENTS): loop.add_callback(partial(run_client, url_client, context, idx)) #client_tasks.append(task) # Logic of LRU loop # - Poll backend always, frontend only if 1+ worker ready # - If worker replies, queue worker as ready and forward reply # to client if necessary # - If client requests, pop next worker and send request to it # Queue of available workers available_workers = 0 workers_list = [] all_workers = set() # init poller poller = Poller() # Always poll for worker activity on backend poller.register(backend, zmq.POLLIN) # Poll front-end only if we have available workers poller.register(frontend, zmq.POLLIN) while True: socks = yield poller.poll() socks = dict(socks) # Handle worker activity on backend if (backend in socks and socks[backend] == zmq.POLLIN): # Queue worker address for LRU routing message = yield backend.recv_multipart() assert available_workers < NBR_WORKERS worker_addr = message[0] # add worker back to the list of workers available_workers += 1 workers_list.append(worker_addr) all_workers.add(worker_addr) # Second frame is empty empty = message[1] assert empty == b"" # Third frame is READY or else a client reply address client_addr = message[2] # If client reply, send rest back to frontend if client_addr != b'READY': # Following frame is empty empty = message[3] assert empty == b"" reply = message[4] yield frontend.send_multipart([client_addr, b"", reply]) printdbg('(run_broker) to frontend -- reply: "{}"'.format( reply)) client_nbr -= 1 if client_nbr == 0: printdbg('(run_broker) exiting') break # Exit after N messages # poll on frontend only if workers are available if available_workers > 0: if (frontend in socks and socks[frontend] == zmq.POLLIN): # Now get next client request, route to LRU worker # Client request is [address][empty][request] response = yield frontend.recv_multipart() [client_addr, empty, request] = response assert empty == b"" # Dequeue and drop the next worker address available_workers += -1 worker_id = workers_list.pop() yield backend.send_multipart( [worker_id, b"", client_addr, b"", request]) printdbg('(run_broker) to backend -- request: "{}"'.format( request)) #out of infinite loop: do some housekeeping printdbg('(run_broker) finishing') for worker_id in workers_list: yield backend.send_multipart([worker_id, b"", b"", b"", b"Stop"]) printdbg('(run_broker) workers cancelled') yield gen.sleep(1) frontend.close() backend.close() #context.term() # Caution: calling term() blocks. printdbg('(run_broker) returning') result = 'finished ok' raise gen.Return(result)