def __init__(self, context, agent_addr, client_addr): self._content = context self._loop = asyncio.get_event_loop() self._agent_addr = agent_addr self._client_addr = client_addr self._agent_socket = context.socket(zmq.ROUTER) self._client_socket = context.socket(zmq.ROUTER) self._logger = logging.getLogger("inginious.backend") # Enable support for ipv6 self._agent_socket.ipv6 = True self._client_socket.ipv6 = True self._poller = Poller() self._poller.register(self._agent_socket, zmq.POLLIN) self._poller.register(self._client_socket, zmq.POLLIN) # dict of available environments. Keys are first the type of environement (docker, mcq, kata...) then the # name of the environment. self._environments: Dict[str, Dict[str, EnvironmentInfo]] = {} self._registered_clients = set() # addr of registered clients self._registered_agents: Dict[bytes, AgentInfo] = {} # all registered agents self._ping_count = {} # ping count per addr of agents # addr of available agents. May contain multiple times the same agent, because some agent can # manage multiple jobs at once! self._available_agents = [] # These two share the same objects! Tuples should never be recreated. self._waiting_jobs_pq = TopicPriorityQueue() # priority queue for waiting jobs self._waiting_jobs: Dict[str, WaitingJob] = {} # all jobs waiting in queue self._job_running: Dict[str, RunningJob] = {} # all running jobs
class AsyncioAuthenticator(Authenticator): """ZAP authentication for use in the asyncio IO loop""" def __init__(self, context=None, loop=None): super().__init__(context) self.loop = loop or asyncio.get_event_loop() self.__poller = None self.__task = None @asyncio.coroutine def __handle_zap(self): while True: events = yield from self.__poller.poll() if self.zap_socket in dict(events): msg = yield from self.zap_socket.recv_multipart() self.handle_zap_message(msg) def start(self): """Start ZAP authentication""" super().start() self.__poller = Poller() self.__poller.register(self.zap_socket, zmq.POLLIN) self.__task = asyncio.ensure_future(self.__handle_zap()) def stop(self): """Stop ZAP authentication""" if self.__task: self.__task.cancel() if self.__poller: self.__poller.unregister(self.zap_socket) self.__poller = None super().stop()
def __init__(self, context, agent_addr, client_addr): self._content = context self._loop = asyncio.get_event_loop() self._agent_addr = agent_addr self._client_addr = client_addr self._agent_socket = context.socket(zmq.ROUTER) self._client_socket = context.socket(zmq.ROUTER) self._logger = logging.getLogger("inginious.backend") # Enable support for ipv6 self._agent_socket.ipv6 = True self._client_socket.ipv6 = True self._poller = Poller() self._poller.register(self._agent_socket, zmq.POLLIN) self._poller.register(self._client_socket, zmq.POLLIN) # List of containers available # { # "name": ("last_id", "created_last", ["agent_addr1", "agent_addr2"]) # } self._containers = {} # Containers available per agent {"agent_addr": ["container_id_1", ...]} self._containers_on_agent = {} self._registered_clients = set() # addr of registered clients self._registered_agents = {} # addr of registered agents self._available_agents = [] # addr of available agents self._ping_count = {} # ping count per addr of agents self._waiting_jobs = OrderedDict() # rb queue for waiting jobs format:[(client_addr_as_bytes, ClientNewJob])] self._job_running = {} # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewJob,start_time)}
async def run_queue(context): # Prepare our context and sockets frontend = context.socket(zmq.ROUTER) backend = context.socket(zmq.ROUTER) frontend.bind("tcp://*:5559") backend.bind("tcp://*:5560") # Initialize poll set poller = Poller() poller.register(frontend, zmq.POLLIN) poller.register(backend, zmq.POLLIN) # Switch messages between sockets while True: socks = await poller.poll() socks = dict(socks) if socks.get(frontend) == zmq.POLLIN: frames = await frontend.recv_multipart() print('received from frontend: {}'.format(frames)) # Add the worker ident to the envelope - simplified for example. frames.insert(0, b'Worker1') await backend.send_multipart(frames) if socks.get(backend) == zmq.POLLIN: frames = await backend.recv_multipart() print('received from backend: {}'.format(frames)) msg = frames[1:] # Slice off worker ident await frontend.send_multipart(msg)
def run_worker(context): poller = Poller() liveness = HEARTBEAT_LIVENESS interval = INTERVAL_INIT heartbeat_at = time.time() + HEARTBEAT_INTERVAL worker = yield from worker_socket(context, poller) cycles = 0 while True: socks = yield from poller.poll(HEARTBEAT_INTERVAL * 1000) socks = dict(socks) # Handle worker activity on backend if socks.get(worker) == zmq.POLLIN: # Get message # - 3-part envelope + content -> request # - 1-part HEARTBEAT -> heartbeat frames = yield from worker.recv_multipart() if not frames: break # Interrupted if len(frames) == 3: # Simulate various problems, after a few cycles cycles += 1 if cycles > 3 and randint(0, 5) == 0: print("I: Simulating a crash") break if cycles > 3 and randint(0, 5) == 0: print("I: Simulating CPU overload") yield from asyncio.sleep(3) print("I: Normal reply") yield from worker.send_multipart(frames) liveness = HEARTBEAT_LIVENESS yield from asyncio.sleep(1) # Do some heavy work elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT: print("I: Queue heartbeat") liveness = HEARTBEAT_LIVENESS else: print("E: Invalid message: %s" % frames) interval = INTERVAL_INIT else: liveness -= 1 if liveness == 0: print("W: Heartbeat failure, can't reach queue") print("W: Reconnecting in %0.2fs..." % interval) yield from asyncio.sleep(interval) if interval < INTERVAL_MAX: interval *= 2 poller.unregister(worker) worker.setsockopt(zmq.LINGER, 0) worker.close() worker = yield from worker_socket(context, poller) liveness = HEARTBEAT_LIVENESS if time.time() > heartbeat_at: heartbeat_at = time.time() + HEARTBEAT_INTERVAL print("I: Worker heartbeat") yield from worker.send(PPP_HEARTBEAT)
def register(self): self.poller = Poller() self.poller.register(self.pull, POLLIN) if settings.SCHEDULER_MONITOR: self.monitor_poller = Poller() self.monitor_poller.register(self.monitor_socket, POLLIN) logger.info('%s - all sockets are successfully registered ' 'in poller objects ...', self)
def __init__(self, context, agent_addr, client_addr): self._content = context self._loop = asyncio.get_event_loop() self._agent_addr = agent_addr self._client_addr = client_addr self._agent_socket = context.socket(zmq.ROUTER) self._client_socket = context.socket(zmq.ROUTER) self._logger = logging.getLogger("inginious.backend") # Enable support for ipv6 self._agent_socket.ipv6 = True self._client_socket.ipv6 = True self._poller = Poller() self._poller.register(self._agent_socket, zmq.POLLIN) self._poller.register(self._client_socket, zmq.POLLIN) # List of containers available # { # "name": ("last_id", "created_last", ["agent_addr1", "agent_addr2"]) # } self._containers = {} # List of batch containers available # { # "name": { # "description": "a description written in RST", # "id": "container img id", # "created": 123456789, # "agents": ["agent_addr1", "agent_addr2"] # "parameters": { # "key": { # "type:" "file", # or "text", # "path": "path/to/file/inside/input/dir", # not mandatory in file, by default "key" # "name": "name of the field", # not mandatory in file, default "key" # "description": "a short description of what this field is used for" # not mandatory, default "" # } # } # } self._batch_containers = {} # Batch containers available per agent {"agent_addr": ["batch_id_1", ...]} self._batch_containers_on_agent = {} # Containers available per agent {"agent_addr": ["container_id_1", ...]} self._containers_on_agent = {} self._registered_clients = set() # addr of registered clients self._registered_agents = {} # addr of registered agents self._available_agents = [] # addr of available agents self._ping_count = {} # ping count per addr of agents self._waiting_jobs = OrderedDict( ) # rb queue for waiting jobs format:[(client_addr_as_bytes, Union[ClientNewJob,ClientNewBatchJob])] self._job_running = { } # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewJob,start_time)} self._batch_job_running = { } # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewBatchJob,
def __init__(self, address): super(Server, self).__init__() self.address = address self.queue = asyncio.Queue() self.context = Context.instance() self.socket = self.context.socket(zmq.PULL) self.poller = Poller() self._listen_future = None
def reinitreq(req=None): if req: req.close() del req req = ctx.socket(zmq.CLIENT) req.connect(url) poller = Poller() poller.register(req, zmq.POLLIN) return req, poller
async def handleZMQMessage(): subscriber = Ctx.socket(zmq.SUB) subscriber.connect("tcp://raspberry_ip:5555") subscriber.setsockopt(zmq.SUBSCRIBE, b"1") poller = Poller() poller.register(subscriber, zmq.POLLOUT) while True: topic, data = await subscriber.recv_multipart() await sendPicture(data)
def establish_socket(self): self.context = Context() self.socket = self.context.socket(REP) self.socket.bind(self.address) self.poller = Poller() self.poller.register(self.socket, POLLIN) self.ping = 0 while self.auth() is False: time.sleep(1)
def run(): subscriber = Ctx.socket(zmq.SUB) subscriber.connect(Url) subscription = b"%03d" % 5 subscriber.setsockopt(zmq.SUBSCRIBE, subscription) poller = Poller() poller.register(subscriber, zmq.POLLOUT) while True: topic, data = yield from subscriber.recv_multipart() # assert topic == subscription print(data)
def run(): subscriber = Ctx.socket(zmq.SUB) subscriber.connect(Url) subscription = b"%03d" % 5 subscriber.setsockopt(zmq.SUBSCRIBE, subscription) poller = Poller() poller.register(subscriber, zmq.POLLOUT) while True: topic, data = yield from subscriber.recv_multipart() #assert topic == subscription print(data)
async def receiver(): """receive messages with polling""" pull = ctx.socket(zmq.PULL) pull.connect(url) poller = Poller() poller.register(pull, zmq.POLLIN) while True: events = await poller.poll() if pull in dict(events): print("recving", events) msg = await pull.recv_multipart() print('recvd', msg)
async def receiver(response): print('listening on {}'.format(url)) """receive messages with polling""" pull = ctx.socket(zmq.PULL) pull.connect(url) poller = Poller() poller.register(pull, zmq.POLLIN) while True: body = await req.stream.get() events = await poller.poll() msg = await pull.recv_multipart() if pull in dict( events) else 'waiting ...' response.write(msg)
def __init__(self, url, logger, request_timeout=None, database=None): self._logger = logger self._database = database self._context = Context.instance() self._poller = Poller() self._request = self._context.socket(zmq.DEALER) self._request_timeout = request_timeout or 60 self._rds_bus_url = url self._request.connect(self._rds_bus_url) self._request_dict = dict() self._io_loop = asyncio.get_event_loop() self._running = False asyncio.ensure_future(self.start())
def run_queue(): context = Context(1) frontend = context.socket(zmq.ROUTER) # ROUTER backend = context.socket(zmq.ROUTER) # ROUTER frontend.bind("tcp://*:5555") # For clients backend.bind("tcp://*:5556") # For workers poll_workers = Poller() poll_workers.register(backend, zmq.POLLIN) poll_both = Poller() poll_both.register(frontend, zmq.POLLIN) poll_both.register(backend, zmq.POLLIN) workers = [] while True: if workers: socks = yield from poll_both.poll() else: socks = yield from poll_workers.poll() socks = dict(socks) # Handle worker activity on backend if socks.get(backend) == zmq.POLLIN: # Use worker address for LRU routing msg = yield from backend.recv_multipart() if not msg: break print('I: received msg: {}'.format(msg)) address = msg[0] workers.append(address) # Everything after the second (delimiter) frame is reply reply = msg[2:] # Forward message to client if it's not a READY if reply[0] != LRU_READY: print('I: sending -- reply: {}'.format(reply)) yield from frontend.send_multipart(reply) else: print('I: received ready -- address: {}'.format(address)) if socks.get(frontend) == zmq.POLLIN: # Get client request, route to first available worker msg = yield from frontend.recv_multipart() worker = workers.pop(0) request = [worker, b''] + msg print('I: sending -- worker: {} msg: {}'.format(worker, msg)) yield from backend.send_multipart(request)
class Server(Killable): _event_class = asyncio.Event def __init__(self, address): super(Server, self).__init__() self.address = address self.queue = asyncio.Queue() self.context = Context.instance() self.socket = self.context.socket(zmq.PULL) self.poller = Poller() self._listen_future = None async def _receive_into_queue(self): while self.alive: try: events = await self.poller.poll(timeout=1e-4) if self.socket in dict(events): data = await self.socket.recv() await self.queue.put(data) except zmq.error.ZMQError: await asyncio.sleep(1e-4) async def start(self): self.socket.bind('tcp://%s:%s' % self.address) self.poller.register(self.socket, zmq.POLLIN) self._listen_future = asyncio.ensure_future(self._receive_into_queue()) await asyncio.sleep(0) async def shutdown(self): self.kill() self.poller.unregister(self.socket) if self._listen_future is not None and not self._listen_future.done(): self._listen_future.cancel() self.socket.close(linger=0) await asyncio.sleep(0) def iter_messages(self): async def wrapped(): while self.alive: try: result = self.queue.get_nowait() except asyncio.QueueEmpty: await asyncio.sleep(1e-6) else: await asyncio.sleep(0) return result return AsyncGenerator(wrapped)
async def app(store: StorageServerStore, context: Context, name: str, command: str, arg: str): print("Starting...") dirserv_commands = context.socket(zmq.REQ) dirserv_commands.connect("tcp://127.0.0.1:5350") print("App is started") if command == "declare": self_addr = await asyncio.wait_for(ping(dirserv_commands, name), 5) print("Directory server report this client is run on {}".format( self_addr)) command_port: Socket = context.socket(zmq.ROUTER) port = command_port.bind_to_random_port("tcp://127.0.0.1") self_entrypoint_addr = "tcp://{}:{}".format(self_addr, port) await asyncio.wait_for( cast_address(dirserv_commands, name, self_entrypoint_addr), 5) print("Address {} casted on directory server".format( self_entrypoint_addr)) with open(arg, mode='rb') as f: store.files[arg] = VirtualFile(arg, f.read(), []) await declare_file(dirserv_commands, arg, name) print("File {} is declared, serving file...".format(arg)) poller = Poller() poller.register(command_port, zmq.POLLIN) while True: events: List[Tuple[Socket, int]] = await poller.poll() for socket, mark in events: frames: List[Frame] = await socket.recv_multipart(copy=False) id_frame = frames.pop(0) frames.pop(0) command_frame = frames.pop(0) command = str(command_frame.bytes, 'utf8') if socket == command_port: if command == 'fs.read_file': await read_file_handler(store, frames, socket, id_frame) elif command == "disown": await disown_file(dirserv_commands, arg, name) context.destroy() return elif command == "show": content = await download_file(context, dirserv_commands, arg) print("==== Content of '{}' ====".format(arg)) print(str(content, 'utf8')) context.destroy() return else: print("Unknown command {}".format(command)) context.destroy() return
def reconnect(self): '''connects or reconnects to the broker''' if self.poller: self.poller.unregister(self.socket) if self.socket: self.socket.close() self.context = Context() self.socket = self.context.socket(zmq.DEALER) self.socket.connect(self.QWeatherStationIP + ':' + self.QWeatherStationSocket) self.subsocket = self.context.socket(zmq.SUB) self.subsocket.connect(self.QWeatherStationIP + ':' + str(int(self.QWeatherStationSocket) + SUBSOCKET)) self.poller = Poller() self.poller.register(self.socket,zmq.POLLIN) self.poller.register(self.subsocket,zmq.POLLIN)
async def receiver(name=''): """receive messages with polling""" rep = ctx.socket(zmq.STREAM) for url in urls: rep.connect(url) poller = Poller() poller.register(rep, zmq.POLLIN) cnt = 0 while True: cnt += 1 events = await poller.poll() if rep in dict(events): msg = rep.recv().result() print(name, 'recvd', recv_zipped_pickle(msg)) print(name, 'send:', cnt) rep.send(send_zipped_pickle(cnt), routing_id=msg.routing_id)
def run_queue(context): frontend = context.socket(zmq.ROUTER) # ROUTER backend = context.socket(zmq.ROUTER) # ROUTER frontend.bind(FRONT_END_ADDRESS) # For clients backend.bind(BACK_END_ADDRESS) # For workers poll_workers = Poller() poll_workers.register(backend, zmq.POLLIN) poll_both = Poller() poll_both.register(frontend, zmq.POLLIN) poll_both.register(backend, zmq.POLLIN) workers = WorkerQueue() heartbeat_at = time.time() + HEARTBEAT_INTERVAL while True: if len(workers.queue) > 0: poller = poll_both else: poller = poll_workers socks = yield from poller.poll(HEARTBEAT_INTERVAL * 1000) socks = dict(socks) # Handle worker activity on backend if socks.get(backend) == zmq.POLLIN: # Use worker address for LRU routing frames = yield from backend.recv_multipart() if not frames: break address = frames[0] workers.ready(Worker(address)) # Validate control message, or return reply to client msg = frames[1:] if len(msg) == 1: if msg[0] not in (PPP_READY, PPP_HEARTBEAT): print("E: Invalid message from worker: %s" % msg) else: yield from frontend.send_multipart(msg) # Send heartbeats to idle workers if it's time if time.time() >= heartbeat_at: for worker in workers.queue: msg = [worker, PPP_HEARTBEAT] yield from backend.send_multipart(msg) heartbeat_at = time.time() + HEARTBEAT_INTERVAL if socks.get(frontend) == zmq.POLLIN: frames = yield from frontend.recv_multipart() if not frames: break frames.insert(0, next(workers)) backend.send_multipart(frames) workers.purge()
async def receiver(name=''): """receive messages with polling""" rep = ctx.socket(zmq.REP) for url in urls: rep.connect(url) poller = Poller() poller.register(rep, zmq.POLLIN) cnt = 0 while True: cnt += 1 events = await poller.poll() if rep in dict(events): msg = await rep.recv() print(name, 'recvd', recv_zipped_pickle(msg)) print(name, 'send:', cnt) A = numpy.ones((1024, 1024)) rep.send(send_zipped_pickle([cnt, A]))
def __init__(self, context, agent_addr, client_addr): self._content = context self._loop = asyncio.get_event_loop() self._agent_addr = agent_addr self._client_addr = client_addr self._agent_socket = context.socket(zmq.ROUTER) self._client_socket = context.socket(zmq.ROUTER) self._logger = logging.getLogger("inginious.backend") # Enable support for ipv6 self._agent_socket.ipv6 = True self._client_socket.ipv6 = True self._poller = Poller() self._poller.register(self._agent_socket, zmq.POLLIN) self._poller.register(self._client_socket, zmq.POLLIN) # Dict of available environments # { # "name": ("last_id", "created_last", ["agent_addr1", "agent_addr2"], "type") # } self._environments = {} self._registered_clients = set() # addr of registered clients # Dict of registered agents # { # agent_address: {"name": "friendly_name", "environments": environment_dict} # } environment_dict is a described in AgentHello. self._registered_agents = {} # addr of available agents. May contain multiple times the same agent, because some agent can # manage multiple jobs at once! self._available_agents = [] # ping count per addr of agents self._ping_count = {} # These two share the same objects! Tuples should never be recreated. self._waiting_jobs_pq = TopicPriorityQueue( ) # priority queue for waiting jobs self._waiting_jobs = { } # mapping job to job message, with key: [(client_addr_as_bytes, ClientNewJob])] self._job_running = { } # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewJob,start_time)}
def run_worker(context): # Socket to receive messages on receiver = context.socket(zmq.PULL) receiver.connect("tcp://localhost:5557") # Socket to send messages to sender = context.socket(zmq.PUSH) sender.connect("tcp://localhost:5558") # Socket for control input controller = context.socket(zmq.SUB) controller.connect("tcp://localhost:5559") controller.setsockopt(zmq.SUBSCRIBE, b"") # Process messages from receiver and controller poller = Poller() poller.register(receiver, zmq.POLLIN) poller.register(controller, zmq.POLLIN) # Process messages from both sockets while True: socks = yield from poller.poll() socks = dict(socks) if socks.get(receiver) == zmq.POLLIN: message = yield from receiver.recv() # Process task workload = int(message) # Workload in msecs # Do the work yield from asyncio.sleep(workload / 1000.0) # Send results to sink yield from sender.send(message) # Simple progress indicator for the viewer sys.stdout.write(".") sys.stdout.flush() # Any waiting controller command acts as 'KILL' if socks.get(controller) == zmq.POLLIN: break
def __init__(self, IP, loop=None, verbose=False, debug=False): if loop is None: #from zmq import Context,Poller # import asyncio # from zmq.asyncio import Context,Poller asyncio.set_event_loop_policy( asyncio.WindowsSelectorEventLoopPolicy()) self.loop = asyncio.get_event_loop() else: self.loop = loop IpAndPort = re.search(IPREPATTERN, IP) assert IpAndPort != None, 'Ip not understood (tcp://xxx.xxx.xxx.xxx:XXXX or txp://*:XXXX)' self.StationIP = IpAndPort.group(1) self.StationSocket = IpAndPort.group(2) assert self.StationIP[: 6] == 'tcp://', 'Ip not understood (tcp://xxx.xxx.xxx.xxx:XXXX or txp://*:XXXX)' assert len( self.StationSocket ) == 4, 'Port not understood (tcp://xxx.xxx.xxx.xxx:XXXX or txp://*:XXXX)' formatting = '{:}: %(levelname)s: %(message)s'.format( 'QWeatherStation') if debug: logging.basicConfig(format=formatting, level=logging.DEBUG) if verbose: logging.basicConfig(format=formatting, level=logging.INFO) self.servers = {} # key:value = clientaddress:value, bytes:string self.clients = {} # key:value = clientaddress:value, bytes:string self.servermethods = {} self.serverjobs = {} self.pinged = [] self.requesttimeoutdict = {} self.cnx = Context() self.socket = self.cnx.socket(zmq.ROUTER) self.socket.bind(self.StationIP + ':' + self.StationSocket) self.proxy = ThreadProxy(zmq.XSUB, zmq.XPUB) self.proxy.bind_in(self.StationIP + ':' + str(int(self.StationSocket) + PUBLISHSOCKET)) self.proxy.bind_out(self.StationIP + ':' + str(int(self.StationSocket) + SUBSOCKET)) self.proxy.start() self.poller = Poller() self.poller.register(self.socket, zmq.POLLIN) logging.info('Ready to run on IP: {:}'.format(self.get_own_ip()))
def run(ident): # Socket to talk to server print("Connecting to hello world server. Ctrl-C to exit early.\n") socket = Ctx.socket(zmq.REQ) socket.connect(Url) poller = Poller() poller.register(socket, zmq.POLLOUT) # Do multiple requests, waiting each time for a response for request in range(10): message = '{} Hello {}'.format(ident, request) message = message.encode('utf-8') print("Sending message: {}".format(message)) yield from socket.send(message) # Get the reply. message = yield from socket.recv() print("Received reply: {}".format(message)) print('exiting') return 'nothing'
async def receiver(): global request pull = context.socket(zmq.PULL) pull.connect(addr) poller = Poller() poller.register(pull, zmq.POLLIN) while True: print("Sending request %s >>>\n" % request) events = await poller.poll() if pull in dict(events): messages_received = pull.recv_multipart() for message_received in messages_received: print("<<< RECEIVED BYTES\n", message_received) unpacked_maessage_split = message_received.decode().split(",") print(f'MESSAGE DECODED SPLIT: {unpacked_maessage_split}\n') request += 1
def run(): print("Getting ready for hello world client. Ctrl-C to exit.\n") socket = Ctx.socket(zmq.REP) socket.bind(Url) poller = Poller() poller.register(socket, zmq.POLLIN) while True: # Wait for next request from client message = yield from socket.recv() print("Received request: {}".format(message)) # Do some 'work' yield from asyncio.sleep(1) # Send reply back to client message = message.decode('utf-8') message = '{}, world'.format(message) message = message.encode('utf-8') print("Sending reply: {}".format(message)) yield from socket.send(message)
async def execute_parser_request(endpoint, parameters, timeout=5): socket = TickerParser.zmqContext.socket(REQ) socket.connect("tcp://parser:6900") socket.setsockopt(LINGER, 0) poller = Poller() poller.register(socket, POLLIN) await socket.send_multipart([endpoint] + parameters) responses = await poller.poll(timeout * 1000) if len(responses) != 0: response = await socket.recv_multipart() socket.close() return response else: socket.close() raise Exception("time out") return None
async def worker_task( worker_url, ident ): #receive the request from the client or send the message to backend #context = zmq.Context(1) context = Context.instance() worker = context.socket(zmq.DEALER) worker.identity = u"Worker-{}".format(ident).encode("ascii") worker.connect(worker_url) # Tell broker we're ready for work worker.send(PPP_READY) poll_worker = Poller() poll_worker.register(worker, zmq.POLLIN) while True: #socks = dict(poll_worker.poll(HEARTBEAT_INTERVAL * 1000)) socks = dict(await poll_worker.poll(HEARTBEAT_INTERVAL * 1000)) #frames = worker.recv_multipart() # format:[b'client',b'',message_body] print("{}".format(worker.identity.decode("ascii"))) if socks.get(worker) == zmq.POLLIN: frames = await worker.recv_multipart() #print("worker gets:",frames) if not frames: break # Interrupted if len(frames) == 3: print("I: Normal reply") t1 = time.time() image = Image.open(BytesIO(frames[2])) result = crnn_recognition(image, model) t2 = time.time() - t1 print("recogition time:", t2) image.save("/home/cad488/test_images/" + time.asctime(time.localtime(time.time())) + ".jpg") await worker.send_multipart( [frames[0], b"", result.encode("ascii")]) else: print("E: Invalid message: %s" % frames)
def run_client(context): print("I: Connecting to server...") client = context.socket(zmq.REQ) client.connect(SERVER_ENDPOINT) poll = Poller() poll.register(client, zmq.POLLIN) sequence = 0 retries_left = REQUEST_RETRIES while retries_left: sequence += 1 request = str(sequence) print("I: Sending (%s)" % request) yield from client.send_string(request) expect_reply = True while expect_reply: socks = yield from poll.poll(REQUEST_TIMEOUT) socks = dict(socks) if socks.get(client) == zmq.POLLIN: reply = yield from client.recv() if not reply: break if int(reply) == sequence: print("I: Server replied OK (%s)" % reply) retries_left = REQUEST_RETRIES expect_reply = False else: print("E: Malformed reply from server: %s" % reply) else: print("W: No response from server, retrying...") # Socket is confused. Close and remove it. print('W: confused') client.setsockopt(zmq.LINGER, 0) client.unbind(SERVER_ENDPOINT) #client.close() poll.unregister(client) retries_left -= 1 if retries_left == 0: print("E: Server seems to be offline, abandoning") return print("I: Reconnecting and resending (%s)" % request) # Create new connection client = context.socket(zmq.REQ) client.connect(SERVER_ENDPOINT) poll.register(client, zmq.POLLIN) yield from client.send_string(request)
async def process_request(service, authorId, request, timeout=60, retries=3): socket = Processor.zmqContext.socket(REQ) payload, responseText = None, None socket.connect(Processor.services[service]) socket.setsockopt(LINGER, 0) poller = Poller() poller.register(socket, POLLIN) request["timestamp"] = time() request["authorId"] = authorId await socket.send_multipart( [Processor.clientId, service.encode(), dumps(request)]) responses = await poller.poll(timeout * 1000) if len(responses) != 0: payload, responseText = await socket.recv_multipart() socket.close() payload = loads(payload) payload = payload if bool(payload) else None responseText = None if responseText == b"" else responseText.decode( ) if service in ["chart", "heatmap", "depth" ] and payload is not None: payload["data"] = BytesIO(decodebytes( payload["data"].encode())) else: socket.close() if retries == 1: raise Exception("time out") else: payload, responseText = await Processor.process_request( service, authorId, request, retries=retries - 1) return payload, responseText
async def execute_database_request(endpoint, parameters, timeout=1): socket = DatabaseConnector.zmqContext.socket(REQ) payload, responseText = None, None socket.connect("tcp://database:6900") socket.setsockopt(LINGER, 0) poller = Poller() poller.register(socket, POLLIN) await socket.send_multipart([ endpoint, bytes(str(int((time() + timeout) * 1000)), encoding='utf8'), parameters ]) responses = await poller.poll(timeout * 1000) if len(responses) != 0: [response] = await socket.recv_multipart() socket.close() return loads(response) else: socket.close() return None
def run_server(loop): """Server routine""" # Prepare our context and sockets # Socket to talk to clients clients = Ctx.socket(zmq.ROUTER) clients.bind(Url_client) workers = Ctx.socket(zmq.DEALER) workers.bind(Url_worker) # Start the workers tasks = [] for idx in range(5): ident = 'worker {}'.format(idx) task = asyncio.ensure_future(run_worker(ident)) tasks.append(task) poller = Poller() poller.register(clients, zmq.POLLIN) poller.register(workers, zmq.POLLIN) print('mtserver ready for requests') while True: events = yield from poller.poll() events = dict(events) if clients in events: message = yield from clients.recv_multipart() printdbg('(run) received from client message_parts: {}'.format( message)) client, empty, message = message[:3] printdbg('(run) received from client message: {}'.format( message)) printdbg('(run) sending message to workers: {}'.format(message)) yield from workers.send_multipart([client, b'', message]) elif workers in events: message = yield from workers.recv_multipart() printdbg('(run) received from worker message_parts: {}'.format( message)) client, empty, message = message[:3] printdbg('(run) received from worker message: {}'.format( message)) yield from clients.send_multipart([client, b'', message]) printdbg('(run) sent message to clients: {}'.format(message))
def run_proxy(socket_from, socket_to): poller = Poller() poller.register(socket_from, zmq.POLLIN) poller.register(socket_to, zmq.POLLIN) printdbg('(run_proxy) started') while True: events = yield from poller.poll() events = dict(events) if socket_from in events: msg = yield from socket_from.recv_multipart() printdbg('(run_proxy) received from frontend -- msg: {}'.format( msg)) yield from socket_to.send_multipart(msg) printdbg('(run_proxy) sent to backend -- msg: {}'.format(msg)) elif socket_to in events: msg = yield from socket_to.recv_multipart() printdbg('(run_proxy) received from backend -- msg: {}'.format( msg)) yield from socket_from.send_multipart(msg) printdbg('(run_proxy) sent to frontend -- msg: {}'.format(msg))
def run_broker(context): # Prepare our context and sockets frontend = context.socket(zmq.ROUTER) backend = context.socket(zmq.DEALER) frontend.bind("tcp://*:5559") backend.bind("tcp://*:5560") # Initialize poll set poller = Poller() poller.register(frontend, zmq.POLLIN) poller.register(backend, zmq.POLLIN) # Switch messages between sockets while True: socks = yield from poller.poll() socks = dict(socks) if socks.get(frontend) == zmq.POLLIN: message = yield from frontend.recv_multipart() print('received from frontend: {}'.format(message)) yield from backend.send_multipart(message) if socks.get(backend) == zmq.POLLIN: message = yield from backend.recv_multipart() print('received from backend: {}'.format(message)) yield from frontend.send_multipart(message)
def start(self): """Start ZAP authentication""" super().start() self.__poller = Poller() self.__poller.register(self.zap_socket, zmq.POLLIN) self.__task = asyncio.ensure_future(self.__handle_zap())
class Backend(object): """ Backend. Central point of the architecture, manages communication between clients (frontends) and agents. Schedule jobs on agents. """ def __init__(self, context, agent_addr, client_addr): self._content = context self._loop = asyncio.get_event_loop() self._agent_addr = agent_addr self._client_addr = client_addr self._agent_socket = context.socket(zmq.ROUTER) self._client_socket = context.socket(zmq.ROUTER) self._logger = logging.getLogger("inginious.backend") # Enable support for ipv6 self._agent_socket.ipv6 = True self._client_socket.ipv6 = True self._poller = Poller() self._poller.register(self._agent_socket, zmq.POLLIN) self._poller.register(self._client_socket, zmq.POLLIN) # List of containers available # { # "name": ("last_id", "created_last", ["agent_addr1", "agent_addr2"]) # } self._containers = {} # Containers available per agent {"agent_addr": ["container_id_1", ...]} self._containers_on_agent = {} self._registered_clients = set() # addr of registered clients self._registered_agents = {} # addr of registered agents self._available_agents = [] # addr of available agents self._ping_count = {} # ping count per addr of agents self._waiting_jobs = OrderedDict() # rb queue for waiting jobs format:[(client_addr_as_bytes, ClientNewJob])] self._job_running = {} # indicates on which agent which job is running. format: {BackendJobId:(addr_as_bytes,ClientNewJob,start_time)} async def handle_agent_message(self, agent_addr, message): """Dispatch messages received from agents to the right handlers""" message_handlers = { AgentHello: self.handle_agent_hello, AgentJobStarted: self.handle_agent_job_started, AgentJobDone: self.handle_agent_job_done, AgentJobSSHDebug: self.handle_agent_job_ssh_debug, Pong: self._handle_pong } try: func = message_handlers[message.__class__] except: raise TypeError("Unknown message type %s" % message.__class__) self._create_safe_task(func(agent_addr, message)) async def handle_client_message(self, client_addr, message): """Dispatch messages received from clients to the right handlers""" # Verify that the client is registered if message.__class__ != ClientHello and client_addr not in self._registered_clients: await ZMQUtils.send_with_addr(self._client_socket, client_addr, Unknown()) return message_handlers = { ClientHello: self.handle_client_hello, ClientNewJob: self.handle_client_new_job, ClientKillJob: self.handle_client_kill_job, ClientGetQueue: self.handle_client_get_queue, Ping: self.handle_client_ping } try: func = message_handlers[message.__class__] except: raise TypeError("Unknown message type %s" % message.__class__) self._create_safe_task(func(client_addr, message)) async def send_container_update_to_client(self, client_addrs): """ :param client_addrs: list of clients to which we should send the update """ self._logger.debug("Sending containers updates...") available_containers = tuple(self._containers.keys()) msg = BackendUpdateContainers(available_containers) for client in client_addrs: await ZMQUtils.send_with_addr(self._client_socket, client, msg) async def handle_client_hello(self, client_addr, _: ClientHello): """ Handle an ClientHello message. Send available containers to the client """ self._logger.info("New client connected %s", client_addr) self._registered_clients.add(client_addr) await self.send_container_update_to_client([client_addr]) async def handle_client_ping(self, client_addr, _: Ping): """ Handle an Ping message. Pong the client """ await ZMQUtils.send_with_addr(self._client_socket, client_addr, Pong()) async def handle_client_new_job(self, client_addr, message: ClientNewJob): """ Handle an ClientNewJob message. Add a job to the queue and triggers an update """ self._logger.info("Adding a new job %s %s to the queue", client_addr, message.job_id) self._waiting_jobs[(client_addr, message.job_id)] = message await self.update_queue() async def handle_client_kill_job(self, client_addr, message: ClientKillJob): """ Handle an ClientKillJob message. Remove a job from the waiting list or send the kill message to the right agent. """ # Check if the job is not in the queue if (client_addr, message.job_id) in self._waiting_jobs: del self._waiting_jobs[(client_addr, message.job_id)] # Do not forget to send a JobDone await ZMQUtils.send_with_addr(self._client_socket, client_addr, BackendJobDone(message.job_id, ("killed", "You killed the job"), 0.0, {}, {}, {}, "", None, "", "")) # If the job is running, transmit the info to the agent elif (client_addr, message.job_id) in self._job_running: agent_addr = self._job_running[(client_addr, message.job_id)][0] await ZMQUtils.send_with_addr(self._agent_socket, agent_addr, BackendKillJob((client_addr, message.job_id))) else: self._logger.warning("Client %s attempted to kill unknown job %s", str(client_addr), str(message.job_id)) async def handle_client_get_queue(self, client_addr, _: ClientGetQueue): """ Handles a ClientGetQueue message. Send back info about the job queue""" #jobs_running: a list of tuples in the form #(job_id, is_current_client_job, agent_name, info, launcher, started_at, max_end) jobs_running = list() for backend_job_id, content in self._job_running.items(): jobs_running.append((content[1].job_id, backend_job_id[0] == client_addr, self._registered_agents[content[0]], content[1].course_id+"/"+content[1].task_id, content[1].launcher, int(content[2]), int(content[2])+content[1].time_limit)) #jobs_waiting: a list of tuples in the form #(job_id, is_current_client_job, info, launcher, max_time) jobs_waiting = list() for job_client_addr, msg in self._waiting_jobs.items(): if isinstance(msg, ClientNewJob): jobs_waiting.append((msg.job_id, job_client_addr[0] == client_addr, msg.course_id+"/"+msg.task_id, msg.launcher, msg.time_limit)) await ZMQUtils.send_with_addr(self._client_socket, client_addr, BackendGetQueue(jobs_running, jobs_waiting)) async def update_queue(self): """ Send waiting jobs to available agents """ # For now, round-robin not_found_for_agent = [] while len(self._available_agents) > 0 and len(self._waiting_jobs) > 0: agent_addr = self._available_agents.pop(0) # Find first job that can be run on this agent found = False client_addr, job_id, job_msg = None, None, None for (client_addr, job_id), job_msg in self._waiting_jobs.items(): if job_msg.environment in self._containers_on_agent[agent_addr]: found = True break if not found: self._logger.debug("Nothing to do for agent %s", agent_addr) not_found_for_agent.append(agent_addr) continue # Remove the job from the queue del self._waiting_jobs[(client_addr, job_id)] job_id = (client_addr, job_msg.job_id) self._job_running[job_id] = (agent_addr, job_msg, time.time()) self._logger.info("Sending job %s %s to agent %s", client_addr, job_msg.job_id, agent_addr) await ZMQUtils.send_with_addr(self._agent_socket, agent_addr, BackendNewJob(job_id, job_msg.course_id, job_msg.task_id, job_msg.inputdata, job_msg.environment, job_msg.enable_network, job_msg.time_limit, job_msg.hard_time_limit, job_msg.mem_limit, job_msg.debug)) # Do not forget to add again for which we did not find jobs to do self._available_agents += not_found_for_agent async def handle_agent_hello(self, agent_addr, message: AgentHello): """ Handle an AgentAvailable message. Add agent_addr to the list of available agents """ self._logger.info("Agent %s (%s) said hello", agent_addr, message.friendly_name) if agent_addr in self._registered_agents: # Delete previous instance of this agent, if any await self._delete_agent(agent_addr) self._registered_agents[agent_addr] = message.friendly_name self._available_agents.extend([agent_addr for _ in range(0, message.available_job_slots)]) self._containers_on_agent[agent_addr] = message.available_containers.keys() self._ping_count[agent_addr] = 0 # update information about available containers for container_name, container_info in message.available_containers.items(): if container_name in self._containers: # check if the id is the same if self._containers[container_name][0] == container_info["id"]: # ok, just add the agent to the list of agents that have the container self._logger.debug("Registering container %s for agent %s", container_name, str(agent_addr)) self._containers[container_name][2].append(agent_addr) elif self._containers[container_name][1] > container_info["created"]: # containers stored have been created after the new one # add the agent, but emit a warning self._logger.warning("Container %s has multiple version: \n" "\t Currently registered agents have version %s (%i)\n" "\t New agent %s has version %s (%i)", container_name, self._containers[container_name][0], self._containers[container_name][1], str(agent_addr), container_info["id"], container_info["created"]) self._containers[container_name][2].append(agent_addr) else: # self._containers[container_name][1] < container_info["created"]: # containers stored have been created before the new one # add the agent, update the infos, and emit a warning self._logger.warning("Container %s has multiple version: \n" "\t Currently registered agents have version %s (%i)\n" "\t New agent %s has version %s (%i)", container_name, self._containers[container_name][0], self._containers[container_name][1], str(agent_addr), container_info["id"], container_info["created"]) self._containers[container_name] = (container_info["id"], container_info["created"], self._containers[container_name][2] + [agent_addr]) else: # just add it self._logger.debug("Registering container %s for agent %s", container_name, str(agent_addr)) self._containers[container_name] = (container_info["id"], container_info["created"], [agent_addr]) # update the queue await self.update_queue() # update clients await self.send_container_update_to_client(self._registered_clients) async def handle_agent_job_started(self, agent_addr, message: AgentJobStarted): """Handle an AgentJobStarted message. Send the data back to the client""" self._logger.debug("Job %s %s started on agent %s", message.job_id[0], message.job_id[1], agent_addr) await ZMQUtils.send_with_addr(self._client_socket, message.job_id[0], BackendJobStarted(message.job_id[1])) async def handle_agent_job_done(self, agent_addr, message: AgentJobDone): """Handle an AgentJobDone message. Send the data back to the client, and start new job if needed""" if agent_addr in self._registered_agents: self._logger.info("Job %s %s finished on agent %s", message.job_id[0], message.job_id[1], agent_addr) # Remove the job from the list of running jobs del self._job_running[message.job_id] # Sent the data back to the client await ZMQUtils.send_with_addr(self._client_socket, message.job_id[0], BackendJobDone(message.job_id[1], message.result, message.grade, message.problems, message.tests, message.custom, message.state, message.archive, message.stdout, message.stderr)) # The agent is available now self._available_agents.append(agent_addr) else: self._logger.warning("Job result %s %s from non-registered agent %s", message.job_id[0], message.job_id[1], agent_addr) # update the queue await self.update_queue() async def handle_agent_job_ssh_debug(self, _, message: AgentJobSSHDebug): """Handle an AgentJobSSHDebug message. Send the data back to the client""" await ZMQUtils.send_with_addr(self._client_socket, message.job_id[0], BackendJobSSHDebug(message.job_id[1], message.host, message.port, message.password)) async def run(self): self._logger.info("Backend started") self._agent_socket.bind(self._agent_addr) self._client_socket.bind(self._client_addr) self._loop.call_later(1, self._create_safe_task, self._do_ping()) try: while True: socks = await self._poller.poll() socks = dict(socks) # New message from agent if self._agent_socket in socks: agent_addr, message = await ZMQUtils.recv_with_addr(self._agent_socket) await self.handle_agent_message(agent_addr, message) # New message from client if self._client_socket in socks: client_addr, message = await ZMQUtils.recv_with_addr(self._client_socket) await self.handle_client_message(client_addr, message) except asyncio.CancelledError: return except KeyboardInterrupt: return async def _handle_pong(self, agent_addr, _ : Pong): """ Handle a pong """ self._ping_count[agent_addr] = 0 async def _do_ping(self): """ Ping the agents """ # the list() call here is needed, as we remove entries from _registered_agents! for agent_addr, friendly_name in list(self._registered_agents.items()): try: ping_count = self._ping_count.get(agent_addr, 0) if ping_count > 5: self._logger.warning("Agent %s (%s) does not respond: removing from list.", agent_addr, friendly_name) delete_agent = True else: self._ping_count[agent_addr] = ping_count + 1 await ZMQUtils.send_with_addr(self._agent_socket, agent_addr, Ping()) delete_agent = False except: # This should not happen, but it's better to check anyway. self._logger.exception("Failed to send ping to agent %s (%s). Removing it from list.", agent_addr, friendly_name) delete_agent = True if delete_agent: try: await self._delete_agent(agent_addr) except: self._logger.exception("Failed to delete agent %s (%s)!", agent_addr, friendly_name) self._loop.call_later(1, self._create_safe_task, self._do_ping()) async def _delete_agent(self, agent_addr): """ Deletes an agent """ self._available_agents = [agent for agent in self._available_agents if agent != agent_addr] del self._registered_agents[agent_addr] await self._recover_jobs(agent_addr) async def _recover_jobs(self, agent_addr): """ Recover the jobs sent to a crashed agent """ for (client_addr, job_id), (agent, job_msg, _) in reversed(list(self._job_running.items())): if agent == agent_addr: await ZMQUtils.send_with_addr(self._client_socket, client_addr, BackendJobDone(job_id, ("crash", "Agent restarted"), 0.0, {}, {}, {}, "", None, None, None)) del self._job_running[(client_addr, job_id)] await self.update_queue() def _create_safe_task(self, coroutine): """ Calls self._loop.create_task with a safe (== with logged exception) coroutine """ task = self._loop.create_task(coroutine) task.add_done_callback(self.__log_safe_task) return task def __log_safe_task(self, task): exception = task.exception() if exception is not None: self._logger.exception("An exception occurred while running a Task", exc_info=exception)
class SchedulerConnection(object): __slots__ = ( 'address', # context object to open socket connections 'context', # pull socket to receive check definitions from scheduler 'pull', # poller object for `pull` socket 'poller', # monitor socket for `pull` socket 'monitor_socket', # poller object for monitor socket 'monitor_poller', 'first_missing', ) def __init__(self, address): self.pull = self.poller = None self.monitor_poller = self.monitor_socket = None self.address = address self.context = Context.instance() self.open() self.first_missing = None def __str__(self): return self.address def __repr__(self): return 'Scheduler({})'.format(self.address) def open(self): self.pull = self.context.socket(PULL) logger.info('%s - opening pull socket ...', self) self.pull.connect(self.address) if settings.SCHEDULER_MONITOR: logger.info('%s - opening monitor socket ...', self) self.monitor_socket = self.pull.get_monitor_socket( events=EVENT_DISCONNECTED ) self.register() def register(self): self.poller = Poller() self.poller.register(self.pull, POLLIN) if settings.SCHEDULER_MONITOR: self.monitor_poller = Poller() self.monitor_poller.register(self.monitor_socket, POLLIN) logger.info('%s - all sockets are successfully registered ' 'in poller objects ...', self) def close(self): """Unregister open sockets from poller objects and close them.""" self.unregister() logger.info('%s - closing open sockets ...', self) self.pull.close() if settings.SCHEDULER_MONITOR: self.monitor_socket.close() logger.info('%s - connection closed successfully ...', self) def unregister(self): """Unregister open sockets from poller object.""" logger.info('%s - unregistering sockets from poller objects ...', self) self.poller.unregister(self.pull) if settings.SCHEDULER_MONITOR: self.monitor_poller.unregister(self.monitor_socket) def reconnect(self): self.close() self.open() self.first_missing = None @asyncio.coroutine def receive(self): check = None events = yield from self.poller.poll(timeout=2000) if self.pull in dict(events): check = yield from self.pull.recv_multipart() check = jsonapi.loads(check[0]) if check: self.first_missing = None elif self.first_missing is None: self.first_missing = datetime.now(tz=pytz.utc) if self.first_missing: diff = datetime.now(tz=pytz.utc) - self.first_missing delta = timedelta(minutes=settings.SCHEDULER_LIVENESS_IN_MINUTES) if diff > delta: logger.warning( 'Alamo worker `%s` pid `%s` try to reconnect to ' '`%s` scheduler.', settings.WORKER_FQDN, settings.WORKER_PID, self ) self.reconnect() return check @asyncio.coroutine def receive_event(self): event = None events = yield from self.monitor_poller.poll(timeout=2000) if self.monitor_socket in dict(events): msg = yield from self.monitor_socket.recv_multipart( flags=NOBLOCK) event = parse_monitor_message(msg) return event
def run_broker(loop): """ main broker method """ print('(run_broker) starting') url_worker = "inproc://workers" url_client = "inproc://clients" client_nbr = NBR_CLIENTS * 3 # Prepare our context and sockets context = Context() frontend = context.socket(zmq.ROUTER) frontend.bind(url_client) backend = context.socket(zmq.ROUTER) backend.bind(url_worker) print('(run_broker) creating workers and clients') # create workers and clients threads worker_tasks = [] for idx in range(NBR_WORKERS): task = asyncio.ensure_future(run_worker(url_worker, context, idx)) worker_tasks.append(task) client_tasks = [] for idx in range(NBR_CLIENTS): task = asyncio.ensure_future(run_client(url_client, context, idx)) client_tasks.append(task) print('(run_broker) after creating workers and clients') # Logic of LRU loop # - Poll backend always, frontend only if 1+ worker ready # - If worker replies, queue worker as ready and forward reply # to client if necessary # - If client requests, pop next worker and send request to it # Queue of available workers available_workers = 0 workers_list = [] # init poller poller = Poller() # Always poll for worker activity on backend poller.register(backend, zmq.POLLIN) # Poll front-end only if we have available workers poller.register(frontend, zmq.POLLIN) while True: socks = yield from poller.poll() socks = dict(socks) # Handle worker activity on backend if (backend in socks and socks[backend] == zmq.POLLIN): # Queue worker address for LRU routing message = yield from backend.recv_multipart() assert available_workers < NBR_WORKERS worker_addr = message[0] # add worker back to the list of workers available_workers += 1 workers_list.append(worker_addr) # Second frame is empty empty = message[1] assert empty == b"" # Third frame is READY or else a client reply address client_addr = message[2] # If client reply, send rest back to frontend if client_addr != b'READY': # Following frame is empty empty = message[3] assert empty == b"" reply = message[4] yield from frontend.send_multipart([client_addr, b"", reply]) printdbg('(run_broker) to frontend -- reply: "{}"'.format( reply)) client_nbr -= 1 if client_nbr == 0: printdbg('(run_broker) exiting') break # Exit after N messages # poll on frontend only if workers are available if available_workers > 0: if (frontend in socks and socks[frontend] == zmq.POLLIN): # Now get next client request, route to LRU worker # Client request is [address][empty][request] response = yield from frontend.recv_multipart() [client_addr, empty, request] = response assert empty == b"" # Dequeue and drop the next worker address available_workers += -1 worker_id = workers_list.pop() yield from backend.send_multipart( [worker_id, b"", client_addr, b"", request]) printdbg('(run_broker) to backend -- request: "{}"'.format( request)) #out of infinite loop: do some housekeeping printdbg('(run_broker) finished') for worker_task in worker_tasks: worker_task.cancel() printdbg('(run_broker) workers cancelled') yield from asyncio.sleep(1) frontend.close() backend.close() #context.term() # Caution: calling term() blocks. loop.stop() printdbg('(run_broker) returning') return 'finished ok'