def send_worker(address, send_queue, worker_alive): timing = AccumDict() log = Logger('./var/log/send_worker.log', opt.verbose) ctx = SerializingContext() sender = ctx.socket(zmq.PUSH) sender.connect(address) log(f"Sending to {address}") try: while worker_alive.value: tt = TicToc() try: msg = send_queue.get(timeout=GET_TIMEOUT) except queue.Empty: continue tt.tic() sender.send_data(*msg) timing.add('SEND', tt.toc()) if opt.verbose: Once(timing, log, per=1) except KeyboardInterrupt: log("send_worker: user interrupt") finally: worker_alive.value = 0 sender.disconnect(address) sender.close() ctx.destroy() log("send_worker exit")
class PredictorRemote: def __init__(self, *args, worker_host='localhost', worker_port=DEFAULT_PORT, **kwargs): self.worker_host = worker_host self.worker_port = worker_port self.predictor_args = (args, kwargs) self.timing = AccumDict() self.address = f"tcp://{worker_host}:{worker_port}" self.context = SerializingContext() self.socket = self.context.socket(zmq.PAIR) self.socket.connect(self.address) if not self.check_connection(): self.socket.disconnect(self.address) # TODO: this hangs, as well as context.__del__ self.context.destroy() raise ConnectionError(f"Could not connect to {worker_host}:{worker_port}") log(f"Connected to {self.address}") self.init_worker()
def recv_worker(address, recv_queue, worker_alive): timing = AccumDict() log = Logger('./var/log/recv_worker.log') ctx = SerializingContext() receiver = ctx.socket(zmq.PULL) receiver.connect(address) receiver.RCVTIMEO = RECV_TIMEOUT log(f"Receiving from {address}") try: while worker_alive.value: tt = TicToc() try: tt.tic() msg = receiver.recv_data() timing.add('RECV', tt.toc()) except zmq.error.Again: continue try: recv_queue.put(msg, timeout=PUT_TIMEOUT) except queue.Full: log('recv_queue full') continue if opt.verbose: Once(timing, log, per=1) except KeyboardInterrupt: log("recv_worker: user interrupt") finally: worker_alive.value = 0 receiver.disconnect(address) receiver.close() ctx.destroy() log("recv_worker exit")