def run(self): self.running = True self.register() self.pinger.start() while self.running and not self.pinger.unresponsive: try: events = dict(self.poller.poll(self.timeout)) except zmq.ZMQError: break for socket in events: msg = unserialize(socket.recv()) if msg[0] == 'JOB': # do the job and send the result start = time.time() try: res = self.target(msg[1:]) except Exception, e: # XXX log the error res = str(e) logger.debug('%.6f' % (time.time() - start)) socket.send(serialize("JOBRES", msg[1], res)) else: socket.send('ERROR')
def run(self): self.running = True self.register() self.pinger.start() while self.running and not self.pinger.unresponsive: try: events = dict(self.poller.poll(self.timeout)) except zmq.ZMQError: break for socket in events: msg = unserialize(socket.recv()) logger.debug(msg) if msg[0] == 'JOB': # do the job and send the result start = time.time() try: res = self.target(msg[1:]) except Exception, e: # XXX log the error res = str(e) logger.debug('%.6f' % (time.time() - start)) socket.send(serialize("JOBRES", msg[1], res)) else: socket.send('ERROR')
def _execute(self, job_id, job_data, timeout=1.): worker = None timeout *= 1000. # timeout is in ms data = serialize("JOB", str(job_id), job_data) try: with self.workers.worker() as worker: try: worker.send(data, zmq.NOBLOCK) except zmq.ZMQError, e: raise ExecutionError(str(e)) poller = zmq.Poller() poller.register(worker, zmq.POLLIN) try: events = dict(poller.poll(timeout)) except zmq.ZMQError, e: raise ExecutionError(str(e)) if events == {}: raise TimeoutError() for socket in events: try: msg = unserialize(socket.recv()) except zmq.ZMQError, e: raise ExecutionError(str(e)) if msg[0] == 'JOBRES': # we got a result return msg[-1] else: raise NotImplementedError(str(msg))
def run(self): """Runs the registration. Should not be called directly. Use :func:`start`. """ self.alive = True # channel to communicate with the workers logger.debug('Starting [workermgr]') client = self.context.socket(zmq.REP) client.identity = 'master' client.bind(self.endpoint) poller = zmq.Poller() poller.register(client, zmq.POLLIN) poll_timeout = 1000 while self.alive: try: events = dict(poller.poll(poll_timeout)) except zmq.ZMQError, e: logger.debug("The poll failed") logger.debug(str(e)) break for socket in events: msg = unserialize(socket.recv()) if len(msg) < 2: # XXX log socket.send('ERROR') if msg[-2] == 'PING': logger.debug("[workermgr] Got a PING") if msg[-1] not in self.workers: name = msg[-1] logger.debug("Registered " + name) # keep track of that worker work = self.context.socket(zmq.REQ) work.connect(name) work.identity = name self.workers.add(work) # in any case we pong back logger.debug("[workermgr] sent a PONG") socket.send('PONG') elif msg[-2] == 'REMOVE': if msg[-1] in self.workers: logger.debug("[workermgr] Removing` " + msg[-1]) self.workers.delete(msg[-1]) socket.send('REMOVED') else: logger.debug('Error') socket.send('ERROR') time.sleep(.1)
def run(self): self.alive = True # channel to communicate with the workers logger.debug('Starting [workermgr]') client = self.context.socket(zmq.REP) client.identity = 'master' client.bind(self.endpoint) poller = zmq.Poller() poller.register(client, zmq.POLLIN) poll_timeout = 1000 while self.alive: try: events = dict(poller.poll(poll_timeout)) except zmq.ZMQError, e: logger.debug("The poll failed") logger.debug(str(e)) break for socket in events: msg = unserialize(socket.recv()) if len(msg) < 2: # XXX log socket.send('ERROR') if msg[-2] == 'PING': logger.debug("[workermgr] Got a PING") if msg[-1] not in self.workers: name = msg[-1] logger.debug("Registered " + name) # keep track of that worker work = self.context.socket(zmq.REQ) work.connect(name) work.identity = name self.workers.add(work) # in any case we pong back logger.debug("[workermgr] sent a PONG") socket.send('PONG') elif msg[-2] == 'REMOVE': if msg[-1] in self.workers: logger.debug("[workermgr] Removing` " + msg[-1]) self.workers.delete(msg[-1]) socket.send('REMOVED') else: logger.debug('Error') socket.send('ERROR') time.sleep(.1)