예제 #1
0
파일: worker.py 프로젝트: almet/powerhose
    def run(self):
        self.running = True
        self.register()
        self.pinger.start()

        while self.running and not self.pinger.unresponsive:
            try:
                events = dict(self.poller.poll(self.timeout))
            except zmq.ZMQError:
                break

            for socket in events:
                msg = unserialize(socket.recv())

                if msg[0] == 'JOB':
                    # do the job and send the result
                    start = time.time()
                    try:
                        res = self.target(msg[1:])
                    except Exception, e:
                        # XXX log the error
                        res = str(e)
                    logger.debug('%.6f' % (time.time() - start))
                    socket.send(serialize("JOBRES", msg[1], res))
                else:
                    socket.send('ERROR')
예제 #2
0
파일: worker.py 프로젝트: jrgm/powerhose
    def run(self):
        self.running = True
        self.register()
        self.pinger.start()

        while self.running and not self.pinger.unresponsive:
            try:
                events = dict(self.poller.poll(self.timeout))
            except zmq.ZMQError:
                break

            for socket in events:
                msg = unserialize(socket.recv())

                logger.debug(msg)
                if msg[0] == 'JOB':
                    # do the job and send the result
                    start = time.time()
                    try:
                        res = self.target(msg[1:])
                    except Exception, e:
                        # XXX log the error
                        res = str(e)
                    logger.debug('%.6f' % (time.time() - start))
                    socket.send(serialize("JOBRES", msg[1], res))
                else:
                    socket.send('ERROR')
예제 #3
0
    def _execute(self, job_id, job_data, timeout=1.):
        worker = None
        timeout *= 1000.  # timeout is in ms
        data = serialize("JOB", str(job_id), job_data)

        try:
            with self.workers.worker() as worker:
                try:
                    worker.send(data, zmq.NOBLOCK)
                except zmq.ZMQError, e:
                    raise ExecutionError(str(e))

                poller = zmq.Poller()
                poller.register(worker, zmq.POLLIN)

                try:
                    events = dict(poller.poll(timeout))
                except zmq.ZMQError, e:
                    raise ExecutionError(str(e))

                if events == {}:
                    raise TimeoutError()

                for socket in events:
                    try:
                        msg = unserialize(socket.recv())
                    except zmq.ZMQError, e:
                        raise ExecutionError(str(e))

                    if msg[0] == 'JOBRES':
                        # we got a result
                        return msg[-1]
                    else:
                        raise NotImplementedError(str(msg))
예제 #4
0
    def _execute(self, job_id, job_data, timeout=1.):
        worker = None
        timeout *= 1000.   # timeout is in ms
        data = serialize("JOB", str(job_id), job_data)

        try:
            with self.workers.worker() as worker:
                try:
                    worker.send(data, zmq.NOBLOCK)
                except zmq.ZMQError, e:
                    raise ExecutionError(str(e))

                poller = zmq.Poller()
                poller.register(worker, zmq.POLLIN)

                try:
                    events = dict(poller.poll(timeout))
                except zmq.ZMQError, e:
                    raise ExecutionError(str(e))

                if events == {}:
                    raise TimeoutError()

                for socket in events:
                    try:
                        msg = unserialize(socket.recv())
                    except zmq.ZMQError, e:
                        raise ExecutionError(str(e))

                    if msg[0] == 'JOBRES':
                        # we got a result
                        return msg[-1]
                    else:
                        raise NotImplementedError(str(msg))
예제 #5
0
파일: workermgr.py 프로젝트: jrgm/powerhose
    def run(self):
        """Runs the registration.

        Should not be called directly. Use :func:`start`.
        """
        self.alive = True

        # channel to communicate with the workers
        logger.debug('Starting [workermgr]')
        client = self.context.socket(zmq.REP)
        client.identity = 'master'
        client.bind(self.endpoint)
        poller = zmq.Poller()
        poller.register(client, zmq.POLLIN)
        poll_timeout = 1000

        while self.alive:
            try:
                events = dict(poller.poll(poll_timeout))
            except zmq.ZMQError, e:
                logger.debug("The poll failed")
                logger.debug(str(e))
                break

            for socket in events:
                msg = unserialize(socket.recv())

                if len(msg) < 2:
                    # XXX log
                    socket.send('ERROR')

                if msg[-2] == 'PING':
                    logger.debug("[workermgr] Got a PING")
                    if msg[-1] not in self.workers:
                        name = msg[-1]
                        logger.debug("Registered " + name)
                        # keep track of that worker
                        work = self.context.socket(zmq.REQ)
                        work.connect(name)
                        work.identity = name
                        self.workers.add(work)

                    # in any case we pong back
                    logger.debug("[workermgr] sent a PONG")
                    socket.send('PONG')
                elif msg[-2] == 'REMOVE':
                    if msg[-1] in self.workers:
                        logger.debug("[workermgr] Removing` " + msg[-1])
                        self.workers.delete(msg[-1])
                    socket.send('REMOVED')
                else:
                    logger.debug('Error')
                    socket.send('ERROR')

            time.sleep(.1)
예제 #6
0
    def run(self):
        self.alive = True

        # channel to communicate with the workers
        logger.debug('Starting [workermgr]')
        client = self.context.socket(zmq.REP)
        client.identity = 'master'
        client.bind(self.endpoint)
        poller = zmq.Poller()
        poller.register(client, zmq.POLLIN)
        poll_timeout = 1000

        while self.alive:
            try:
                events = dict(poller.poll(poll_timeout))
            except zmq.ZMQError, e:
                logger.debug("The poll failed")
                logger.debug(str(e))
                break

            for socket in events:
                msg = unserialize(socket.recv())

                if len(msg) < 2:
                    # XXX log
                    socket.send('ERROR')

                if msg[-2] == 'PING':
                    logger.debug("[workermgr] Got a PING")
                    if msg[-1] not in self.workers:
                        name = msg[-1]
                        logger.debug("Registered " + name)
                        # keep track of that worker
                        work = self.context.socket(zmq.REQ)
                        work.connect(name)
                        work.identity = name
                        self.workers.add(work)

                    # in any case we pong back
                    logger.debug("[workermgr] sent a PONG")
                    socket.send('PONG')
                elif msg[-2] == 'REMOVE':
                    if msg[-1] in self.workers:
                        logger.debug("[workermgr] Removing` " + msg[-1])
                        self.workers.delete(msg[-1])
                    socket.send('REMOVED')
                else:
                    logger.debug('Error')
                    socket.send('ERROR')

            time.sleep(.1)