def run(self):
self.running = True
self.register()
self.pinger.start()
while self.running and not self.pinger.unresponsive:
try:
events = dict(self.poller.poll(self.timeout))
except zmq.ZMQError:
break
for socket in events:
msg = unserialize(socket.recv())
if msg[0] == 'JOB':
# do the job and send the result
start = time.time()
try:
res = self.target(msg[1:])
except Exception, e:
# XXX log the error
res = str(e)
logger.debug('%.6f' % (time.time() - start))
socket.send(serialize("JOBRES", msg[1], res))
else:
socket.send('ERROR')
def run(self):
self.running = True
self.register()
self.pinger.start()
while self.running and not self.pinger.unresponsive:
try:
events = dict(self.poller.poll(self.timeout))
except zmq.ZMQError:
break
for socket in events:
msg = unserialize(socket.recv())
logger.debug(msg)
if msg[0] == 'JOB':
# do the job and send the result
start = time.time()
try:
res = self.target(msg[1:])
except Exception, e:
# XXX log the error
res = str(e)
logger.debug('%.6f' % (time.time() - start))
socket.send(serialize("JOBRES", msg[1], res))
else:
socket.send('ERROR')
def _execute(self, job_id, job_data, timeout=1.):
worker = None
timeout *= 1000. # timeout is in ms
data = serialize("JOB", str(job_id), job_data)
try:
with self.workers.worker() as worker:
try:
worker.send(data, zmq.NOBLOCK)
except zmq.ZMQError, e:
raise ExecutionError(str(e))
poller = zmq.Poller()
poller.register(worker, zmq.POLLIN)
try:
events = dict(poller.poll(timeout))
except zmq.ZMQError, e:
raise ExecutionError(str(e))
if events == {}:
raise TimeoutError()
for socket in events:
try:
msg = unserialize(socket.recv())
except zmq.ZMQError, e:
raise ExecutionError(str(e))
if msg[0] == 'JOBRES':
# we got a result
return msg[-1]
else:
raise NotImplementedError(str(msg))
def _execute(self, job_id, job_data, timeout=1.):
worker = None
timeout *= 1000. # timeout is in ms
data = serialize("JOB", str(job_id), job_data)
try:
with self.workers.worker() as worker:
try:
worker.send(data, zmq.NOBLOCK)
except zmq.ZMQError, e:
raise ExecutionError(str(e))
poller = zmq.Poller()
poller.register(worker, zmq.POLLIN)
try:
events = dict(poller.poll(timeout))
except zmq.ZMQError, e:
raise ExecutionError(str(e))
if events == {}:
raise TimeoutError()
for socket in events:
try:
msg = unserialize(socket.recv())
except zmq.ZMQError, e:
raise ExecutionError(str(e))
if msg[0] == 'JOBRES':
# we got a result
return msg[-1]
else:
raise NotImplementedError(str(msg))
def run(self):
"""Runs the registration.
Should not be called directly. Use :func:`start`.
"""
self.alive = True
# channel to communicate with the workers
logger.debug('Starting [workermgr]')
client = self.context.socket(zmq.REP)
client.identity = 'master'
client.bind(self.endpoint)
poller = zmq.Poller()
poller.register(client, zmq.POLLIN)
poll_timeout = 1000
while self.alive:
try:
events = dict(poller.poll(poll_timeout))
except zmq.ZMQError, e:
logger.debug("The poll failed")
logger.debug(str(e))
break
for socket in events:
msg = unserialize(socket.recv())
if len(msg) < 2:
# XXX log
socket.send('ERROR')
if msg[-2] == 'PING':
logger.debug("[workermgr] Got a PING")
if msg[-1] not in self.workers:
name = msg[-1]
logger.debug("Registered " + name)
# keep track of that worker
work = self.context.socket(zmq.REQ)
work.connect(name)
work.identity = name
self.workers.add(work)
# in any case we pong back
logger.debug("[workermgr] sent a PONG")
socket.send('PONG')
elif msg[-2] == 'REMOVE':
if msg[-1] in self.workers:
logger.debug("[workermgr] Removing` " + msg[-1])
self.workers.delete(msg[-1])
socket.send('REMOVED')
else:
logger.debug('Error')
socket.send('ERROR')
time.sleep(.1)
def run(self):
self.alive = True
# channel to communicate with the workers
logger.debug('Starting [workermgr]')
client = self.context.socket(zmq.REP)
client.identity = 'master'
client.bind(self.endpoint)
poller = zmq.Poller()
poller.register(client, zmq.POLLIN)
poll_timeout = 1000
while self.alive:
try:
events = dict(poller.poll(poll_timeout))
except zmq.ZMQError, e:
logger.debug("The poll failed")
logger.debug(str(e))
break
for socket in events:
msg = unserialize(socket.recv())
if len(msg) < 2:
# XXX log
socket.send('ERROR')
if msg[-2] == 'PING':
logger.debug("[workermgr] Got a PING")
if msg[-1] not in self.workers:
name = msg[-1]
logger.debug("Registered " + name)
# keep track of that worker
work = self.context.socket(zmq.REQ)
work.connect(name)
work.identity = name
self.workers.add(work)
# in any case we pong back
logger.debug("[workermgr] sent a PONG")
socket.send('PONG')
elif msg[-2] == 'REMOVE':
if msg[-1] in self.workers:
logger.debug("[workermgr] Removing` " + msg[-1])
self.workers.delete(msg[-1])
socket.send('REMOVED')
else:
logger.debug('Error')
socket.send('ERROR')
time.sleep(.1)