def _handle_recv_back(self, msg):
# do the job and send the result
if self.debug:
logger.debug('Job received')
target = timed()(self.target)
else:
target = self.target
duration = -1
# results are sent with a PID:OK: or a PID:ERROR prefix
try:
with self.timer.run_job():
res = target(Job.load_from_string(msg[0]))
# did we timout ?
if self.timer.timed_out:
# let's dump the last
for line in self.timer.last_dump:
logger.error(line)
if self.debug:
duration, res = res
res = '%d:OK:%s' % (self.pid, res)
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = '%d:ERROR:%s' % (self.pid, '\n'.join(exc))
logger.error(res)
def __init__(self, target, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT, ping_delay=1., ping_retries=3,
params=None, timeout=DEFAULT_TIMEOUT_MOVF,
max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA):
logger.debug('Initializing the worker.')
self.ctx = zmq.Context()
self.backend = backend
self._backend = self.ctx.socket(zmq.REP)
self._backend.connect(self.backend)
self.target = target
self.running = False
self.loop = ioloop.IOLoop()
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self.ping = Stethoscope(heartbeat, onbeatlost=self.lost,
delay=ping_delay, retries=ping_retries,
ctx=self.ctx)
self.debug = logger.isEnabledFor(logging.DEBUG)
self.params = params
self.pid = os.getpid()
self.timer = ExecutionTimer(timeout=timeout)
self.max_age = max_age
self.max_age_delta = max_age_delta
self.delayed_exit = None
self.lock = threading.RLock()
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT):
logger.debug('Initializing the broker.')
for endpoint in (frontend, backend, heartbeat):
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context()
# setting up the two sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.DEALER)
self._backend.bind(backend)
# setting up the poller
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
# heartbeat
self.pong = Heartbeat(heartbeat)
# status
self.started = False
self.poll_timeout = None
def _handle_reg(self, msg):
if msg[0] == 'REGISTER':
if msg[1] not in self._workers:
logger.debug('%r registered' % msg[1])
self._workers.append(msg[1])
elif msg[0] == 'UNREGISTER':
if msg[1] in self._workers:
self._remove_worker(msg[1])
def _handle_recv_back(self, msg):
# back => front
logger.debug('front <- back')
try:
self._frontstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _handle_recv_back(self, msg):
# back => front
logger.debug('front <- back')
try:
self._frontstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _initialize(self):
logger.debug('Subscribing to ' + self.endpoint)
self._endpoint = self.context.socket(zmq.SUB)
self._endpoint.setsockopt(zmq.SUBSCRIBE, '')
self._endpoint.linger = 0
#self._endpoint.identity = str(os.getpid())
self._endpoint.connect(self.endpoint)
self._stream = zmqstream.ZMQStream(self._endpoint, self.loop)
self._stream.on_recv(self._handle_recv)
self._timer = ioloop.PeriodicCallback(self._delayed,
self.delay * 1000, io_loop=self.loop)
def stop(self):
"""Stops the worker.
"""
logger.debug('Stopping the worker')
self.ping.stop()
self.timer.stop()
self.loop.stop()
self.running = False
time.sleep(.1)
self.ctx.destroy(0)
logger.debug('Worker is stopped')
def stop(self):
"""Stops the Pinger"""
logger.debug('Stopping the Pinger')
self.running = False
self._stream.flush()
if self._stop_loop:
self.loop.stop()
if self.isAlive():
try:
self.join()
except RuntimeError:
pass
def _initialize(self):
logger.debug('Subscribing to ' + self.endpoint)
self._endpoint = self.context.socket(zmq.SUB)
self._endpoint.setsockopt(zmq.SUBSCRIBE, '')
self._endpoint.linger = 0
#self._endpoint.identity = str(os.getpid())
self._endpoint.connect(self.endpoint)
self._stream = zmqstream.ZMQStream(self._endpoint, self.loop)
self._stream.on_recv(self._handle_recv)
self._timer = ioloop.PeriodicCallback(self._delayed,
self.delay * 1000,
io_loop=self.loop)
def __init__(self, timeout=DEFAULT_TIMEOUT_MOVF, interval=0.1):
logger.debug("Initializing the execution timer. timeout is %.2f" % timeout)
threading.Thread.__init__(self)
self.armed = self.running = False
self.timeout = timeout
self.daemon = True
# creating a queue for I/O with the worker
self.queue = Queue.Queue()
self.interval = interval
self.timed_out = self.working = False
self.last_dump = None
def stop(self):
"""Stops the Pinger"""
logger.debug('Stopping the Pinger')
self.running = False
self._stream.flush()
if self._stop_loop:
self.loop.stop()
if self.isAlive():
try:
self.join()
except RuntimeError:
pass
def _stop(self):
logger.debug('Stopping the worker')
self.running = False
try:
self._backstream.flush()
except zmq.core.error.ZMQError:
pass
self.loop.stop()
self.ping.stop()
self.timer.stop()
time.sleep(.1)
self.ctx.destroy(0)
logger.debug('Worker is stopped')
def __init__(self, endpoint=DEFAULT_HEARTBEAT, interval=2.):
threading.Thread.__init__(self)
self.daemon = True
self.context = zmq.Context()
self.endpoint = endpoint
self.running = False
self.interval = interval
logger.debug('Publishing to ' + self.endpoint)
self._endpoint = self.context.socket(zmq.PUB)
self._endpoint.linger = 0
self._endpoint.identity = b'HB'
self._endpoint.hwm = 0
self._endpoint.bind(self.endpoint)
def stop(self):
"""Stops the broker.
"""
if not self.started:
return
self._backstream.flush()
logger.debug('Stopping the heartbeat')
self.pong.stop()
logger.debug('Stopping the loop')
self.loop.stop()
self.started = False
self.context.destroy(0)
def _stop(self):
logger.debug('Stopping the worker')
self.running = False
try:
self._backstream.flush()
except zmq.core.error.ZMQError:
pass
self.loop.stop()
self.ping.stop()
self.timer.stop()
time.sleep(.1)
self.ctx.destroy(0)
logger.debug('Worker is stopped')
def __init__(self, timeout=DEFAULT_TIMEOUT_MOVF, interval=.1):
logger.debug('Initializing the execution timer. timeout is %.2f' \
% timeout)
threading.Thread.__init__(self)
self.armed = self.running = False
self.timeout = timeout
self.daemon = True
# creating a queue for I/O with the worker
self.queue = Queue.Queue()
self.interval = interval
self.timed_out = self.working = False
self.last_dump = None
def stop(self):
"""Stops the broker.
"""
if not self.started:
return
self._backstream.flush()
logger.debug('Stopping the heartbeat')
self.pong.stop()
logger.debug('Stopping the loop')
self.loop.stop()
self.started = False
self.context.destroy(0)
def start(self):
"""Starts the worker
"""
util.PARAMS = self.params
logger.debug('Starting the worker loop')
# running the pinger
self.ping.start()
self.timer.start()
self.running = True
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
logger.debug('Worker loop over')
def run(self):
time.sleep(self.warmup_delay)
self.running = True
while self.running:
# waiting for the BEAT
tries = 0
logger.debug('Waiting for the BEAT')
while tries < self.retries:
events = dict(self._poller.poll(self.delay * 1000))
if events != {}:
break
logger.debug('nothing came back')
tries += 1
if tries == self.retries:
if self.onbeatlost is not None and self.onbeatlost():
self.running = False
return
if events and events[self._endpoint] == zmq.POLLIN:
msg = self._endpoint.recv()
if self.onbeat is not None:
self.onbeat()
logger.debug(msg)
logger.debug('Ping loop over')
def stop(self):
"""Stops the worker.
"""
if not self.running:
return
# telling the broker we are stopping
try:
self._reg.send_multipart(["UNREGISTER", str(os.getpid())])
except zmq.ZMQError:
logger.debug("Could not unregister")
# give it a chance to finish a job
logger.debug("Starting the graceful period")
self.graceful_delay = ioloop.DelayedCallback(self._stop, self.timeout * 1000, io_loop=self.loop)
self.graceful_delay.start()
def __init__(self, frontend=DEFAULT_FRONTEND, timeout=DEFAULT_TIMEOUT,
timeout_max_overflow=DEFAULT_TIMEOUT_MOVF,
timeout_overflows=DEFAULT_TIMEOUT_OVF,
iothreads=5, debug=False):
self.ctx = zmq.Context(io_threads=iothreads)
self.master = self.ctx.socket(zmq.REQ)
self.master.connect(frontend)
logger.debug('Client connected to %s' % frontend)
self.poller = zmq.Poller()
self.poller.register(self.master, zmq.POLLIN)
self.timeout = timeout * 1000
self.lock = threading.Lock()
self.timeout_max_overflow = timeout_max_overflow * 1000
self.timeout_overflows = timeout_overflows
self.timeout_counters = defaultdict(int)
self.debug = debug
def __init__(self, endpoint=DEFAULT_HEARTBEAT, interval=2.,
io_loop=None):
self.loop = io_loop or ioloop.IOLoop.instance()
self.daemon = True
self.context = zmq.Context()
self.endpoint = endpoint
self.running = False
self.interval = interval
logger.debug('Publishing to ' + self.endpoint)
self._endpoint = self.context.socket(zmq.PUB)
self._endpoint.linger = 0
#self._endpoint.identity = b'HB'
self._endpoint.hwm = 0
self._endpoint.bind(self.endpoint)
self._cb = ioloop.PeriodicCallback(self._ping, interval * 1000,
io_loop=self.loop)
def _handle_recv_front(self, msg, tentative=0):
# front => back
# if the last part of the message is 'PING', we just PONG back
# this is used as a health check
if msg[-1] == 'PING':
self._frontstream.send_multipart(msg[:-1] + [str(os.getpid())])
return
#logger.debug('front -> back [choosing a worker]')
if tentative == 3:
logger.debug('No workers')
self._frontstream.send_multipart(msg[:-1] +
['%d:ERROR:No worker' % os.getpid()])
return
# we want to decide who's going to do the work
found_worker = False
while not found_worker and len(self._workers) > 0:
worker_id = random.choice(self._workers)
if not self._check_worker(worker_id):
self._remove_worker(worker_id)
else:
found_worker = True
if not found_worker:
logger.debug('No worker, will try later')
later = time.time() + 0.5 + (tentative * 0.2)
self.loop.add_timeout(later, lambda: self._handle_recv_front(msg,
tentative + 1))
return
# start the timer
self._worker_times[worker_id] = time.time(), None
# now we can send to the right guy
msg.insert(0, worker_id)
#logger.debug('front -> back [%s]' % worker_id)
try:
self._backstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def _check_worker(self, worker_id):
# box-specific, will do better later XXX
exists = psutil.pid_exists(int(worker_id))
if not exists:
logger.debug('The worker %r is gone' % worker_id)
return False
if worker_id in self._worker_times:
start, stop = self._worker_times[worker_id]
if stop is not None:
duration = start - stop
if duration > self.worker_timeout:
logger.debug('The worker %r is slow (%.2f)' % (worker_id,
duration))
return False
return True
def _handle_recv_back(self, msg):
# do the job and send the result
if self.debug:
logger.debug("Job received")
target = timed()(self.target)
else:
target = self.target
duration = -1
# results are sent with a PID:OK: or a PID:ERROR prefix
riemann_message = {
"host": socket.gethostname(),
"service": "powerhose-worker",
"metric": 1,
"state": "ok",
"tags": ["running"],
}
try:
with self.timer.run_job():
self.riemann.send(riemann_message)
res = target(Job.load_from_string(msg[0]))
# did we timout ?
if self.timer.timed_out:
# let's dump the last
riemann_message["state"] = "error"
riemann_message["tags"] = ["timeouts"]
self.riemann.send(riemann_message)
for line in self.timer.last_dump:
logger.error(line)
if self.debug:
duration, res = res
res = "%d:OK:%s" % (self.pid, res)
except Exception, e:
riemann_message["description"] = str(e)
riemann_message["state"] = "error"
riemann_message["tags"] = ["faults"]
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = "%d:ERROR:%s" % (self.pid, "\n".join(exc))
logger.error(res)
def stop(self):
"""Stops the worker.
"""
if not self.running:
return
# telling the broker we are stopping
try:
self._reg.send_multipart(['UNREGISTER', str(os.getpid())])
except zmq.ZMQError:
logger.debug('Could not unregister')
# give it a chance to finish a job
logger.debug('Starting the graceful period')
self.graceful_delay = ioloop.DelayedCallback(self._stop,
self.timeout * 1000,
io_loop=self.loop)
self.graceful_delay.start()
def _handle_recv_front(self, msg):
# front => back
logger.debug('front -> back')
# if the last part of the message is 'PING', we just PONG back
# this is used as a health check
if msg[-1] == 'PING':
self._frontstream.send_multipart(msg[:-1] + [str(os.getpid())])
return
try:
self._backstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def __init__(self, frontend=DEFAULT_FRONTEND, timeout=DEFAULT_TIMEOUT,
timeout_max_overflow=DEFAULT_TIMEOUT_MOVF,
timeout_overflows=DEFAULT_TIMEOUT_OVF,
debug=False, ctx=None):
self.kill_ctx = ctx is None
self.ctx = ctx or zmq.Context()
self.frontend = frontend
self.master = self.ctx.socket(zmq.REQ)
self.master.connect(frontend)
logger.debug('Client connected to %s' % frontend)
self.poller = zmq.Poller()
self.poller.register(self.master, zmq.POLLIN)
self.timeout = timeout * 1000
self.lock = threading.Lock()
self.timeout_max_overflow = timeout_max_overflow * 1000
self.timeout_overflows = timeout_overflows
self.timeout_counters = defaultdict(int)
self.debug = debug
def _handle_recv_front(self, msg):
# front => back
logger.debug('front -> back')
# if the last part of the message is 'PING', we just PONG back
# this is used as a health check
if msg[-1] == 'PING':
self._frontstream.send_multipart(msg[:-1] + [str(os.getpid())])
return
try:
self._backstream.send_multipart(msg)
except Exception, e:
# we don't want to die on error. we just log it
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
logger.error('\n'.join(exc))
def __init__(self,
endpoint=DEFAULT_HEARTBEAT,
interval=10.,
io_loop=None,
ctx=None):
self.loop = io_loop or ioloop.IOLoop.instance()
self.daemon = True
self.kill_context = ctx is None
self.context = ctx or zmq.Context()
self.endpoint = endpoint
self.running = False
self.interval = interval
logger.debug('Publishing to ' + self.endpoint)
self._endpoint = self.context.socket(zmq.PUB)
self._endpoint.linger = 0
#self._endpoint.identity = b'HB'
self._endpoint.hwm = 0
self._endpoint.bind(self.endpoint)
self._cb = ioloop.PeriodicCallback(self._ping,
interval * 1000,
io_loop=self.loop)
def __init__(self, endpoint=DEFAULT_HEARTBEAT, warmup_delay=.5, delay=3.,
retries=3,
onbeatlost=None, onbeat=None,):
threading.Thread.__init__(self)
self.daemon = True
self.context = zmq.Context()
self.endpoint = endpoint
self.running = False
self.delay = delay
self.retries = retries
self.onbeatlost = onbeatlost
self.onbeat = onbeat
self.warmup_delay = warmup_delay
self._endpoint = self.context.socket(zmq.SUB)
logger.debug('Subscribing to ' + self.endpoint)
self._endpoint.setsockopt(zmq.SUBSCRIBE, '')
self._endpoint.linger = 0
self._endpoint.identity = str(os.getpid())
self._endpoint.connect(self.endpoint)
self._poller = zmq.Poller()
self._poller.register(self._endpoint, zmq.POLLIN)
def start(self):
"""Starts the broker.
"""
logger.debug('Starting the loop')
if self.started:
return
# running the heartbeat
self.pong.start()
self.started = True
while self.started:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
def start(self):
"""Starts the broker.
"""
logger.debug('Starting the loop')
if self.started:
return
# running the heartbeat
self.pong.start()
self.started = True
while self.started:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
def run(self):
"""Starts the loop"""
logger.debug('Starting the loop')
if self.running:
return
self._initialize()
time.sleep(self.warmup_delay)
self._timer.start()
self.running = True
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
def run(self):
"""Starts the loop"""
logger.debug('Starting the loop')
if self.running:
return
self._initialize()
time.sleep(self.warmup_delay)
self._timer.start()
self.running = True
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
def start(self):
"""Starts the worker
"""
util.PARAMS = self.params
logger.debug('Starting the worker loop')
# running the pinger
self.ping.start()
self.timer.start()
self.running = True
# telling the broker we are ready
self._reg.send_multipart(['REGISTER', str(os.getpid())])
# arming the exit callback
if self.max_age != -1:
if self.max_age_delta > 0:
delta = random.randint(0, self.max_age_delta)
else:
delta = 0
cb_time = self.max_age + delta
self.delayed_exit = ioloop.DelayedCallback(self.stop,
cb_time * 1000,
io_loop=self.loop)
self.delayed_exit.start()
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
logger.debug('Worker loop over')
def start(self):
"""Starts the worker
"""
util.PARAMS = self.params
logger.debug('Starting the worker loop')
# running the pinger
self.ping.start()
self.timer.start()
self.running = True
# telling the broker we are ready
self._reg.send_multipart(['REGISTER', str(os.getpid())])
# arming the exit callback
if self.max_age != -1:
if self.max_age_delta > 0:
delta = random.randint(0, self.max_age_delta)
else:
delta = 0
cb_time = self.max_age + delta
self.delayed_exit = ioloop.DelayedCallback(self.stop,
cb_time * 1000,
io_loop=self.loop)
self.delayed_exit.start()
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
logger.debug('Worker loop over')
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
worker_timeout=DEFAULT_TIMEOUT_MOVF):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
logger.debug('Initializing the broker.')
for endpoint in (frontend, backend, heartbeat):
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the three sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
# heartbeat
self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context)
# status
self.started = False
self.poll_timeout = None
# workers registration and timers
self._workers = []
self._worker_times = {}
self.worker_timeout = worker_timeout
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
logger.debug('Initializing the broker.')
for endpoint in (frontend, backend, heartbeat):
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context()
# setting up the two sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.DEALER)
self._backend.bind(backend)
# setting up the poller
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
# heartbeat
self.pong = Heartbeat(heartbeat, io_loop=self.loop)
# status
self.started = False
self.poll_timeout = None
class Worker(object):
"""Class that links a callable to a broker.
Options:
- **target**: The Python callable that will be called when the broker
send a job.
- **backend**: The ZMQ socket to connect to the broker.
- **heartbeat**: The ZMQ socket to perform PINGs on the broker to make
sure it's still alive.
- **ping_delay**: the delay in seconds betweem two pings.
- **ping_retries**: the number of attempts to ping the broker before
quitting.
- **params** a dict containing the params to set for this worker.
- **timeout** the maximum time allowed before the thread stacks is dump
and the job result not sent back.
- **max_age**: maximum age for a worker in seconds. After that delay,
the worker will simply quit. When set to -1, never quits.
Defaults to -1.
- **max_age_delta**: maximum value in seconds added to max age.
The Worker will quit after *max_age + random(0, max_age_delta)*
This is done to avoid having all workers quit at the same instant.
Defaults to 0. The value must be an integer.
"""
def __init__(self, target, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT, ping_delay=1., ping_retries=3,
params=None, timeout=DEFAULT_TIMEOUT_MOVF,
max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA):
logger.debug('Initializing the worker.')
self.ctx = zmq.Context()
self.backend = backend
self._backend = self.ctx.socket(zmq.REP)
self._backend.connect(self.backend)
self.target = target
self.running = False
self.loop = ioloop.IOLoop()
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self.ping = Stethoscope(heartbeat, onbeatlost=self.lost,
delay=ping_delay, retries=ping_retries,
ctx=self.ctx)
self.debug = logger.isEnabledFor(logging.DEBUG)
self.params = params
self.pid = os.getpid()
self.timer = ExecutionTimer(timeout=timeout)
self.max_age = max_age
self.max_age_delta = max_age_delta
self.delayed_exit = None
self.lock = threading.RLock()
def _handle_recv_back(self, msg):
# do the job and send the result
if self.debug:
logger.debug('Job received')
target = timed()(self.target)
else:
target = self.target
duration = -1
# results are sent with a PID:OK: or a PID:ERROR prefix
try:
with self.timer.run_job():
res = target(Job.load_from_string(msg[0]))
# did we timout ?
if self.timer.timed_out:
# let's dump the last
for line in self.timer.last_dump:
logger.error(line)
if self.debug:
duration, res = res
res = '%d:OK:%s' % (self.pid, res)
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = '%d:ERROR:%s' % (self.pid, '\n'.join(exc))
logger.error(res)
if self.timer.timed_out:
# let's not send back anything, we know the client
# is gone anyway
return
if self.debug:
logger.debug('Duration - %.6f' % duration)
try:
self._backstream.send(res)
except Exception:
logging.error("Could not send back the result", exc_info=True)
def _ping(self):
logger.debug('*beat*')
self._endpoint.send('BEAT')
def _handle_recv(self, msg):
self.tries = 0
if self.onbeat is not None:
self.onbeat()
logger.debug(msg[0])
def _p(msg):
sys.stdout.write(msg + '\n')
logger.debug(msg)
sys.stdout.flush()
def _handle_recv(self, msg):
self.tries = 0
if self.onbeat is not None:
self.onbeat()
logger.debug(msg[0])
def _delayed(self):
self.tries += 1
if self.tries >= self.retries:
logger.debug('Nothing came back')
if self.onbeatlost is None or self.onbeatlost():
self.stop() # bye !
def _ping(self):
logger.debug('*beat*')
self._endpoint.send('BEAT')
def _remove_worker(self, worker_id):
logger.debug('%r removed' % worker_id)
self._workers.remove(worker_id)
if worker_id in self._worker_times:
del self._worker_times[worker_id]
def _delayed(self):
self.tries += 1
if self.tries >= self.retries:
logger.debug('Nothing came back')
if self.onbeatlost is None or self.onbeatlost():
self.stop() # bye !
