def apply_target(target,
args=(),
kwargs={},
callback=None,
accept_callback=None,
pid=None,
getpid=os.getpid,
propagate=(),
monotonic=monotonic,
**_):
if accept_callback:
accept_callback(pid or getpid(), monotonic())
try:
ret = target(*args, **kwargs)
except propagate:
raise
except Exception:
raise
except (WorkerShutdown, WorkerTerminate):
raise
except BaseException as exc:
try:
reraise(WorkerLostError, WorkerLostError(repr(exc)),
sys.exc_info()[2])
except WorkerLostError:
callback(ExceptionInfo())
else:
callback(ret)
def on_poll_init(self, w, hub):
"""Initialize async pool using the eventloop hub."""
pool = self._pool
pool._active_writers = self._active_writers
self._create_timelimit_handlers(hub)
self._create_process_handlers(hub)
self._create_write_handlers(hub)
# did_start_ok will verify that pool processes were able to start,
# but this will only work the first time we start, as
# maxtasksperchild will mess up metrics.
if not w.consumer.restart_count and not pool.did_start_ok():
raise WorkerLostError('Could not start worker processes')
# Maintain_pool is called whenever a process exits.
hub.add(pool.process_sentinels, self.maintain_pool, READ | ERR)
# Handle_result_event is called whenever one of the
# result queues are readable.
hub.add(pool._fileno_to_outq, self.handle_result_event, READ | ERR)
# Timers include calling maintain_pool at a regular interval
# to be certain processes are restarted.
for handler, interval in items(self.timers):
hub.timer.apply_interval(interval * 1000.0, handler)
def on_poll_init(self, pool, w, hub):
apply_after = hub.timer.apply_after
apply_at = hub.timer.apply_at
on_soft_timeout = pool.on_soft_timeout
on_hard_timeout = pool.on_hard_timeout
maintain_pool = pool.maintain_pool
add_reader = hub.add_reader
remove = hub.remove
now = time.time
# did_start_ok will verify that pool processes were able to start,
# but this will only work the first time we start, as
# maxtasksperchild will mess up metrics.
if not w.consumer.restart_count and not pool.did_start_ok():
raise WorkerLostError('Could not start worker processes')
# need to handle pool results before every task
# since multiple tasks can be received in a single poll()
hub.on_task.append(pool.maybe_handle_result)
hub.update_readers(pool.readers)
for handler, interval in pool.timers.iteritems():
hub.timer.apply_interval(interval * 1000.0, handler)
def on_timeout_set(R, soft, hard):
def _on_soft_timeout():
if hard:
R._tref = apply_at(now() + (hard - soft), on_hard_timeout,
(R, ))
on_soft_timeout(R)
if soft:
R._tref = apply_after(soft * 1000.0, _on_soft_timeout)
elif hard:
R._tref = apply_after(hard * 1000.0, on_hard_timeout, (R, ))
def on_timeout_cancel(result):
try:
result._tref.cancel()
delattr(result, '_tref')
except AttributeError:
pass
pool.init_callbacks(
on_process_up=lambda w: add_reader(w.sentinel, maintain_pool),
on_process_down=lambda w: remove(w.sentinel),
on_timeout_set=on_timeout_set,
on_timeout_cancel=on_timeout_cancel,
)
def on_poll_init(self, pool, hub):
apply_after = hub.timer.apply_after
apply_at = hub.timer.apply_at
on_soft_timeout = pool.on_soft_timeout
on_hard_timeout = pool.on_hard_timeout
maintain_pool = pool.maintain_pool
add_reader = hub.add_reader
remove = hub.remove
now = time.time
if not pool.did_start_ok():
raise WorkerLostError("Could not start worker processes")
hub.update_readers(pool.readers)
for handler, interval in pool.timers.iteritems():
hub.timer.apply_interval(interval * 1000.0, handler)
def on_timeout_set(R, soft, hard):
def _on_soft_timeout():
if hard:
R._tref = apply_at(now() + (hard - soft), on_hard_timeout,
(R, ))
on_soft_timeout(R)
if soft:
R._tref = apply_after(soft * 1000.0, _on_soft_timeout)
elif hard:
R._tref = apply_after(hard * 1000.0, on_hard_timeout, (R, ))
def on_timeout_cancel(result):
try:
result._tref.cancel()
delattr(result, "_tref")
except AttributeError:
pass
pool.init_callbacks(
on_process_up=lambda w: add_reader(w.sentinel, maintain_pool),
on_process_down=lambda w: remove(w.sentinel),
on_timeout_set=on_timeout_set,
on_timeout_cancel=on_timeout_cancel,
)
def on_poll_init(self,
w,
hub,
now=time,
protocol=HIGHEST_PROTOCOL,
pack=struct.pack,
dumps=_pickle.dumps):
pool = self._pool
apply_after = hub.timer.apply_after
apply_at = hub.timer.apply_at
maintain_pool = self.maintain_pool
on_soft_timeout = self.on_soft_timeout
on_hard_timeout = self.on_hard_timeout
fileno_to_inq = pool._fileno_to_inq
fileno_to_outq = pool._fileno_to_outq
fileno_to_synq = pool._fileno_to_synq
outbound = self.outbound_buffer
pop_message = outbound.popleft
put_message = outbound.append
all_inqueues = pool._all_inqueues
active_writes = self._active_writes
diff = all_inqueues.difference
hub_add, hub_remove = hub.add, hub.remove
mark_write_fd_as_active = active_writes.add
mark_write_gen_as_active = self._active_writers.add
write_generator_gone = self._active_writers.discard
get_job = pool._cache.__getitem__
pool._put_back = put_message
# did_start_ok will verify that pool processes were able to start,
# but this will only work the first time we start, as
# maxtasksperchild will mess up metrics.
if not w.consumer.restart_count and not pool.did_start_ok():
raise WorkerLostError('Could not start worker processes')
hub_add(pool.process_sentinels, self.maintain_pool, READ | ERR)
hub_add(fileno_to_outq, self.handle_result_event, READ | ERR)
for handler, interval in items(self.timers):
hub.timer.apply_interval(interval * 1000.0, handler)
# need to handle pool results before every task
# since multiple tasks can be received in a single poll()
# XXX do we need this now?!?
# hub.on_task.append(pool.maybe_handle_result)
def on_timeout_set(R, soft, hard):
def _on_soft_timeout():
if hard:
R._tref = apply_at(now() + (hard - soft), on_hard_timeout,
(R, ))
on_soft_timeout(R)
if soft:
R._tref = apply_after(soft * 1000.0, _on_soft_timeout)
elif hard:
R._tref = apply_after(hard * 1000.0, on_hard_timeout, (R, ))
self._pool.on_timeout_set = on_timeout_set
def on_timeout_cancel(result):
try:
result._tref.cancel()
delattr(result, '_tref')
except AttributeError:
pass
self._pool.on_timeout_cancel = on_timeout_cancel
def on_process_up(proc):
fileno_to_outq[proc.outqR_fd] = proc
hub_add(proc.sentinel, maintain_pool, READ | ERR)
hub_add(proc.outqR_fd, pool.handle_result_event, READ | ERR)
self._pool.on_process_up = on_process_up
def on_process_down(proc):
pool.process_flush_queues(proc)
fileno_to_outq.pop(proc.outqR_fd, None)
fileno_to_inq.pop(proc.inqW_fd, None)
fileno_to_synq.pop(proc.synqW_fd, None)
all_inqueues.discard(proc.inqW_fd)
hub_remove(proc.sentinel)
hub_remove(proc.outqR_fd)
self._pool.on_process_down = on_process_down
class Ack(object):
__slots__ = ('id', 'fd', '_payload')
def __init__(self, id, fd, payload):
self.id = id
self.fd = fd
self._payload = payload
def __eq__(self, other):
return self.i == other.i
def __hash__(self):
return self.i
def _write_ack(fd, ack, callback=None):
header, body, body_size = ack._payload
try:
try:
proc = fileno_to_synq[fd]
except KeyError:
raise StopIteration()
send_offset = proc.synq._writer.send_offset
Hw = Bw = 0
while Hw < 4:
try:
Hw += send_offset(header, Hw)
except Exception as exc:
if get_errno(exc) not in UNAVAIL:
raise
yield
while Bw < body_size:
try:
Bw += send_offset(body, Bw)
except Exception as exc:
if get_errno(exc) not in UNAVAIL:
raise
# suspend until more data
yield
finally:
if callback:
callback()
active_writes.discard(fd)
def _write_job(fd, job, callback=None):
header, body, body_size = job._payload
try:
try:
proc = fileno_to_inq[fd]
except KeyError:
put_message(job)
raise StopIteration()
# job result keeps track of what process the job is sent to.
job._write_to = proc
send_offset = proc.inq._writer.send_offset
Hw = Bw = 0
while Hw < 4:
try:
Hw += send_offset(header, Hw)
except Exception as exc:
if get_errno(exc) not in UNAVAIL:
raise
# suspend until more data
yield
while Bw < body_size:
try:
Bw += send_offset(body, Bw)
except Exception as exc:
if get_errno(exc) not in UNAVAIL:
raise
# suspend until more data
yield
finally:
if callback:
callback()
active_writes.discard(fd)
def on_inqueue_close(fd):
active_writes.discard(fd)
all_inqueues.discard(fd)
self._pool.on_inqueue_close = on_inqueue_close
def schedule_writes(ready_fd, events):
if ready_fd in active_writes:
return
try:
job = pop_message()
except IndexError:
for inqfd in diff(active_writes):
hub_remove(inqfd)
else:
if not job._accepted:
callback = promise(write_generator_gone)
try:
job._scheduled_for = fileno_to_inq[ready_fd]
except KeyError:
# process gone since scheduled, put back
return put_message(job)
cor = _write_job(ready_fd, job, callback=callback)
mark_write_gen_as_active(cor)
mark_write_fd_as_active(ready_fd)
callback.args = (cor, ) # tricky as we need to pass ref
hub_add((ready_fd, ), cor, WRITE | ERR)
def _create_payload(type_, args):
body = dumps((type_, args), protocol=protocol)
size = len(body)
header = pack('>I', size)
return header, body, size
MESSAGES = {
ACK: _create_payload(ACK, (0, )),
NACK: _create_payload(NACK, (0, ))
}
def send_ack(response, pid, job, fd):
msg = Ack(job, fd, MESSAGES[response])
callback = promise(write_generator_gone)
cor = _write_ack(fd, msg, callback=callback)
mark_write_gen_as_active(cor)
mark_write_fd_as_active(fd)
callback.args = (cor, )
hub_add((fd, ), cor, WRITE | ERR)
self._pool.send_ack = send_ack
def on_poll_start(hub):
if outbound:
hub_add(diff(active_writes), schedule_writes, WRITE | ERR)
self.on_poll_start = on_poll_start
def quick_put(tup):
body = dumps(tup, protocol=protocol)
body_size = len(body)
header = pack('>I', body_size)
# index 1,0 is the job ID.
job = get_job(tup[1][0])
job._payload = header, buffer(body), body_size
put_message(job)
self._pool._quick_put = quick_put
def on_poll_init(self, pool, w, hub):
apply_after = hub.timer.apply_after
apply_at = hub.timer.apply_at
on_soft_timeout = pool.on_soft_timeout
on_hard_timeout = pool.on_hard_timeout
maintain_pool = pool.maintain_pool
add_reader = hub.add_reader
remove = hub.remove
now = time.time
cache = getattr(pool._pool, '_cache', None)
# did_start_ok will verify that pool processes were able to start,
# but this will only work the first time we start, as
# maxtasksperchild will mess up metrics.
if not w.consumer.restart_count and not pool.did_start_ok():
raise WorkerLostError('Could not start worker processes')
# need to handle pool results before every task
# since multiple tasks can be received in a single poll()
hub.on_task.append(pool.maybe_handle_result)
hub.update_readers(pool.readers)
for handler, interval in pool.timers.iteritems():
hub.timer.apply_interval(interval * 1000.0, handler)
trefs = pool._tref_for_id = WeakValueDictionary()
def _discard_tref(job):
try:
tref = trefs.pop(job)
tref.cancel()
del (tref)
except (KeyError, AttributeError):
pass # out of scope
def _on_hard_timeout(job):
try:
result = cache[job]
except KeyError:
pass # job ready
else:
on_hard_timeout(result)
finally:
# remove tref
_discard_tref(job)
def _on_soft_timeout(job, soft, hard, hub):
if hard:
trefs[job] = apply_at(
now() + (hard - soft),
_on_hard_timeout,
(job, ),
)
try:
result = cache[job]
except KeyError:
pass # job ready
else:
on_soft_timeout(result)
finally:
if not hard:
# remove tref
_discard_tref(job)
def on_timeout_set(R, soft, hard):
if soft:
trefs[R._job] = apply_after(
soft * 1000.0,
_on_soft_timeout,
(R._job, soft, hard, hub),
)
elif hard:
trefs[R._job] = apply_after(hard * 1000.0, _on_hard_timeout,
(R._job, ))
def on_timeout_cancel(R):
_discard_tref(R._job)
pool.init_callbacks(
on_process_up=lambda w: add_reader(w.sentinel, maintain_pool),
on_process_down=lambda w: remove(w.sentinel),
on_timeout_set=on_timeout_set,
on_timeout_cancel=on_timeout_cancel,
)
