def flush(self):
if self._state == TERMINATE:
return
# cancel all tasks that have not been accepted so that NACK is sent.
for job in values(self._cache):
if not job._accepted:
job._cancel()
# clear the outgoing buffer as the tasks will be redelivered by
# the broker anyway.
if self.outbound_buffer:
self.outbound_buffer.clear()
self.maintain_pool()
try:
# ...but we must continue writing the payloads we already started
# to keep message boundaries.
# The messages may be NACK'ed later if synack is enabled.
if self._state == RUN:
# flush outgoing buffers
intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
owned_by = {}
for job in values(self._cache):
writer = _get_job_writer(job)
if writer is not None:
owned_by[writer] = job
while self._active_writers:
writers = list(self._active_writers)
for gen in writers:
if gen.__name__ == "_write_job" and gen_not_started(gen):
# has not started writing the job so can
# discard the task, but we must also remove
# it from the Pool._cache.
try:
job = owned_by[gen]
except KeyError:
pass
else:
# removes from Pool._cache
job.discard()
self._active_writers.discard(gen)
else:
try:
job = owned_by[gen]
except KeyError:
pass
else:
job_proc = job._write_to
if job_proc._is_alive():
self._flush_writer(job_proc, gen)
# workers may have exited in the meantime.
self.maintain_pool()
sleep(next(intervals)) # don't busyloop
finally:
self.outbound_buffer.clear()
self._active_writers.clear()
self._active_writes.clear()
self._busy_workers.clear()
def flush(self):
# cancel all tasks that have not been accepted so that NACK is sent.
for job in values(self._pool._cache):
if not job._accepted:
job._cancel()
# clear the outgoing buffer as the tasks will be redelivered by
# the broker anyway.
if self.outbound_buffer:
self.outbound_buffer.clear()
try:
# ...but we must continue writing the payloads we already started
# to keep message boundaries.
# The messages may be NACK'ed later if synack is enabled.
if self._pool._state == RUN:
# flush outgoing buffers
intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
while self._active_writers:
writers = list(self._active_writers)
for gen in writers:
if gen.__name__ == "_write_job" and gen.gi_frame and gen.gi_frame.f_lasti != -1:
# has not started writing the job so can
# safely discard
self._active_writers.discard(gen)
else:
try:
next(gen)
except StopIteration:
self._active_writers.discard(gen)
# workers may have exited in the meantime.
self.maintain_pool()
sleep(next(intervals)) # don't busyloop
finally:
self.outbound_buffer.clear()
self._active_writers.clear()
def flush(self):
if self._state == TERMINATE:
return
# cancel all tasks that have not been accepted so that NACK is sent.
for job in values(self._cache):
if not job._accepted:
job._cancel()
# clear the outgoing buffer as the tasks will be redelivered by
# the broker anyway.
if self.outbound_buffer:
self.outbound_buffer.clear()
self.maintain_pool()
try:
# ...but we must continue writing the payloads we already started
# to keep message boundaries.
# The messages may be NACK'ed later if synack is enabled.
if self._state == RUN:
# flush outgoing buffers
intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
owned_by = {}
for job in values(self._cache):
writer = _get_job_writer(job)
if writer is not None:
owned_by[writer] = job
while self._active_writers:
writers = list(self._active_writers)
for gen in writers:
if (gen.__name__ == '_write_job' and
gen_not_started(gen)):
# has not started writing the job so can
# discard the task, but we must also remove
# it from the Pool._cache.
try:
job = owned_by[gen]
except KeyError:
pass
else:
# removes from Pool._cache
job.discard()
self._active_writers.discard(gen)
else:
try:
job = owned_by[gen]
except KeyError:
pass
else:
if job._write_to.exitcode is None:
self._flush_writer(gen)
# workers may have exited in the meantime.
self.maintain_pool()
sleep(next(intervals)) # don't busyloop
finally:
self.outbound_buffer.clear()
self._active_writers.clear()
self._active_writes.clear()
self._busy_workers.clear()
def flush(self):
if self._state == TERMINATE:
return
# cancel all tasks that have not been accepted so that NACK is sent.
for job in values(self._cache):
if not job._accepted:
job._cancel()
# clear the outgoing buffer as the tasks will be redelivered by
# the broker anyway.
if self.outbound_buffer:
self.outbound_buffer.clear()
try:
# ...but we must continue writing the payloads we already started
# to keep message boundaries.
# The messages may be NACK'ed later if synack is enabled.
if self._state == RUN:
# flush outgoing buffers
intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
while self._active_writers:
writers = list(self._active_writers)
for gen in writers:
if (gen.__name__ == '_write_job' and
gen_not_started(gen)):
# has not started writing the job so can
# discard the task, but we must also remove
# it from the Pool._cache.
job_to_discard = None
for job in values(self._cache):
if job._writer() is gen: # _writer is saferef
# removes from Pool._cache
job_to_discard = job
break
if job_to_discard:
job_to_discard.discard()
self._active_writers.discard(gen)
else:
try:
next(gen)
except StopIteration:
self._active_writers.discard(gen)
# workers may have exited in the meantime.
self.maintain_pool()
sleep(next(intervals)) # don't busyloop
finally:
self.outbound_buffer.clear()
self._active_writers.clear()
def flush(self):
# cancel all tasks that have not been accepted so that NACK is sent.
for job in values(self._pool._cache):
if not job._accepted:
job._cancel()
# clear the outgoing buffer as the tasks will be redelivered by
# the broker anyway.
if self.outbound_buffer:
self.outbound_buffer.clear()
try:
# ...but we must continue writing the payloads we already started
# to keep message boundaries.
# The messages may be NACK'ed later if synack is enabled.
if self._pool._state == RUN:
# flush outgoing buffers
intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
while self._active_writers:
writers = list(self._active_writers)
for gen in writers:
if (gen.__name__ == '_write_job' and
gen_not_started(gen)):
# has not started writing the job so can
# discard the task, but we must also remove
# it from the Pool._cache.
job_to_discard = None
for job in values(self._pool._cache):
if job._writer() is gen: # _writer is saferef
# removes from Pool._cache
job_to_discard = job
break
if job_to_discard:
job_to_discard.discard()
self._active_writers.discard(gen)
else:
try:
next(gen)
except StopIteration:
self._active_writers.discard(gen)
# workers may have exited in the meantime.
self.maintain_pool()
sleep(next(intervals)) # don't busyloop
finally:
self.outbound_buffer.clear()
self._active_writers.clear()
def flush(self):
if self.outbound_buffer:
self.outbound_buffer.clear()
try:
if self._pool._state == RUN:
# flush outgoing buffers
intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
while self._active_writers:
writers = list(self._active_writers)
for gen in writers:
if gen.gi_frame.f_lasti != -1: # generator started?
try:
next(gen)
except StopIteration:
self._active_writers.discard(gen)
# workers may have exited in the meantime.
self.maintain_pool()
sleep(next(intervals)) # don't busyloop
finally:
self.outbound_buffer.clear()
self._active_writers.clear()
def flush(self):
# cancel all tasks that have not been accepted to that NACK is sent.
for job in values(self._pool._cache):
if not job._accepted:
job._cancel()
# clear the outgoing buffer as the tasks will be redelivered by
# the broker anyway.
if self.outbound_buffer:
self.outbound_buffer.clear()
try:
# but we must continue to write the payloads we already started
# otherwise we will not recover the message boundaries.
# the messages will be NACK'ed later.
if self._pool._state == RUN:
# flush outgoing buffers
intervals = fxrange(0.01, 0.1, 0.01, repeatlast=True)
while self._active_writers:
writers = list(self._active_writers)
for gen in writers:
if (gen.__name__ == '_write_job'
and gen.gi_frame.f_lasti != -1):
# has not started writing the job so can
# safely discard
self._active_writers.discard(gen)
else:
try:
next(gen)
except StopIteration:
self._active_writers.discard(gen)
# workers may have exited in the meantime.
self.maintain_pool()
sleep(next(intervals)) # don't busyloop
finally:
self.outbound_buffer.clear()
self._active_writers.clear()
def skew(self, start=1.0, stop=None, step=1.0):
_next_skew = fxrange(start, stop, step, repeatlast=True).next
for task in self.tasks:
task.set(countdown=_next_skew())
return self
def skew(self, start=1.0, stop=None, step=1.0):
_next_skew = fxrange(start, stop, step, repeatlast=True).next
for task in self.tasks:
task.set(countdown=_next_skew())
return self
def skew(self, start=1.0, stop=None, step=1.0):
it = fxrange(start, stop, step, repeatlast=True)
for task in self.tasks:
task.set(countdown=next(it))
return self
def test_fxrange_no_repeatlast(self):
self.assertEqual(list(utils.fxrange(1.0, 3.0, 1.0)), [1.0, 2.0, 3.0])
def test_fxrange_no_repeatlast(self):
self.assertEqual(list(utils.fxrange(1.0, 3.0, 1.0)),
[1.0, 2.0, 3.0])
def skew(self, start=1.0, stop=None, step=1.0):
it = fxrange(start, stop, step, repeatlast=True)
for task in self.tasks:
task.set(countdown=next(it))
return self