def exit(self):
"""Exit the scheduler by closing the queues and terminating the servant process."""
if not self._is_queue_closed:
utils.drain_queue(self._job_queue, close_queue=True)
self._job_queue.join_thread()
utils.drain_queue(self._result_queue, close_queue=True)
self._result_queue.join_thread()
self._is_queue_closed = True
self.interrupt()
# wait a bit for the subprocess to exit gracefully
n_retries = 3
while self._servant.is_alive() and n_retries > 0:
n_retries -= 1
time.sleep(0.05)
self._servant.terminate()
def _run_servant(self):
"""Run the pool of workers on the dispatched jobs, fetched from the job queue and
collect the results into the result queue.
Notes:
The runner will take as long as all jobs from the job queue finish before any results are
written to the result queue.
"""
# TODO: Switch backend back to default "loky", after the leakage of semaphores is fixed
with joblib.Parallel(n_jobs=self.n_parallel,
backend="multiprocessing") as parallel:
while not self._interrupt_event.is_set():
current_jobs = utils.drain_queue(self._job_queue)
if not current_jobs:
continue
# the order of the results corresponds to the that of the jobs
# and the IDs don't need to be shuffled.
ids = [job.id for job in current_jobs]
# TODO: in a future version of joblib, this could be a generator and then the inputs
# would be stored immediately in the results queue. Be ready to update whenever
# this PR gets merged: https://github.com/joblib/joblib/pull/588
results = parallel(
joblib.delayed(job)() for job in current_jobs)
assert len(ids) == len(results)
for res in results:
self._result_queue.put_nowait(res)
def collect(self, n_results: int, timeout: float = None) -> List[Result]:
"""Collect all the available results or wait until they become available.
Args:
n_results: :obj:`int`, number of results to wait for.
If `n_results` ≤ 0 then all available results will be returned.
timeout: (optional) :obj:`float`, number of seconds to wait for results to appear.
If None (default) then it will wait until all `n_results` are collected.
Returns:
A list of :class:`Result` objects with length `n_results` at least.
Notes:
If `n_results` is overestimated and timeout is None, then this method will hang forever.
Therefore it is recommended that a timeout is set.
Raises:
:obj:`TimeoutError`: if more than `timeout` seconds elapse before a :class:`Result` is collected.
"""
if n_results > 0:
results = []
for i in range(n_results):
results.append(
self._result_queue.get(block=True, timeout=timeout))
else:
results = utils.drain_queue(self._result_queue)
return results