def exit(self): """Exit the scheduler by closing the queues and terminating the servant process.""" if not self._is_queue_closed: utils.drain_queue(self._job_queue, close_queue=True) self._job_queue.join_thread() utils.drain_queue(self._result_queue, close_queue=True) self._result_queue.join_thread() self._is_queue_closed = True self.interrupt() # wait a bit for the subprocess to exit gracefully n_retries = 3 while self._servant.is_alive() and n_retries > 0: n_retries -= 1 time.sleep(0.05) self._servant.terminate()
def _run_servant(self): """Run the pool of workers on the dispatched jobs, fetched from the job queue and collect the results into the result queue. Notes: The runner will take as long as all jobs from the job queue finish before any results are written to the result queue. """ # TODO: Switch backend back to default "loky", after the leakage of semaphores is fixed with joblib.Parallel(n_jobs=self.n_parallel, backend="multiprocessing") as parallel: while not self._interrupt_event.is_set(): current_jobs = utils.drain_queue(self._job_queue) if not current_jobs: continue # the order of the results corresponds to the that of the jobs # and the IDs don't need to be shuffled. ids = [job.id for job in current_jobs] # TODO: in a future version of joblib, this could be a generator and then the inputs # would be stored immediately in the results queue. Be ready to update whenever # this PR gets merged: https://github.com/joblib/joblib/pull/588 results = parallel( joblib.delayed(job)() for job in current_jobs) assert len(ids) == len(results) for res in results: self._result_queue.put_nowait(res)
def collect(self, n_results: int, timeout: float = None) -> List[Result]: """Collect all the available results or wait until they become available. Args: n_results: :obj:`int`, number of results to wait for. If `n_results` ≤ 0 then all available results will be returned. timeout: (optional) :obj:`float`, number of seconds to wait for results to appear. If None (default) then it will wait until all `n_results` are collected. Returns: A list of :class:`Result` objects with length `n_results` at least. Notes: If `n_results` is overestimated and timeout is None, then this method will hang forever. Therefore it is recommended that a timeout is set. Raises: :obj:`TimeoutError`: if more than `timeout` seconds elapse before a :class:`Result` is collected. """ if n_results > 0: results = [] for i in range(n_results): results.append( self._result_queue.get(block=True, timeout=timeout)) else: results = utils.drain_queue(self._result_queue) return results