def dispatch(self, task): try: self.pool_channel.send(WorkerTask(task.id, task.payload)) except (pickle.PicklingError, TypeError) as error: raise error except (OSError, EnvironmentError, TypeError) as error: raise BrokenProcessPool(error)
def new_worker(self): try: worker = launch_process( worker_process, self.worker_parameters, self.workers_channel) self.workers[worker.pid] = worker except (OSError, EnvironmentError) as error: raise BrokenProcessPool(error)
def submit(self, func, *args, **kwargs): """Submit a function to the pool, `self.submit(function,arg1,arg2,arg3=3)`""" with self._shutdown_lock: if PY3 and self._broken: raise BrokenProcessPool( "A child process terminated " "abruptly, the process pool is not usable anymore") if self._shutdown_thread: raise RuntimeError( "cannot schedule new futures after shutdown") callback = kwargs.pop("callback", self.default_callback) future = NewFuture( self._timeout, args, kwargs, callback=callback, catch_exception=self.catch_exception, ) w = _WorkItem(future, func, args, kwargs) self._pending_work_items[self._queue_count] = w self._work_ids.put(self._queue_count) self._queue_count += 1 self._result_queue.put(None) self._start_queue_management_thread() if PY2: self._adjust_process_count() self._all_futures.add(future) return future
def receive(self, timeout): try: if self.pool_channel.poll(timeout): return self.pool_channel.recv() else: return NoMessage() except (OSError, EnvironmentError) as error: raise BrokenProcessPool(error)
def find_expired_task(self, worker_id): tasks = tuple(self.task_manager.tasks.values()) running_tasks = tuple(t for t in tasks if t.worker_id != 0) if running_tasks: return task_worker_lookup(running_tasks, worker_id) else: raise BrokenProcessPool("All workers expired")
def new_worker(self): try: worker = launch_process(WORKERS_NAME, worker_process, False, self.mp_context, self.worker_parameters, self.workers_channel) self.workers[worker.pid] = worker except (OSError, EnvironmentError) as error: raise BrokenProcessPool(error)
def stop_worker(self, worker_id, force=False): try: if force: stop_process(self.workers.pop(worker_id)) else: with self.workers_channel.lock: stop_process(self.workers.pop(worker_id)) except ChannelError as error: raise BrokenProcessPool(error) except KeyError: return # worker already expired
def submit(self, fn, *args, **kwargs): self.submit_count += 1 result = concurrent.futures.Future() result.set_exception(BrokenProcessPool('This pool is broken, yo')) return result
def get_future_result(future, future_mapping): try: return future.result() except BrokenProcessPool as e: raise BrokenProcessPool('Broken pool {}'.format( future_mapping[future])) from e