def dispatch(self, task):
try:
self.pool_channel.send(WorkerTask(task.id, task.payload))
except (pickle.PicklingError, TypeError) as error:
raise error
except (OSError, EnvironmentError, TypeError) as error:
raise BrokenProcessPool(error)
def new_worker(self):
try:
worker = launch_process(
worker_process, self.worker_parameters, self.workers_channel)
self.workers[worker.pid] = worker
except (OSError, EnvironmentError) as error:
raise BrokenProcessPool(error)
def submit(self, func, *args, **kwargs):
"""Submit a function to the pool, `self.submit(function,arg1,arg2,arg3=3)`"""
with self._shutdown_lock:
if PY3 and self._broken:
raise BrokenProcessPool(
"A child process terminated "
"abruptly, the process pool is not usable anymore")
if self._shutdown_thread:
raise RuntimeError(
"cannot schedule new futures after shutdown")
callback = kwargs.pop("callback", self.default_callback)
future = NewFuture(
self._timeout,
args,
kwargs,
callback=callback,
catch_exception=self.catch_exception,
)
w = _WorkItem(future, func, args, kwargs)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._queue_count += 1
self._result_queue.put(None)
self._start_queue_management_thread()
if PY2:
self._adjust_process_count()
self._all_futures.add(future)
return future
def receive(self, timeout):
try:
if self.pool_channel.poll(timeout):
return self.pool_channel.recv()
else:
return NoMessage()
except (OSError, EnvironmentError) as error:
raise BrokenProcessPool(error)
def find_expired_task(self, worker_id):
tasks = tuple(self.task_manager.tasks.values())
running_tasks = tuple(t for t in tasks if t.worker_id != 0)
if running_tasks:
return task_worker_lookup(running_tasks, worker_id)
else:
raise BrokenProcessPool("All workers expired")
def new_worker(self):
try:
worker = launch_process(WORKERS_NAME, worker_process, False,
self.mp_context, self.worker_parameters,
self.workers_channel)
self.workers[worker.pid] = worker
except (OSError, EnvironmentError) as error:
raise BrokenProcessPool(error)
def stop_worker(self, worker_id, force=False):
try:
if force:
stop_process(self.workers.pop(worker_id))
else:
with self.workers_channel.lock:
stop_process(self.workers.pop(worker_id))
except ChannelError as error:
raise BrokenProcessPool(error)
except KeyError:
return # worker already expired
def submit(self, fn, *args, **kwargs):
self.submit_count += 1
result = concurrent.futures.Future()
result.set_exception(BrokenProcessPool('This pool is broken, yo'))
return result
def get_future_result(future, future_mapping):
try:
return future.result()
except BrokenProcessPool as e:
raise BrokenProcessPool('Broken pool {}'.format(
future_mapping[future])) from e