def _wrapped_callback(*args, **kwargs):
logger.debug("Future callback called.")
try:
return callback(*args, **kwargs)
finally:
self._executor.release_slot()
self._executor.remove_done_future(self._original_future)
def add_worker(self, added_available_workers):
"""This method should only be called by ComputeResourceManager."""
logger.debug("Add worker: %s", added_available_workers)
with self._lock:
self._available_workers += added_available_workers
new_effective_workers = self._calc_effective_workers(
self._available_workers)
if new_effective_workers > self._effective_workers:
for _ in range(new_effective_workers -
self._effective_workers):
self._slots.release()
self._effective_workers = new_effective_workers
def __init__(self, mgr: ComputingResourceManager, executor_id,
available_workers):
self._mgr = mgr
self.closed = False
self._available_workers = available_workers
self._effective_workers = self._calc_effective_workers(
self._available_workers)
self._slots = threading.Semaphore(value=self._effective_workers)
self._id = executor_id
self._futures = set()
self._lock = threading.Lock()
logger.debug("Available workers: %s.", available_workers)
def submit(self, *args, **kwargs):
"""
Submit task.
See concurrent.futures.Executor.submit() for details. This method is not thread safe.
"""
logger.debug("Task submitted to executor %s.", self._id)
if self.closed:
raise ComputingResourceManagerException(
"Cannot submit task to a closed executor.")
# Thread will wait on acquire().
self._slots.acquire()
future = self._mgr.submit(*args, **kwargs)
# set.add is atomic in c-python.
self._futures.add(future)
return WrappedFuture(self, future)
def _close(self):
self.closed = True
logger.debug("Executor is being closed, futures to wait: %s",
self._futures)
futures.wait(self._futures)
logger.debug("Executor wait futures completed.")
self._mgr.destroy_executor(self._id)
logger.debug("Executor is closed.")
