def _wrapped_callback(*args, **kwargs): logger.debug("Future callback called.") try: return callback(*args, **kwargs) finally: self._executor.release_slot() self._executor.remove_done_future(self._original_future)
def add_worker(self, added_available_workers): """This method should only be called by ComputeResourceManager.""" logger.debug("Add worker: %s", added_available_workers) with self._lock: self._available_workers += added_available_workers new_effective_workers = self._calc_effective_workers( self._available_workers) if new_effective_workers > self._effective_workers: for _ in range(new_effective_workers - self._effective_workers): self._slots.release() self._effective_workers = new_effective_workers
def __init__(self, mgr: ComputingResourceManager, executor_id, available_workers): self._mgr = mgr self.closed = False self._available_workers = available_workers self._effective_workers = self._calc_effective_workers( self._available_workers) self._slots = threading.Semaphore(value=self._effective_workers) self._id = executor_id self._futures = set() self._lock = threading.Lock() logger.debug("Available workers: %s.", available_workers)
def submit(self, *args, **kwargs): """ Submit task. See concurrent.futures.Executor.submit() for details. This method is not thread safe. """ logger.debug("Task submitted to executor %s.", self._id) if self.closed: raise ComputingResourceManagerException( "Cannot submit task to a closed executor.") # Thread will wait on acquire(). self._slots.acquire() future = self._mgr.submit(*args, **kwargs) # set.add is atomic in c-python. self._futures.add(future) return WrappedFuture(self, future)
def _close(self): self.closed = True logger.debug("Executor is being closed, futures to wait: %s", self._futures) futures.wait(self._futures) logger.debug("Executor wait futures completed.") self._mgr.destroy_executor(self._id) logger.debug("Executor is closed.")