def wait(self, fs=None, throw_except=True, return_when=ALL_COMPLETED, download_results=False, timeout=None, threadpool_size=THREADPOOL_SIZE, wait_dur_sec=WAIT_DUR_SEC): """ Wait for the Future instances (possibly created by different Executor instances) given by fs to complete. Returns a named 2-tuple of sets. The first set, named done, contains the futures that completed (finished or cancelled futures) before the wait completed. The second set, named not_done, contains the futures that did not complete (pending or running futures). timeout can be used to control the maximum number of seconds to wait before returning. :param fs: Futures list. Default None :param throw_except: Re-raise exception if call raised. Default True. :param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS` :param download_results: Download results. Default false (Only get statuses) :param timeout: Timeout of waiting for results. :param threadpool_size: Number of threads to use. Default 64 :param wait_dur_sec: Time interval between each check. :return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` is a list of futures that have not completed. :rtype: 2-tuple of list """ futures = fs or self.futures if type(futures) != list: futures = [futures] # Start waiting for results try: wait(fs=futures, internal_storage=self.internal_storage, job_monitor=self.job_monitor, download_results=download_results, throw_except=throw_except, return_when=return_when, timeout=timeout, threadpool_size=threadpool_size, wait_dur_sec=wait_dur_sec) except Exception as e: self.invoker.stop() if not fs and is_notebook(): del self.futures[len(self.futures) - len(futures):] if self.data_cleaner and not self.is_lithops_worker: self.clean(clean_cloudobjects=False, force=True) raise e finally: present_jobs = {f.job_key for f in futures} self.job_monitor.stop(present_jobs) if self.data_cleaner and not self.is_lithops_worker: self.clean(clean_cloudobjects=False) if download_results: fs_done = [f for f in futures if f.done] fs_notdone = [f for f in futures if not f.done] else: fs_done = [f for f in futures if f.success or f.done] fs_notdone = [f for f in futures if not f.success and not f.done] return fs_done, fs_notdone
def wait(self, fs=None, throw_except=True, return_when=ALL_COMPLETED, download_results=False, timeout=None, THREADPOOL_SIZE=128, WAIT_DUR_SEC=1): """ Wait for the Future instances (possibly created by different Executor instances) given by fs to complete. Returns a named 2-tuple of sets. The first set, named done, contains the futures that completed (finished or cancelled futures) before the wait completed. The second set, named not_done, contains the futures that did not complete (pending or running futures). timeout can be used to control the maximum number of seconds to wait before returning. :param fs: Futures list. Default None :param throw_except: Re-raise exception if call raised. Default True. :param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS` :param download_results: Download results. Default false (Only get statuses) :param timeout: Timeout of waiting for results. :param THREADPOOL_SIZE: Number of threads to use. Default 64 :param WAIT_DUR_SEC: Time interval between each check. :return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` is a list of futures that have not completed. :rtype: 2-tuple of list """ futures = fs or self.futures if type(futures) != list: futures = [futures] if not futures: raise Exception( 'You must run the call_async(), map() or map_reduce(), or provide' ' a list of futures before calling the wait()/get_result() method' ) if download_results: msg = 'ExecutorID {} - Getting results'.format(self.executor_id) fs_done = [f for f in futures if f.done] fs_not_done = [f for f in futures if not f.done] fs_not_ready = [f for f in futures if not f.ready] else: msg = 'ExecutorID {} - Waiting for functions to complete'.format( self.executor_id) fs_done = [f for f in futures if f.ready or f.done] fs_not_done = [f for f in futures if not f.ready and not f.done] fs_not_ready = [f for f in futures if not f.ready] if not fs_not_done: return fs_done, fs_not_done logger.info(msg) if is_unix_system() and timeout is not None: logger.debug( 'Setting waiting timeout to {} seconds'.format(timeout)) error_msg = 'Timeout of {} seconds exceeded waiting for function activations to finish'.format( timeout) signal.signal(signal.SIGALRM, partial(timeout_handler, error_msg)) signal.alarm(timeout) pbar = None error = False if not self.is_lithops_worker and self.setup_progressbar and fs_not_ready: from tqdm.auto import tqdm if is_notebook(): pbar = tqdm(bar_format='{n}/|/ {n_fmt}/{total_fmt}', total=len(fs_not_done)) # ncols=800 else: print() pbar = tqdm(bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ', total=len(fs_not_done), disable=None) try: if self.rabbitmq_monitor: logger.debug('Using RabbitMQ to monitor function activations') wait_rabbitmq(futures, self.internal_storage, rabbit_amqp_url=self.rabbit_amqp_url, download_results=download_results, throw_except=throw_except, pbar=pbar, return_when=return_when, THREADPOOL_SIZE=THREADPOOL_SIZE) else: wait_storage(futures, self.internal_storage, download_results=download_results, throw_except=throw_except, return_when=return_when, pbar=pbar, THREADPOOL_SIZE=THREADPOOL_SIZE, WAIT_DUR_SEC=WAIT_DUR_SEC) except KeyboardInterrupt as e: if download_results: not_dones_call_ids = [(f.job_id, f.call_id) for f in futures if not f.done] else: not_dones_call_ids = [(f.job_id, f.call_id) for f in futures if not f.ready and not f.done] msg = ('ExecutorID {} - Cancelled - Total Activations not done: {}' .format(self.executor_id, len(not_dones_call_ids))) if pbar: pbar.close() print() logger.info(msg) error = True if self.data_cleaner and not self.is_lithops_worker: self.clean(clean_cloudobjects=False, force=True) raise e except Exception as e: error = True if self.data_cleaner and not self.is_lithops_worker: self.clean(clean_cloudobjects=False, force=True) raise e finally: self.invoker.stop() if is_unix_system(): signal.alarm(0) if pbar and not pbar.disable: pbar.close() if not is_notebook(): print() if self.data_cleaner and not self.is_lithops_worker: self.clean(clean_cloudobjects=False) if not fs and error and is_notebook(): del self.futures[len(self.futures) - len(futures):] if download_results: fs_done = [f for f in futures if f.done] fs_notdone = [f for f in futures if not f.done] else: fs_done = [f for f in futures if f.ready or f.done] fs_notdone = [f for f in futures if not f.ready and not f.done] return fs_done, fs_notdone
def wait( self, fs: Optional[Union[ResponseFuture, FuturesList, List[ResponseFuture]]] = None, throw_except: Optional[bool] = True, return_when: Optional[Any] = ALL_COMPLETED, download_results: Optional[bool] = False, timeout: Optional[int] = None, threadpool_size: Optional[int] = THREADPOOL_SIZE, wait_dur_sec: Optional[int] = WAIT_DUR_SEC ) -> Tuple[FuturesList, FuturesList]: """ Wait for the Future instances (possibly created by different Executor instances) given by fs to complete. Returns a named 2-tuple of sets. The first set, named done, contains the futures that completed (finished or cancelled futures) before the wait completed. The second set, named not_done, contains the futures that did not complete (pending or running futures). timeout can be used to control the maximum number of seconds to wait before returning. :param fs: Futures list. Default None :param throw_except: Re-raise exception if call raised. Default True :param return_when: Percentage of done futures :param download_results: Download results. Default false (Only get statuses) :param timeout: Timeout of waiting for results :param threadpool_size: Number of threads to use. Default 64 :param wait_dur_sec: Time interval between each check :return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` is a list of futures that have not completed. """ futures = fs or self.futures if type(futures) != list and type(futures) != FuturesList: futures = [futures] # Start waiting for results try: wait(fs=futures, internal_storage=self.internal_storage, job_monitor=self.job_monitor, download_results=download_results, throw_except=throw_except, return_when=return_when, timeout=timeout, threadpool_size=threadpool_size, wait_dur_sec=wait_dur_sec) if self.data_cleaner and return_when == ALL_COMPLETED: present_jobs = {f.job_key for f in futures} self.compute_handler.clear(present_jobs) self.clean(clean_cloudobjects=False) except (KeyboardInterrupt, Exception) as e: self.invoker.stop() self.job_monitor.stop() if not fs and is_notebook(): del self.futures[len(self.futures) - len(futures):] if self.data_cleaner: present_jobs = {f.job_key for f in futures} self.compute_handler.clear(present_jobs) self.clean(clean_cloudobjects=False, force=True) raise e if download_results: fs_done = [f for f in futures if f.done] fs_notdone = [f for f in futures if not f.done] else: fs_done = [f for f in futures if f.success or f.done] fs_notdone = [f for f in futures if not f.success and not f.done] return create_futures_list(fs_done, self), create_futures_list( fs_notdone, self)
def wait(fs, internal_storage=None, throw_except=True, timeout=None, return_when=ALL_COMPLETED, download_results=False, job_monitor=None, threadpool_size=THREADPOOL_SIZE, wait_dur_sec=WAIT_DUR_SEC): """ Wait for the Future instances (possibly created by different Executor instances) given by fs to complete. Returns a named 2-tuple of sets. The first set, named done, contains the futures that completed (finished or cancelled futures) before the wait completed. The second set, named not_done, contains the futures that did not complete (pending or running futures). timeout can be used to control the maximum number of seconds to wait before returning. :param fs: Futures list. Default None :param throw_except: Re-raise exception if call raised. Default True. :param return_when: Percentage of done futures :param download_results: Download results. Default false (Only get statuses) :param timeout: Timeout of waiting for results. :param threadpool_zise: Number of threads to use. Default 64 :param wait_dur_sec: Time interval between each check. :return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` is a list of futures that have not completed. :rtype: 2-tuple of list """ if not fs: return if type(fs) != list and type(fs) != FuturesList: fs = [fs] if download_results: msg = 'ExecutorID {} - Getting results from functions'.format( fs[0].executor_id) fs_done = [f for f in fs if f.done] fs_not_done = [f for f in fs if not f.done] else: msg = 'ExecutorID {} - Waiting for {}% of functions to complete'.format( fs[0].executor_id, return_when) fs_done = [f for f in fs if f.success or f.done] fs_not_done = [f for f in fs if not (f.success or f.done)] logger.info(msg) if not fs_not_done: return fs_done, fs_not_done if is_unix_system() and timeout is not None: logger.debug('Setting waiting timeout to {} seconds'.format(timeout)) error_msg = 'Timeout of {} seconds exceeded waiting for function activations to finish'.format( timeout) signal.signal(signal.SIGALRM, partial(timeout_handler, error_msg)) signal.alarm(timeout) # Setup progress bar pbar = None if not is_lithops_worker() and logger.getEffectiveLevel() == logging.INFO: from tqdm.auto import tqdm if not is_notebook(): print() pbar = tqdm(bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ', total=len(fs), disable=None) pbar.update(len(fs_done)) try: executors_data = _create_executors_data_from_futures( fs, internal_storage) if not job_monitor: for executor_data in executors_data: job_monitor = JobMonitor( executor_id=executor_data.executor_id, internal_storage=executor_data.internal_storage) job_monitor.start(fs=executor_data.futures) sleep_sec = wait_dur_sec if job_monitor.backend == 'storage' else 0.3 if return_when == ALWAYS: for executor_data in executors_data: _get_executor_data(fs, executor_data, pbar=pbar, throw_except=throw_except, download_results=download_results, threadpool_size=threadpool_size) else: while not _check_done(fs, return_when, download_results): for executor_data in executors_data: new_data = _get_executor_data( fs, executor_data, pbar=pbar, throw_except=throw_except, download_results=download_results, threadpool_size=threadpool_size) time.sleep(0 if new_data else sleep_sec) except KeyboardInterrupt as e: if download_results: not_dones_call_ids = [(f.job_id, f.call_id) for f in fs if not f.done] else: not_dones_call_ids = [(f.job_id, f.call_id) for f in fs if not f.success and not f.done] msg = ('Cancelled - Total Activations not done: {}'.format( len(not_dones_call_ids))) if pbar: pbar.close() print() logger.info(msg) raise e except Exception as e: raise e finally: if is_unix_system(): signal.alarm(0) if pbar and not pbar.disable: pbar.close() if not is_notebook(): print() if download_results: fs_done = [f for f in fs if f.done] fs_notdone = [f for f in fs if not f.done] else: fs_done = [f for f in fs if f.success or f.done] fs_notdone = [f for f in fs if not f.success and not f.done] return fs_done, fs_notdone
def wait(fs, throw_except=True, return_when=ALL_COMPLETED, download_results=False, timeout=None, THREADPOOL_SIZE=128, WAIT_DUR_SEC=1, internal_storage=None): """ Wait for the Future instances (possibly created by different Executor instances) given by fs to complete. Returns a named 2-tuple of sets. The first set, named done, contains the futures that completed (finished or cancelled futures) before the wait completed. The second set, named not_done, contains the futures that did not complete (pending or running futures). timeout can be used to control the maximum number of seconds to wait before returning. :param fs: Futures list. Default None :param throw_except: Re-raise exception if call raised. Default True. :param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS` :param download_results: Download results. Default false (Only get statuses) :param timeout: Timeout of waiting for results. :param THREADPOOL_SIZE: Number of threads to use. Default 64 :param WAIT_DUR_SEC: Time interval between each check. :return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` is a list of futures that have not completed. :rtype: 2-tuple of list """ if not internal_storage: internal_storage = InternalStorage(fs[0].storage_config) if type(fs) != list: fs = [fs] setup_progressbar = (not is_lithops_worker() and logger.getEffectiveLevel() == logging.INFO) if download_results: msg = 'Getting results from functions' fs_done = [f for f in fs if f.done] fs_not_done = [f for f in fs if not f.done] # fs_not_ready = [f for f in futures if not f.ready and not f.done] else: msg = 'Waiting for functions to complete' fs_done = [f for f in fs if f.ready or f.done] fs_not_done = [f for f in fs if not f.done] # fs_not_ready = [f for f in futures if not f.ready and not f.done] if not fs_not_done: return fs_done, fs_not_done logger.info(msg) if is_unix_system() and timeout is not None: logger.debug('Setting waiting timeout to {} seconds'.format(timeout)) error_msg = 'Timeout of {} seconds exceeded waiting for function activations to finish'.format( timeout) signal.signal(signal.SIGALRM, partial(timeout_handler, error_msg)) signal.alarm(timeout) pbar = None if not is_lithops_worker() and setup_progressbar: from tqdm.auto import tqdm if not is_notebook(): print() pbar = tqdm(bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ', total=len(fs_not_done), disable=None) try: wait_storage(fs, internal_storage, download_results=download_results, throw_except=throw_except, return_when=return_when, pbar=pbar, THREADPOOL_SIZE=THREADPOOL_SIZE, WAIT_DUR_SEC=WAIT_DUR_SEC) except KeyboardInterrupt as e: if download_results: not_dones_call_ids = [(f.job_id, f.call_id) for f in fs if not f.done] else: not_dones_call_ids = [(f.job_id, f.call_id) for f in fs if not f.ready and not f.done] msg = ('Cancelled - Total Activations not done: {}'.format( len(not_dones_call_ids))) if pbar: pbar.close() print() logger.info(msg) raise e except Exception as e: raise e finally: if is_unix_system(): signal.alarm(0) if pbar and not pbar.disable: pbar.close() if not is_notebook(): print() if download_results: fs_done = [f for f in fs if f.done] fs_notdone = [f for f in fs if not f.done] else: fs_done = [f for f in fs if f.ready or f.done] fs_notdone = [f for f in fs if not f.ready and not f.done] return fs_done, fs_notdone