def wait(self,
fs=None,
throw_except=True,
return_when=ALL_COMPLETED,
download_results=False,
timeout=None,
threadpool_size=THREADPOOL_SIZE,
wait_dur_sec=WAIT_DUR_SEC):
"""
Wait for the Future instances (possibly created by different Executor instances)
given by fs to complete. Returns a named 2-tuple of sets. The first set, named done,
contains the futures that completed (finished or cancelled futures) before the wait
completed. The second set, named not_done, contains the futures that did not complete
(pending or running futures). timeout can be used to control the maximum number of
seconds to wait before returning.
:param fs: Futures list. Default None
:param throw_except: Re-raise exception if call raised. Default True.
:param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS`
:param download_results: Download results. Default false (Only get statuses)
:param timeout: Timeout of waiting for results.
:param threadpool_size: Number of threads to use. Default 64
:param wait_dur_sec: Time interval between each check.
:return: `(fs_done, fs_notdone)`
where `fs_done` is a list of futures that have completed
and `fs_notdone` is a list of futures that have not completed.
:rtype: 2-tuple of list
"""
futures = fs or self.futures
if type(futures) != list:
futures = [futures]
# Start waiting for results
try:
wait(fs=futures,
internal_storage=self.internal_storage,
job_monitor=self.job_monitor,
download_results=download_results,
throw_except=throw_except,
return_when=return_when,
timeout=timeout,
threadpool_size=threadpool_size,
wait_dur_sec=wait_dur_sec)
except Exception as e:
self.invoker.stop()
if not fs and is_notebook():
del self.futures[len(self.futures) - len(futures):]
if self.data_cleaner and not self.is_lithops_worker:
self.clean(clean_cloudobjects=False, force=True)
raise e
finally:
present_jobs = {f.job_key for f in futures}
self.job_monitor.stop(present_jobs)
if self.data_cleaner and not self.is_lithops_worker:
self.clean(clean_cloudobjects=False)
if download_results:
fs_done = [f for f in futures if f.done]
fs_notdone = [f for f in futures if not f.done]
else:
fs_done = [f for f in futures if f.success or f.done]
fs_notdone = [f for f in futures if not f.success and not f.done]
return fs_done, fs_notdone
def wait(self,
fs=None,
throw_except=True,
return_when=ALL_COMPLETED,
download_results=False,
timeout=None,
THREADPOOL_SIZE=128,
WAIT_DUR_SEC=1):
"""
Wait for the Future instances (possibly created by different Executor instances)
given by fs to complete. Returns a named 2-tuple of sets. The first set, named done,
contains the futures that completed (finished or cancelled futures) before the wait
completed. The second set, named not_done, contains the futures that did not complete
(pending or running futures). timeout can be used to control the maximum number of
seconds to wait before returning.
:param fs: Futures list. Default None
:param throw_except: Re-raise exception if call raised. Default True.
:param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS`
:param download_results: Download results. Default false (Only get statuses)
:param timeout: Timeout of waiting for results.
:param THREADPOOL_SIZE: Number of threads to use. Default 64
:param WAIT_DUR_SEC: Time interval between each check.
:return: `(fs_done, fs_notdone)`
where `fs_done` is a list of futures that have completed
and `fs_notdone` is a list of futures that have not completed.
:rtype: 2-tuple of list
"""
futures = fs or self.futures
if type(futures) != list:
futures = [futures]
if not futures:
raise Exception(
'You must run the call_async(), map() or map_reduce(), or provide'
' a list of futures before calling the wait()/get_result() method'
)
if download_results:
msg = 'ExecutorID {} - Getting results'.format(self.executor_id)
fs_done = [f for f in futures if f.done]
fs_not_done = [f for f in futures if not f.done]
fs_not_ready = [f for f in futures if not f.ready]
else:
msg = 'ExecutorID {} - Waiting for functions to complete'.format(
self.executor_id)
fs_done = [f for f in futures if f.ready or f.done]
fs_not_done = [f for f in futures if not f.ready and not f.done]
fs_not_ready = [f for f in futures if not f.ready]
if not fs_not_done:
return fs_done, fs_not_done
logger.info(msg)
if is_unix_system() and timeout is not None:
logger.debug(
'Setting waiting timeout to {} seconds'.format(timeout))
error_msg = 'Timeout of {} seconds exceeded waiting for function activations to finish'.format(
timeout)
signal.signal(signal.SIGALRM, partial(timeout_handler, error_msg))
signal.alarm(timeout)
pbar = None
error = False
if not self.is_lithops_worker and self.setup_progressbar and fs_not_ready:
from tqdm.auto import tqdm
if is_notebook():
pbar = tqdm(bar_format='{n}/|/ {n_fmt}/{total_fmt}',
total=len(fs_not_done)) # ncols=800
else:
print()
pbar = tqdm(bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ',
total=len(fs_not_done),
disable=None)
try:
if self.rabbitmq_monitor:
logger.debug('Using RabbitMQ to monitor function activations')
wait_rabbitmq(futures,
self.internal_storage,
rabbit_amqp_url=self.rabbit_amqp_url,
download_results=download_results,
throw_except=throw_except,
pbar=pbar,
return_when=return_when,
THREADPOOL_SIZE=THREADPOOL_SIZE)
else:
wait_storage(futures,
self.internal_storage,
download_results=download_results,
throw_except=throw_except,
return_when=return_when,
pbar=pbar,
THREADPOOL_SIZE=THREADPOOL_SIZE,
WAIT_DUR_SEC=WAIT_DUR_SEC)
except KeyboardInterrupt as e:
if download_results:
not_dones_call_ids = [(f.job_id, f.call_id) for f in futures
if not f.done]
else:
not_dones_call_ids = [(f.job_id, f.call_id) for f in futures
if not f.ready and not f.done]
msg = ('ExecutorID {} - Cancelled - Total Activations not done: {}'
.format(self.executor_id, len(not_dones_call_ids)))
if pbar:
pbar.close()
print()
logger.info(msg)
error = True
if self.data_cleaner and not self.is_lithops_worker:
self.clean(clean_cloudobjects=False, force=True)
raise e
except Exception as e:
error = True
if self.data_cleaner and not self.is_lithops_worker:
self.clean(clean_cloudobjects=False, force=True)
raise e
finally:
self.invoker.stop()
if is_unix_system():
signal.alarm(0)
if pbar and not pbar.disable:
pbar.close()
if not is_notebook():
print()
if self.data_cleaner and not self.is_lithops_worker:
self.clean(clean_cloudobjects=False)
if not fs and error and is_notebook():
del self.futures[len(self.futures) - len(futures):]
if download_results:
fs_done = [f for f in futures if f.done]
fs_notdone = [f for f in futures if not f.done]
else:
fs_done = [f for f in futures if f.ready or f.done]
fs_notdone = [f for f in futures if not f.ready and not f.done]
return fs_done, fs_notdone
def wait(
self,
fs: Optional[Union[ResponseFuture, FuturesList,
List[ResponseFuture]]] = None,
throw_except: Optional[bool] = True,
return_when: Optional[Any] = ALL_COMPLETED,
download_results: Optional[bool] = False,
timeout: Optional[int] = None,
threadpool_size: Optional[int] = THREADPOOL_SIZE,
wait_dur_sec: Optional[int] = WAIT_DUR_SEC
) -> Tuple[FuturesList, FuturesList]:
"""
Wait for the Future instances (possibly created by different Executor instances)
given by fs to complete. Returns a named 2-tuple of sets. The first set, named done,
contains the futures that completed (finished or cancelled futures) before the wait
completed. The second set, named not_done, contains the futures that did not complete
(pending or running futures). timeout can be used to control the maximum number of
seconds to wait before returning.
:param fs: Futures list. Default None
:param throw_except: Re-raise exception if call raised. Default True
:param return_when: Percentage of done futures
:param download_results: Download results. Default false (Only get statuses)
:param timeout: Timeout of waiting for results
:param threadpool_size: Number of threads to use. Default 64
:param wait_dur_sec: Time interval between each check
:return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` is a list of futures that have not completed.
"""
futures = fs or self.futures
if type(futures) != list and type(futures) != FuturesList:
futures = [futures]
# Start waiting for results
try:
wait(fs=futures,
internal_storage=self.internal_storage,
job_monitor=self.job_monitor,
download_results=download_results,
throw_except=throw_except,
return_when=return_when,
timeout=timeout,
threadpool_size=threadpool_size,
wait_dur_sec=wait_dur_sec)
if self.data_cleaner and return_when == ALL_COMPLETED:
present_jobs = {f.job_key for f in futures}
self.compute_handler.clear(present_jobs)
self.clean(clean_cloudobjects=False)
except (KeyboardInterrupt, Exception) as e:
self.invoker.stop()
self.job_monitor.stop()
if not fs and is_notebook():
del self.futures[len(self.futures) - len(futures):]
if self.data_cleaner:
present_jobs = {f.job_key for f in futures}
self.compute_handler.clear(present_jobs)
self.clean(clean_cloudobjects=False, force=True)
raise e
if download_results:
fs_done = [f for f in futures if f.done]
fs_notdone = [f for f in futures if not f.done]
else:
fs_done = [f for f in futures if f.success or f.done]
fs_notdone = [f for f in futures if not f.success and not f.done]
return create_futures_list(fs_done, self), create_futures_list(
fs_notdone, self)
def wait(fs,
internal_storage=None,
throw_except=True,
timeout=None,
return_when=ALL_COMPLETED,
download_results=False,
job_monitor=None,
threadpool_size=THREADPOOL_SIZE,
wait_dur_sec=WAIT_DUR_SEC):
"""
Wait for the Future instances (possibly created by different Executor instances)
given by fs to complete. Returns a named 2-tuple of sets. The first set, named done,
contains the futures that completed (finished or cancelled futures) before the wait
completed. The second set, named not_done, contains the futures that did not complete
(pending or running futures). timeout can be used to control the maximum number of
seconds to wait before returning.
:param fs: Futures list. Default None
:param throw_except: Re-raise exception if call raised. Default True.
:param return_when: Percentage of done futures
:param download_results: Download results. Default false (Only get statuses)
:param timeout: Timeout of waiting for results.
:param threadpool_zise: Number of threads to use. Default 64
:param wait_dur_sec: Time interval between each check.
:return: `(fs_done, fs_notdone)`
where `fs_done` is a list of futures that have completed
and `fs_notdone` is a list of futures that have not completed.
:rtype: 2-tuple of list
"""
if not fs:
return
if type(fs) != list and type(fs) != FuturesList:
fs = [fs]
if download_results:
msg = 'ExecutorID {} - Getting results from functions'.format(
fs[0].executor_id)
fs_done = [f for f in fs if f.done]
fs_not_done = [f for f in fs if not f.done]
else:
msg = 'ExecutorID {} - Waiting for {}% of functions to complete'.format(
fs[0].executor_id, return_when)
fs_done = [f for f in fs if f.success or f.done]
fs_not_done = [f for f in fs if not (f.success or f.done)]
logger.info(msg)
if not fs_not_done:
return fs_done, fs_not_done
if is_unix_system() and timeout is not None:
logger.debug('Setting waiting timeout to {} seconds'.format(timeout))
error_msg = 'Timeout of {} seconds exceeded waiting for function activations to finish'.format(
timeout)
signal.signal(signal.SIGALRM, partial(timeout_handler, error_msg))
signal.alarm(timeout)
# Setup progress bar
pbar = None
if not is_lithops_worker() and logger.getEffectiveLevel() == logging.INFO:
from tqdm.auto import tqdm
if not is_notebook():
print()
pbar = tqdm(bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ',
total=len(fs),
disable=None)
pbar.update(len(fs_done))
try:
executors_data = _create_executors_data_from_futures(
fs, internal_storage)
if not job_monitor:
for executor_data in executors_data:
job_monitor = JobMonitor(
executor_id=executor_data.executor_id,
internal_storage=executor_data.internal_storage)
job_monitor.start(fs=executor_data.futures)
sleep_sec = wait_dur_sec if job_monitor.backend == 'storage' else 0.3
if return_when == ALWAYS:
for executor_data in executors_data:
_get_executor_data(fs,
executor_data,
pbar=pbar,
throw_except=throw_except,
download_results=download_results,
threadpool_size=threadpool_size)
else:
while not _check_done(fs, return_when, download_results):
for executor_data in executors_data:
new_data = _get_executor_data(
fs,
executor_data,
pbar=pbar,
throw_except=throw_except,
download_results=download_results,
threadpool_size=threadpool_size)
time.sleep(0 if new_data else sleep_sec)
except KeyboardInterrupt as e:
if download_results:
not_dones_call_ids = [(f.job_id, f.call_id) for f in fs
if not f.done]
else:
not_dones_call_ids = [(f.job_id, f.call_id) for f in fs
if not f.success and not f.done]
msg = ('Cancelled - Total Activations not done: {}'.format(
len(not_dones_call_ids)))
if pbar:
pbar.close()
print()
logger.info(msg)
raise e
except Exception as e:
raise e
finally:
if is_unix_system():
signal.alarm(0)
if pbar and not pbar.disable:
pbar.close()
if not is_notebook():
print()
if download_results:
fs_done = [f for f in fs if f.done]
fs_notdone = [f for f in fs if not f.done]
else:
fs_done = [f for f in fs if f.success or f.done]
fs_notdone = [f for f in fs if not f.success and not f.done]
return fs_done, fs_notdone
def wait(fs,
throw_except=True,
return_when=ALL_COMPLETED,
download_results=False,
timeout=None,
THREADPOOL_SIZE=128,
WAIT_DUR_SEC=1,
internal_storage=None):
"""
Wait for the Future instances (possibly created by different Executor instances)
given by fs to complete. Returns a named 2-tuple of sets. The first set, named done,
contains the futures that completed (finished or cancelled futures) before the wait
completed. The second set, named not_done, contains the futures that did not complete
(pending or running futures). timeout can be used to control the maximum number of
seconds to wait before returning.
:param fs: Futures list. Default None
:param throw_except: Re-raise exception if call raised. Default True.
:param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS`
:param download_results: Download results. Default false (Only get statuses)
:param timeout: Timeout of waiting for results.
:param THREADPOOL_SIZE: Number of threads to use. Default 64
:param WAIT_DUR_SEC: Time interval between each check.
:return: `(fs_done, fs_notdone)`
where `fs_done` is a list of futures that have completed
and `fs_notdone` is a list of futures that have not completed.
:rtype: 2-tuple of list
"""
if not internal_storage:
internal_storage = InternalStorage(fs[0].storage_config)
if type(fs) != list:
fs = [fs]
setup_progressbar = (not is_lithops_worker()
and logger.getEffectiveLevel() == logging.INFO)
if download_results:
msg = 'Getting results from functions'
fs_done = [f for f in fs if f.done]
fs_not_done = [f for f in fs if not f.done]
# fs_not_ready = [f for f in futures if not f.ready and not f.done]
else:
msg = 'Waiting for functions to complete'
fs_done = [f for f in fs if f.ready or f.done]
fs_not_done = [f for f in fs if not f.done]
# fs_not_ready = [f for f in futures if not f.ready and not f.done]
if not fs_not_done:
return fs_done, fs_not_done
logger.info(msg)
if is_unix_system() and timeout is not None:
logger.debug('Setting waiting timeout to {} seconds'.format(timeout))
error_msg = 'Timeout of {} seconds exceeded waiting for function activations to finish'.format(
timeout)
signal.signal(signal.SIGALRM, partial(timeout_handler, error_msg))
signal.alarm(timeout)
pbar = None
if not is_lithops_worker() and setup_progressbar:
from tqdm.auto import tqdm
if not is_notebook():
print()
pbar = tqdm(bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ',
total=len(fs_not_done),
disable=None)
try:
wait_storage(fs,
internal_storage,
download_results=download_results,
throw_except=throw_except,
return_when=return_when,
pbar=pbar,
THREADPOOL_SIZE=THREADPOOL_SIZE,
WAIT_DUR_SEC=WAIT_DUR_SEC)
except KeyboardInterrupt as e:
if download_results:
not_dones_call_ids = [(f.job_id, f.call_id) for f in fs
if not f.done]
else:
not_dones_call_ids = [(f.job_id, f.call_id) for f in fs
if not f.ready and not f.done]
msg = ('Cancelled - Total Activations not done: {}'.format(
len(not_dones_call_ids)))
if pbar:
pbar.close()
print()
logger.info(msg)
raise e
except Exception as e:
raise e
finally:
if is_unix_system():
signal.alarm(0)
if pbar and not pbar.disable:
pbar.close()
if not is_notebook():
print()
if download_results:
fs_done = [f for f in fs if f.done]
fs_notdone = [f for f in fs if not f.done]
else:
fs_done = [f for f in fs if f.ready or f.done]
fs_notdone = [f for f in fs if not f.ready and not f.done]
return fs_done, fs_notdone