def reduce_function_wrapper(fut_list, internal_storage, storage,
ibm_cos):
logger.info('Waiting for results')
if 'SHOW_MEMORY_USAGE' in os.environ:
show_memory = eval(os.environ['SHOW_MEMORY_USAGE'])
else:
show_memory = False
# Wait for all results
wait(fut_list,
executor_id,
internal_storage,
download_results=True)
results = [
f.result() for f in fut_list if f.done and not f.futures
]
reduce_func_args = {'results': results}
if show_memory:
logger.debug(
"Memory usage after getting the results: {}".format(
wrenutil.get_current_memory_usage()))
# Run reduce function
func_sig = inspect.signature(reduce_function)
if 'storage' in func_sig.parameters:
reduce_func_args['storage'] = storage
if 'ibm_cos' in func_sig.parameters:
reduce_func_args['ibm_cos'] = ibm_cos
return reduce_function(**reduce_func_args)
def reduce_func(fut_list, storage_handler):
logger.info('Starting reduce_func() function')
logger.info('Waiting for results')
# Wait for all results
wait(fut_list, executor_id, storage_handler, throw_except)
accum_list = []
# Get all results
for f in fut_list:
accum_list.append(
f.result(throw_except=throw_except,
storage_handler=storage_handler))
# Run reduce function
func_sig = inspect.signature(function)
if 'futures' in func_sig.parameters and 'storage_handler' in func_sig.parameters:
return function(accum_list,
futures=fut_list,
storage_handler=storage_handler)
if 'storage_handler' in func_sig.parameters:
return function(accum_list, storage_handler=storage_handler)
if 'futures' in func_sig.parameters:
return function(accum_list, futures=fut_list)
return function(accum_list)
def reduce(self,
reduce_function,
list_of_futures,
throw_except=True,
wait_local=True,
extra_env=None,
extra_meta=None):
"""
Apply a function across all futures.
"""
executor_id = self.executor_id
if wait_local:
logger.info('Waiting locally for results')
wait(list_of_futures,
executor_id,
self.internal_storage,
throw_except=throw_except)
def reduce_function_wrapper(fut_list, internal_storage, storage,
ibm_cos):
logger.info('Waiting for results')
if 'SHOW_MEMORY_USAGE' in os.environ:
show_memory = eval(os.environ['SHOW_MEMORY_USAGE'])
else:
show_memory = False
# Wait for all results
wait(fut_list,
executor_id,
internal_storage,
throw_except=throw_except)
results = [
f.result() for f in fut_list if f.done and not f.futures
]
reduce_func_args = {'results': results}
if show_memory:
logger.debug(
"Memory usage after getting the results: {}".format(
wrenutil.get_current_memory_usage()))
# Run reduce function
func_sig = inspect.signature(reduce_function)
if 'futures' in func_sig.parameters:
reduce_func_args['futures'] = fut_list
if 'storage' in func_sig.parameters:
reduce_func_args['storage'] = storage
if 'ibm_cos' in func_sig.parameters:
reduce_func_args['ibm_cos'] = ibm_cos
return reduce_function(**reduce_func_args)
return self.map(reduce_function_wrapper, [[
list_of_futures,
]],
extra_env=extra_env,
extra_meta=extra_meta,
original_func_name=reduce_function.__name__)
def wait(self, throw_except=True, verbose=True, return_when=ALL_COMPLETED,
THREADPOOL_SIZE=64, WAIT_DUR_SEC=4):
"""
Wait for the Future instances `fs` to complete. Returns a 2-tuple of
lists. The first list contains the futures that completed
(finished or cancelled) before the wait completed. The second
contains uncompleted futures.
:param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS`
:param THREADPOOL_SIZE: Number of threads to use. Default 64
:param WAIT_DUR_SEC: Time interval between each check.
:return: `(fs_dones, fs_notdones)`
where `fs_dones` is a list of futures that have completed
and `fs_notdones` is a list of futures that have not completed.
:rtype: 2-tuple of lists
Usage
>>> import pywren
>>> pw = pywren.ibm_cf_executor()
>>> pw.map(foo, data_list)
>>> dones, not_dones = pw.wait()
>>> # not_dones should be an empty list.
>>> results = [f.result() for f in dones]
"""
if not self._state == ExecutorState.map or not self._state == ExecutorState.map_reduce:
raise Exception('You must run pw.map() or pw.map_reduce() before call pw.wait()')
return wait(self.futures, self.executor_id, self.storage_handler,
throw_except, verbose, return_when, THREADPOOL_SIZE, WAIT_DUR_SEC)
def reduce(self,
function,
list_of_futures,
throw_except=True,
wait_local=True,
extra_env=None,
extra_meta=None):
"""
Apply a function across all futures.
"""
executor_id = self.executor_id
if wait_local:
wait(list_of_futures, executor_id, self.storage_handler,
throw_except)
def reduce_func(fut_list, storage_handler):
logger.info('Starting reduce_func() function')
logger.info('Waiting for results')
# Wait for all results
wait(fut_list, executor_id, storage_handler, throw_except)
accum_list = []
# Get all results
for f in fut_list:
accum_list.append(
f.result(throw_except=throw_except,
storage_handler=storage_handler))
# Run reduce function
func_sig = inspect.signature(function)
if 'futures' in func_sig.parameters and 'storage_handler' in func_sig.parameters:
return function(accum_list,
futures=fut_list,
storage_handler=storage_handler)
if 'storage_handler' in func_sig.parameters:
return function(accum_list, storage_handler=storage_handler)
if 'futures' in func_sig.parameters:
return function(accum_list, futures=fut_list)
return function(accum_list)
return self.call_async(reduce_func, [
list_of_futures,
],
extra_env=extra_env,
extra_meta=extra_meta)[0]
def monitor(self, futures=None, throw_except=True, return_when=ALL_COMPLETED,
download_results=False, timeout=EXECUTION_TIMEOUT,
THREADPOOL_SIZE=128, WAIT_DUR_SEC=1):
"""
Wait for the Future instances `fs` to complete. Returns a 2-tuple of
lists. The first list contains the futures that completed
(finished or cancelled) before the wait completed. The second
contains uncompleted futures.
:param futures: Futures list. Default None
:param throw_except: Re-raise exception if call raised. Default True.
:param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS`
:param download_results: Download results. Default false (Only download statuses)
:param timeout: Timeout of waiting for results.
:param THREADPOOL_SIZE: Number of threads to use. Default 64
:param WAIT_DUR_SEC: Time interval between each check.
:return: `(fs_done, fs_notdone)`
where `fs_done` is a list of futures that have completed
and `fs_notdone` is a list of futures that have not completed.
:rtype: 2-tuple of list
"""
if not futures:
futures = []
for job in self.jobs:
if self.jobs[job]['state'] == JobState.running:
futures.extend(self.jobs[job]['futures'])
self.jobs[job]['state'] = JobState.ready
if type(futures) != list:
ftrs = [futures]
else:
ftrs = futures
if not ftrs:
raise Exception('You must run call_async(), map() or map_reduce()'
' before calling get_result() method')
rabbit_amqp_url = None
if self.rabbitmq_monitor:
rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
if rabbit_amqp_url and not download_results:
logger.info('Going to use RabbitMQ to monitor function activations')
logging.getLogger('pika').setLevel(logging.WARNING)
if download_results:
msg = 'ExecutorID {} - Getting results...'.format(self.executor_id)
else:
msg = 'ExecutorID {} - Waiting for functions to complete...'.format(self.executor_id)
logger.info(msg)
if not self.log_level and self._state == ExecutorState.running:
print(msg)
if is_unix_system():
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(timeout)
pbar = None
if not self.is_cf_cluster and self._state == ExecutorState.running \
and not self.log_level:
from tqdm.auto import tqdm
if is_notebook():
pbar = tqdm(bar_format='{n}/|/ {n_fmt}/{total_fmt}', total=len(ftrs)) # ncols=800
else:
print()
pbar = tqdm(bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ', total=len(ftrs), disable=False)
try:
wait(ftrs, self.executor_id, self.internal_storage, download_results=download_results,
throw_except=throw_except, return_when=return_when, rabbit_amqp_url=rabbit_amqp_url,
pbar=pbar, THREADPOOL_SIZE=THREADPOOL_SIZE, WAIT_DUR_SEC=WAIT_DUR_SEC)
except FunctionException as e:
if is_unix_system():
signal.alarm(0)
if pbar:
pbar.close()
logger.info(e.msg)
if not is_notebook():
print()
if not self.log_level:
print(e.msg)
if e.exc_msg:
print('--> Exception: ' + e.exc_msg)
else:
print()
traceback.print_exception(*e.exception)
sys.exit()
except TimeoutError:
if download_results:
not_dones_activation_ids = [f.activation_id for f in ftrs if not f.done and not (f.ready and not f.produce_output)]
else:
not_dones_activation_ids = [f.activation_id for f in ftrs if not f.ready]
msg = ('ExecutorID {} - Raised timeout of {} seconds waiting for results '
'\nActivations not done: {}'.format(self.executor_id, timeout, not_dones_activation_ids))
self._state = ExecutorState.error
except KeyboardInterrupt:
if download_results:
not_dones_activation_ids = [f.activation_id for f in ftrs if not f.done and not (f.ready and not f.produce_output)]
else:
not_dones_activation_ids = [f.activation_id for f in ftrs if not f.ready]
msg = 'ExecutorID {} - Cancelled \nActivations not done: {}'.format(self.executor_id, not_dones_activation_ids)
self._state = ExecutorState.error
finally:
if is_unix_system():
signal.alarm(0)
if pbar:
pbar.close()
if not is_notebook():
print()
if self._state == ExecutorState.error:
logger.info(msg)
if not self.log_level:
print(msg)
if download_results and self.data_cleaner and not self.is_cf_cluster:
self.clean()
if download_results:
fs_dones = [f for f in ftrs if f.done]
fs_notdones = [f for f in ftrs if not f.done]
else:
fs_dones = [f for f in ftrs if f.ready]
fs_notdones = [f for f in ftrs if not f.ready]
self._state = ExecutorState.ready
return fs_dones, fs_notdones
def monitor(self,
futures=None,
throw_except=True,
return_when=ALL_COMPLETED,
download_results=False,
timeout=wrenconfig.RUNTIME_TIMEOUT,
THREADPOOL_SIZE=128,
WAIT_DUR_SEC=1):
"""
Wait for the Future instances `fs` to complete. Returns a 2-tuple of
lists. The first list contains the futures that completed
(finished or cancelled) before the wait completed. The second
contains uncompleted futures.
:param futures: Futures list. Default None
:param throw_except: Re-raise exception if call raised. Default True.
:param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS`
:param download_results: Download results. Default false (Only download statuses)
:param timeout: Timeout of waiting for results.
:param THREADPOOL_SIZE: Number of threads to use. Default 64
:param WAIT_DUR_SEC: Time interval between each check.
:return: `(fs_done, fs_notdone)`
where `fs_done` is a list of futures that have completed
and `fs_notdone` is a list of futures that have not completed.
:rtype: 2-tuple of lists
Usage
>>> import pywren_ibm_cloud as pywren
>>> pw = pywren.ibm_cf_executor()
>>> pw.map(foo, data_list)
>>> dones, not_dones = pw.monitor()
>>> # not_dones should be an empty list.
>>> results = [f.result() for f in dones]
"""
if futures:
# Ensure futures is a list
if type(futures) != list:
ftrs = [futures]
else:
ftrs = futures
else:
# In this case self.futures is always a list
ftrs = self.futures
if not ftrs:
raise Exception('You must run pw.call_async(), pw.map()'
' or pw.map_reduce() before call pw.get_result()')
rabbit_amqp_url = None
if self._state == ExecutorState.running:
if self.rabbitmq_monitor:
rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
if rabbit_amqp_url and not download_results:
logger.info(
'Going to use RabbitMQ to monitor function activations')
if download_results:
msg = 'Executor ID {} Getting results...'.format(
self.executor_id)
else:
msg = 'Executor ID {} Waiting for functions to complete...'.format(
self.executor_id)
logger.info(msg)
if not self.log_level and self._state == ExecutorState.running:
print(msg)
if is_unix_system():
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(timeout)
pbar = None
if not self.is_cf_cluster and self._state == ExecutorState.running \
and not self.log_level and not is_notebook():
import tqdm
print()
pbar = tqdm.tqdm(
bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ',
total=len(ftrs),
disable=False)
try:
wait(ftrs,
self.executor_id,
self.internal_storage,
download_results=download_results,
throw_except=throw_except,
return_when=return_when,
rabbit_amqp_url=rabbit_amqp_url,
pbar=pbar,
THREADPOOL_SIZE=THREADPOOL_SIZE,
WAIT_DUR_SEC=WAIT_DUR_SEC)
except TimeoutError:
if download_results:
not_dones_activation_ids = [
f.activation_id for f in ftrs if not f.done
]
else:
not_dones_activation_ids = [
f.activation_id for f in ftrs if not f.ready
]
msg = (
'Executor ID {} Raised timeout of {} seconds waiting for results '
'\nActivations not done: {}'.format(self.executor_id, timeout,
not_dones_activation_ids))
self._state = ExecutorState.error
except KeyboardInterrupt:
if download_results:
not_dones_activation_ids = [
f.activation_id for f in ftrs if not f.done
]
else:
not_dones_activation_ids = [
f.activation_id for f in ftrs if not f.ready
]
msg = 'Executor ID {} Cancelled \nActivations not done: {}'.format(
self.executor_id, not_dones_activation_ids)
self._state = ExecutorState.error
finally:
if is_unix_system():
signal.alarm(0)
if pbar:
pbar.close()
print()
if self._state == ExecutorState.error:
logger.info(msg)
if not self.log_level:
print(msg)
if self.data_cleaner and not self.is_cf_cluster and self._state != ExecutorState.ready:
self.clean()
if download_results:
fs_dones = [f for f in ftrs if f.done]
fs_notdones = [f for f in ftrs if not f.done]
else:
fs_dones = [f for f in ftrs if f.ready]
fs_notdones = [f for f in ftrs if not f.ready]
self._state = ExecutorState.ready
return fs_dones, fs_notdones
def get_result(self,
futures=None,
throw_except=True,
timeout=wrenconfig.CF_RUNTIME_TIMEOUT,
THREADPOOL_SIZE=64,
WAIT_DUR_SEC=2):
"""
For getting PyWren results
:param futures: Futures list. Default None
:param throw_except: Reraise exception if call raised. Default True.
:param verbose: Shows some information prints. Default False
:param timeout: Timeout for waiting results.
:param THREADPOOL_SIZE: Number of threads to use. Default 64
:return: The result of the future/s
Usage
>>> import pywren_ibm_cloud as pywren
>>> pw = pywren.ibm_cf_executor()
>>> pw.map(foo, data)
>>> result = pw.get_result()
"""
if futures:
# Ensure futures is a list
if type(futures) != list:
ftrs = [futures]
else:
ftrs = futures
else:
# In this case self.futures is always a list
ftrs = self.futures
if not ftrs:
raise Exception('You must run pw.call_async(), pw.map()'
' or pw.map_reduce() before call pw.get_result()')
msg = 'Executor ID {} Getting results'.format(self.executor_id)
logger.debug(msg)
if (logger.getEffectiveLevel() == logging.WARNING):
print(msg)
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(timeout)
if self.cf_cluster or logger.getEffectiveLevel() != logging.WARNING:
pbar = None
else:
import tqdm
print()
pbar = tqdm.tqdm(
bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ',
total=len(ftrs),
disable=False)
try:
wait(ftrs,
self.executor_id,
self.internal_storage,
throw_except=throw_except,
THREADPOOL_SIZE=THREADPOOL_SIZE,
WAIT_DUR_SEC=WAIT_DUR_SEC,
pbar=pbar)
result = [f.result() for f in ftrs if f.done and not f.futures]
except TimeoutError:
if pbar:
pbar.close()
print()
not_dones_activation_ids = set(
[f.activation_id for f in ftrs if not f.done])
msg = (
'Executor ID {} Raised timeout of {} seconds getting results '
'\nActivations not done: {}'.format(self.executor_id, timeout,
not_dones_activation_ids))
logger.debug(msg)
if (logger.getEffectiveLevel() == logging.WARNING):
print(msg)
self._state = ExecutorState.error
result = None
except KeyboardInterrupt:
if pbar:
pbar.close()
print()
not_dones_activation_ids = [
f.activation_id for f in ftrs if not f.done
]
msg = 'Executor ID {} Cancelled \nActivations not done: {}'.format(
self.executor_id, not_dones_activation_ids)
logger.debug(msg)
if (logger.getEffectiveLevel() == logging.WARNING):
print(msg)
if self.data_cleaner and not self.cf_cluster:
self.clean()
exit()
finally:
signal.alarm(0)
if pbar:
pbar.close()
print()
if self.data_cleaner and not self.cf_cluster:
self.clean()
msg = "Executor ID {} Finished\n".format(self.executor_id)
logger.debug(msg)
if (logger.getEffectiveLevel() == logging.WARNING
and self.data_cleaner):
print(msg)
if result and len(result) == 1:
return result[0]
return result
