def reduce_function_wrapper(fut_list, internal_storage, storage, ibm_cos): logger.info('Waiting for results') if 'SHOW_MEMORY_USAGE' in os.environ: show_memory = eval(os.environ['SHOW_MEMORY_USAGE']) else: show_memory = False # Wait for all results wait(fut_list, executor_id, internal_storage, download_results=True) results = [ f.result() for f in fut_list if f.done and not f.futures ] reduce_func_args = {'results': results} if show_memory: logger.debug( "Memory usage after getting the results: {}".format( wrenutil.get_current_memory_usage())) # Run reduce function func_sig = inspect.signature(reduce_function) if 'storage' in func_sig.parameters: reduce_func_args['storage'] = storage if 'ibm_cos' in func_sig.parameters: reduce_func_args['ibm_cos'] = ibm_cos return reduce_function(**reduce_func_args)
def reduce_func(fut_list, storage_handler): logger.info('Starting reduce_func() function') logger.info('Waiting for results') # Wait for all results wait(fut_list, executor_id, storage_handler, throw_except) accum_list = [] # Get all results for f in fut_list: accum_list.append( f.result(throw_except=throw_except, storage_handler=storage_handler)) # Run reduce function func_sig = inspect.signature(function) if 'futures' in func_sig.parameters and 'storage_handler' in func_sig.parameters: return function(accum_list, futures=fut_list, storage_handler=storage_handler) if 'storage_handler' in func_sig.parameters: return function(accum_list, storage_handler=storage_handler) if 'futures' in func_sig.parameters: return function(accum_list, futures=fut_list) return function(accum_list)
def reduce(self, reduce_function, list_of_futures, throw_except=True, wait_local=True, extra_env=None, extra_meta=None): """ Apply a function across all futures. """ executor_id = self.executor_id if wait_local: logger.info('Waiting locally for results') wait(list_of_futures, executor_id, self.internal_storage, throw_except=throw_except) def reduce_function_wrapper(fut_list, internal_storage, storage, ibm_cos): logger.info('Waiting for results') if 'SHOW_MEMORY_USAGE' in os.environ: show_memory = eval(os.environ['SHOW_MEMORY_USAGE']) else: show_memory = False # Wait for all results wait(fut_list, executor_id, internal_storage, throw_except=throw_except) results = [ f.result() for f in fut_list if f.done and not f.futures ] reduce_func_args = {'results': results} if show_memory: logger.debug( "Memory usage after getting the results: {}".format( wrenutil.get_current_memory_usage())) # Run reduce function func_sig = inspect.signature(reduce_function) if 'futures' in func_sig.parameters: reduce_func_args['futures'] = fut_list if 'storage' in func_sig.parameters: reduce_func_args['storage'] = storage if 'ibm_cos' in func_sig.parameters: reduce_func_args['ibm_cos'] = ibm_cos return reduce_function(**reduce_func_args) return self.map(reduce_function_wrapper, [[ list_of_futures, ]], extra_env=extra_env, extra_meta=extra_meta, original_func_name=reduce_function.__name__)
def wait(self, throw_except=True, verbose=True, return_when=ALL_COMPLETED, THREADPOOL_SIZE=64, WAIT_DUR_SEC=4): """ Wait for the Future instances `fs` to complete. Returns a 2-tuple of lists. The first list contains the futures that completed (finished or cancelled) before the wait completed. The second contains uncompleted futures. :param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS` :param THREADPOOL_SIZE: Number of threads to use. Default 64 :param WAIT_DUR_SEC: Time interval between each check. :return: `(fs_dones, fs_notdones)` where `fs_dones` is a list of futures that have completed and `fs_notdones` is a list of futures that have not completed. :rtype: 2-tuple of lists Usage >>> import pywren >>> pw = pywren.ibm_cf_executor() >>> pw.map(foo, data_list) >>> dones, not_dones = pw.wait() >>> # not_dones should be an empty list. >>> results = [f.result() for f in dones] """ if not self._state == ExecutorState.map or not self._state == ExecutorState.map_reduce: raise Exception('You must run pw.map() or pw.map_reduce() before call pw.wait()') return wait(self.futures, self.executor_id, self.storage_handler, throw_except, verbose, return_when, THREADPOOL_SIZE, WAIT_DUR_SEC)
def reduce(self, function, list_of_futures, throw_except=True, wait_local=True, extra_env=None, extra_meta=None): """ Apply a function across all futures. """ executor_id = self.executor_id if wait_local: wait(list_of_futures, executor_id, self.storage_handler, throw_except) def reduce_func(fut_list, storage_handler): logger.info('Starting reduce_func() function') logger.info('Waiting for results') # Wait for all results wait(fut_list, executor_id, storage_handler, throw_except) accum_list = [] # Get all results for f in fut_list: accum_list.append( f.result(throw_except=throw_except, storage_handler=storage_handler)) # Run reduce function func_sig = inspect.signature(function) if 'futures' in func_sig.parameters and 'storage_handler' in func_sig.parameters: return function(accum_list, futures=fut_list, storage_handler=storage_handler) if 'storage_handler' in func_sig.parameters: return function(accum_list, storage_handler=storage_handler) if 'futures' in func_sig.parameters: return function(accum_list, futures=fut_list) return function(accum_list) return self.call_async(reduce_func, [ list_of_futures, ], extra_env=extra_env, extra_meta=extra_meta)[0]
def monitor(self, futures=None, throw_except=True, return_when=ALL_COMPLETED, download_results=False, timeout=EXECUTION_TIMEOUT, THREADPOOL_SIZE=128, WAIT_DUR_SEC=1): """ Wait for the Future instances `fs` to complete. Returns a 2-tuple of lists. The first list contains the futures that completed (finished or cancelled) before the wait completed. The second contains uncompleted futures. :param futures: Futures list. Default None :param throw_except: Re-raise exception if call raised. Default True. :param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS` :param download_results: Download results. Default false (Only download statuses) :param timeout: Timeout of waiting for results. :param THREADPOOL_SIZE: Number of threads to use. Default 64 :param WAIT_DUR_SEC: Time interval between each check. :return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` is a list of futures that have not completed. :rtype: 2-tuple of list """ if not futures: futures = [] for job in self.jobs: if self.jobs[job]['state'] == JobState.running: futures.extend(self.jobs[job]['futures']) self.jobs[job]['state'] = JobState.ready if type(futures) != list: ftrs = [futures] else: ftrs = futures if not ftrs: raise Exception('You must run call_async(), map() or map_reduce()' ' before calling get_result() method') rabbit_amqp_url = None if self.rabbitmq_monitor: rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') if rabbit_amqp_url and not download_results: logger.info('Going to use RabbitMQ to monitor function activations') logging.getLogger('pika').setLevel(logging.WARNING) if download_results: msg = 'ExecutorID {} - Getting results...'.format(self.executor_id) else: msg = 'ExecutorID {} - Waiting for functions to complete...'.format(self.executor_id) logger.info(msg) if not self.log_level and self._state == ExecutorState.running: print(msg) if is_unix_system(): signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(timeout) pbar = None if not self.is_cf_cluster and self._state == ExecutorState.running \ and not self.log_level: from tqdm.auto import tqdm if is_notebook(): pbar = tqdm(bar_format='{n}/|/ {n_fmt}/{total_fmt}', total=len(ftrs)) # ncols=800 else: print() pbar = tqdm(bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ', total=len(ftrs), disable=False) try: wait(ftrs, self.executor_id, self.internal_storage, download_results=download_results, throw_except=throw_except, return_when=return_when, rabbit_amqp_url=rabbit_amqp_url, pbar=pbar, THREADPOOL_SIZE=THREADPOOL_SIZE, WAIT_DUR_SEC=WAIT_DUR_SEC) except FunctionException as e: if is_unix_system(): signal.alarm(0) if pbar: pbar.close() logger.info(e.msg) if not is_notebook(): print() if not self.log_level: print(e.msg) if e.exc_msg: print('--> Exception: ' + e.exc_msg) else: print() traceback.print_exception(*e.exception) sys.exit() except TimeoutError: if download_results: not_dones_activation_ids = [f.activation_id for f in ftrs if not f.done and not (f.ready and not f.produce_output)] else: not_dones_activation_ids = [f.activation_id for f in ftrs if not f.ready] msg = ('ExecutorID {} - Raised timeout of {} seconds waiting for results ' '\nActivations not done: {}'.format(self.executor_id, timeout, not_dones_activation_ids)) self._state = ExecutorState.error except KeyboardInterrupt: if download_results: not_dones_activation_ids = [f.activation_id for f in ftrs if not f.done and not (f.ready and not f.produce_output)] else: not_dones_activation_ids = [f.activation_id for f in ftrs if not f.ready] msg = 'ExecutorID {} - Cancelled \nActivations not done: {}'.format(self.executor_id, not_dones_activation_ids) self._state = ExecutorState.error finally: if is_unix_system(): signal.alarm(0) if pbar: pbar.close() if not is_notebook(): print() if self._state == ExecutorState.error: logger.info(msg) if not self.log_level: print(msg) if download_results and self.data_cleaner and not self.is_cf_cluster: self.clean() if download_results: fs_dones = [f for f in ftrs if f.done] fs_notdones = [f for f in ftrs if not f.done] else: fs_dones = [f for f in ftrs if f.ready] fs_notdones = [f for f in ftrs if not f.ready] self._state = ExecutorState.ready return fs_dones, fs_notdones
def monitor(self, futures=None, throw_except=True, return_when=ALL_COMPLETED, download_results=False, timeout=wrenconfig.RUNTIME_TIMEOUT, THREADPOOL_SIZE=128, WAIT_DUR_SEC=1): """ Wait for the Future instances `fs` to complete. Returns a 2-tuple of lists. The first list contains the futures that completed (finished or cancelled) before the wait completed. The second contains uncompleted futures. :param futures: Futures list. Default None :param throw_except: Re-raise exception if call raised. Default True. :param return_when: One of `ALL_COMPLETED`, `ANY_COMPLETED`, `ALWAYS` :param download_results: Download results. Default false (Only download statuses) :param timeout: Timeout of waiting for results. :param THREADPOOL_SIZE: Number of threads to use. Default 64 :param WAIT_DUR_SEC: Time interval between each check. :return: `(fs_done, fs_notdone)` where `fs_done` is a list of futures that have completed and `fs_notdone` is a list of futures that have not completed. :rtype: 2-tuple of lists Usage >>> import pywren_ibm_cloud as pywren >>> pw = pywren.ibm_cf_executor() >>> pw.map(foo, data_list) >>> dones, not_dones = pw.monitor() >>> # not_dones should be an empty list. >>> results = [f.result() for f in dones] """ if futures: # Ensure futures is a list if type(futures) != list: ftrs = [futures] else: ftrs = futures else: # In this case self.futures is always a list ftrs = self.futures if not ftrs: raise Exception('You must run pw.call_async(), pw.map()' ' or pw.map_reduce() before call pw.get_result()') rabbit_amqp_url = None if self._state == ExecutorState.running: if self.rabbitmq_monitor: rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') if rabbit_amqp_url and not download_results: logger.info( 'Going to use RabbitMQ to monitor function activations') if download_results: msg = 'Executor ID {} Getting results...'.format( self.executor_id) else: msg = 'Executor ID {} Waiting for functions to complete...'.format( self.executor_id) logger.info(msg) if not self.log_level and self._state == ExecutorState.running: print(msg) if is_unix_system(): signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(timeout) pbar = None if not self.is_cf_cluster and self._state == ExecutorState.running \ and not self.log_level and not is_notebook(): import tqdm print() pbar = tqdm.tqdm( bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ', total=len(ftrs), disable=False) try: wait(ftrs, self.executor_id, self.internal_storage, download_results=download_results, throw_except=throw_except, return_when=return_when, rabbit_amqp_url=rabbit_amqp_url, pbar=pbar, THREADPOOL_SIZE=THREADPOOL_SIZE, WAIT_DUR_SEC=WAIT_DUR_SEC) except TimeoutError: if download_results: not_dones_activation_ids = [ f.activation_id for f in ftrs if not f.done ] else: not_dones_activation_ids = [ f.activation_id for f in ftrs if not f.ready ] msg = ( 'Executor ID {} Raised timeout of {} seconds waiting for results ' '\nActivations not done: {}'.format(self.executor_id, timeout, not_dones_activation_ids)) self._state = ExecutorState.error except KeyboardInterrupt: if download_results: not_dones_activation_ids = [ f.activation_id for f in ftrs if not f.done ] else: not_dones_activation_ids = [ f.activation_id for f in ftrs if not f.ready ] msg = 'Executor ID {} Cancelled \nActivations not done: {}'.format( self.executor_id, not_dones_activation_ids) self._state = ExecutorState.error finally: if is_unix_system(): signal.alarm(0) if pbar: pbar.close() print() if self._state == ExecutorState.error: logger.info(msg) if not self.log_level: print(msg) if self.data_cleaner and not self.is_cf_cluster and self._state != ExecutorState.ready: self.clean() if download_results: fs_dones = [f for f in ftrs if f.done] fs_notdones = [f for f in ftrs if not f.done] else: fs_dones = [f for f in ftrs if f.ready] fs_notdones = [f for f in ftrs if not f.ready] self._state = ExecutorState.ready return fs_dones, fs_notdones
def get_result(self, futures=None, throw_except=True, timeout=wrenconfig.CF_RUNTIME_TIMEOUT, THREADPOOL_SIZE=64, WAIT_DUR_SEC=2): """ For getting PyWren results :param futures: Futures list. Default None :param throw_except: Reraise exception if call raised. Default True. :param verbose: Shows some information prints. Default False :param timeout: Timeout for waiting results. :param THREADPOOL_SIZE: Number of threads to use. Default 64 :return: The result of the future/s Usage >>> import pywren_ibm_cloud as pywren >>> pw = pywren.ibm_cf_executor() >>> pw.map(foo, data) >>> result = pw.get_result() """ if futures: # Ensure futures is a list if type(futures) != list: ftrs = [futures] else: ftrs = futures else: # In this case self.futures is always a list ftrs = self.futures if not ftrs: raise Exception('You must run pw.call_async(), pw.map()' ' or pw.map_reduce() before call pw.get_result()') msg = 'Executor ID {} Getting results'.format(self.executor_id) logger.debug(msg) if (logger.getEffectiveLevel() == logging.WARNING): print(msg) signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(timeout) if self.cf_cluster or logger.getEffectiveLevel() != logging.WARNING: pbar = None else: import tqdm print() pbar = tqdm.tqdm( bar_format=' {l_bar}{bar}| {n_fmt}/{total_fmt} ', total=len(ftrs), disable=False) try: wait(ftrs, self.executor_id, self.internal_storage, throw_except=throw_except, THREADPOOL_SIZE=THREADPOOL_SIZE, WAIT_DUR_SEC=WAIT_DUR_SEC, pbar=pbar) result = [f.result() for f in ftrs if f.done and not f.futures] except TimeoutError: if pbar: pbar.close() print() not_dones_activation_ids = set( [f.activation_id for f in ftrs if not f.done]) msg = ( 'Executor ID {} Raised timeout of {} seconds getting results ' '\nActivations not done: {}'.format(self.executor_id, timeout, not_dones_activation_ids)) logger.debug(msg) if (logger.getEffectiveLevel() == logging.WARNING): print(msg) self._state = ExecutorState.error result = None except KeyboardInterrupt: if pbar: pbar.close() print() not_dones_activation_ids = [ f.activation_id for f in ftrs if not f.done ] msg = 'Executor ID {} Cancelled \nActivations not done: {}'.format( self.executor_id, not_dones_activation_ids) logger.debug(msg) if (logger.getEffectiveLevel() == logging.WARNING): print(msg) if self.data_cleaner and not self.cf_cluster: self.clean() exit() finally: signal.alarm(0) if pbar: pbar.close() print() if self.data_cleaner and not self.cf_cluster: self.clean() msg = "Executor ID {} Finished\n".format(self.executor_id) logger.debug(msg) if (logger.getEffectiveLevel() == logging.WARNING and self.data_cleaner): print(msg) if result and len(result) == 1: return result[0] return result