def invoke(executor_id, job_id, call_id, func_key, invoke_metadata, data_key, data_byte_range): output_key = create_output_key(self.storage_config['prefix'], executor_id, job_id, call_id) status_key = create_status_key(self.storage_config['prefix'], executor_id, job_id, call_id) payload = { 'config': self.config, 'log_level': self.log_level, 'func_key': func_key, 'data_key': data_key, 'output_key': output_key, 'status_key': status_key, 'task_execution_timeout': job.task_execution_timeout, 'data_byte_range': data_byte_range, 'executor_id': executor_id, 'job_id': job_id, 'call_id': call_id, 'pywren_version': __version__ } if job.extra_env is not None: logger.debug("Extra environment vars {}".format(job.extra_env)) payload['extra_env'] = job.extra_env if job.extra_meta is not None: # sanity for k, v in job.extra_meta.items(): if k in payload: raise ValueError("Key {} already in dict".format(k)) payload[k] = v # overwrite explicit args, mostly used for testing via injection if job.overwrite_invoke_args is not None: payload.update(job.overwrite_invoke_args) host_submit_time = time.time() payload['host_submit_time'] = host_submit_time # do the invocation activation_id = self.internal_compute.invoke( job.runtime_name, job.runtime_memory, payload) if not activation_id: raise Exception( "ExecutorID {} - Activation {} failed, therefore job is failed" .format(executor_id, call_id)) invoke_metadata['activation_id'] = activation_id invoke_metadata['invoke_time'] = time.time() - host_submit_time invoke_metadata.update(payload) del invoke_metadata['config'] fut = ResponseFuture(call_id, job_id, executor_id, activation_id, self.storage_config, invoke_metadata) fut._set_state(JobState.invoked) return fut
def invoke(executor_id, job_id, call_id, func_key, invoke_metadata, data_key, data_byte_range): output_key = create_output_key(self.storage_config['prefix'], executor_id, job_id, call_id) status_key = create_status_key(self.storage_config['prefix'], executor_id, job_id, call_id) payload = { 'config': self.config, 'log_level': self.log_level, 'func_key': func_key, 'data_key': data_key, 'output_key': output_key, 'status_key': status_key, 'execution_timeout': job.execution_timeout, 'data_byte_range': data_byte_range, 'executor_id': executor_id, 'job_id': job_id, 'call_id': call_id, 'pywren_version': __version__ } if job.extra_env is not None: logger.debug("Extra environment vars {}".format(job.extra_env)) payload['extra_env'] = job.extra_env host_submit_time = time.time() payload['host_submit_time'] = host_submit_time # do the invocation activation_id = self.compute.invoke(job.runtime_name, job.runtime_memory, payload) if not activation_id: raise Exception( "ExecutorID {} | JobID {} - Retrying mechanism finished with no success. " "Failed to invoke the job".format(executor_id, job_id)) invoke_metadata['activation_id'] = activation_id invoke_metadata['invoke_time'] = time.time() - host_submit_time invoke_metadata.update(payload) del invoke_metadata['config'] fut = ResponseFuture(call_id, job_id, executor_id, activation_id, self.storage_config, invoke_metadata) fut._set_state(CallState.invoked) return fut
def invoke(executor_id, job_id, call_id, func_key, job_metadata, data_key, data_byte_range): output_key = create_output_key(self.storage_config['prefix'], executor_id, job_id, call_id) status_key = create_status_key(self.storage_config['prefix'], executor_id, job_id, call_id) payload = { 'config': self.pywren_config, 'log_level': self.log_level, 'func_key': func_key, 'data_key': data_key, 'output_key': output_key, 'status_key': status_key, 'extra_env': job.extra_env, 'execution_timeout': job.execution_timeout, 'data_byte_range': data_byte_range, 'executor_id': executor_id, 'job_id': job_id, 'call_id': call_id, 'host_submit_time': time.time(), 'pywren_version': __version__ } # do the invocation compute_handler = random.choice(self.compute_handlers) activation_id = compute_handler.invoke(job.runtime_name, job.runtime_memory, payload) if not activation_id: raise Exception( "ExecutorID {} | JobID {} - Retrying mechanism finished with no success. " "Failed to invoke the job".format(executor_id, job_id)) job_metadata['activation_id'] = activation_id fut = ResponseFuture(executor_id, job_id, call_id, self.storage_config, job_metadata) fut._set_state(ResponseFuture.State.Invoked) return fut
def invoke_with_keys(self, func_key, data_key, output_key, status_key, executor_id, callgroup_id, call_id, extra_env, extra_meta, data_byte_range, host_job_meta, job_max_runtime, overwrite_invoke_args=None): log_level = 'INFO' if not self.log_level else self.log_level arg_dict = { 'config': self.config, 'log_level': log_level, 'func_key': func_key, 'data_key': data_key, 'output_key': output_key, 'status_key': status_key, 'job_max_runtime': job_max_runtime, 'data_byte_range': data_byte_range, 'executor_id': executor_id, 'callgroup_id': callgroup_id, 'call_id': call_id, 'pywren_version': version.__version__} if extra_env is not None: logger.debug("Extra environment vars {}".format(extra_env)) arg_dict['extra_env'] = extra_env if extra_meta is not None: # sanity for k, v in extra_meta.items(): if k in arg_dict: raise ValueError("Key {} already in dict".format(k)) arg_dict[k] = v host_submit_time = time.time() arg_dict['host_submit_time'] = host_submit_time # logger.debug("Executor ID {} Activation {} invoke".format(executor_id, call_id)) # overwrite explicit args, mostly used for testing via injection if overwrite_invoke_args is not None: arg_dict.update(overwrite_invoke_args) cf_invoke_time_start = time.time() # do the invocation activation_id = self.invoker.invoke(arg_dict) if not activation_id: raise Exception("Executor ID {} Activation {} failed, therefore job is failed".format(executor_id, call_id)) host_job_meta['cf_activation_id'] = activation_id host_job_meta['cf_invoke_timestamp'] = cf_invoke_time_start host_job_meta['cf_invoke_time'] = time.time() - cf_invoke_time_start # logger.debug("Executor ID {} Activation {} complete".format(executor_id, call_id)) host_job_meta.update(self.invoker.config()) host_job_meta.update(arg_dict) del host_job_meta['config'] storage_config = self.internal_storage.get_storage_config() fut = ResponseFuture(call_id, callgroup_id, executor_id, activation_id, host_job_meta, storage_config) fut._set_state(JobState.invoked) return fut
def run(self, job_description): """ Run a job described in job_description """ job = SimpleNamespace(**job_description) try: while True: self.token_bucket_q.get_nowait() self.ongoing_activations -= 1 except Exception: pass if self.remote_invoker and job.total_calls > 1: old_stdout = sys.stdout sys.stdout = open(os.devnull, 'w') self.select_runtime(job.job_id, 1024) sys.stdout = old_stdout log_msg = ( 'ExecutorID {} | JobID {} - Starting remote function invocation: {}() ' '- Total: {} activations'.format(job.executor_id, job.job_id, job.func_name, job.total_calls)) logger.info(log_msg) if not self.log_level: print(log_msg) th = Thread(target=self._remote_invoke, args=(job_description, )) th.daemon = True th.start() else: log_msg = ( 'ExecutorID {} | JobID {} - Starting function invocation: {}() - Total: {} ' 'activations'.format(job.executor_id, job.job_id, job.func_name, job.total_calls)) logger.info(log_msg) if not self.log_level: print(log_msg) if self.ongoing_activations < self.workers: callids = range(job.total_calls) total_direct = self.workers - self.ongoing_activations callids_to_invoke_direct = callids[:total_direct] callids_to_invoke_nondirect = callids[total_direct:] self.ongoing_activations += len(callids_to_invoke_direct) call_futures = [] with ThreadPoolExecutor( max_workers=job.invoke_pool_threads) as executor: for i in callids_to_invoke_direct: call_id = "{:05d}".format(i) future = executor.submit(self._invoke, job, call_id) call_futures.append(future) # Block until all direct invocations have finished callids_invoked = [ft.result() for ft in call_futures] # Put into the queue the rest of the callids to invoke within the process for i in callids_to_invoke_nondirect: call_id = "{:05d}".format(i) self.pending_calls_q.put((job, call_id)) self.start_job_status_checker(job) else: for i in range(job.total_calls): call_id = "{:05d}".format(i) self.pending_calls_q.put((job, call_id)) self.start_job_status_checker(job) # Create all futures futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(self.executor_id, job.job_id, call_id, self.storage_config, job.metadata) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures
def run(self, job_description): """ Run a job described in job_description """ job = SimpleNamespace(**job_description) try: while True: self.token_bucket_q.get_nowait() self.ongoing_activations -= 1 except Exception: pass if self.remote_invoker: old_stdout = sys.stdout sys.stdout = open(os.devnull, 'w') self.select_runtime(job.job_id, REMOTE_INVOKER_MEMORY) sys.stdout = old_stdout log_msg = ( 'ExecutorID {} | JobID {} - Starting remote function invocation: {}() ' '- Total: {} activations'.format(job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) if not self.log_level: print(log_msg) th = Thread(target=self._invoke_remote, args=(job_description, )) th.daemon = True th.start() time.sleep(0.1) else: try: if self.running_flag.value == 0: self.ongoing_activations = 0 self.running_flag.value = 1 self._start_invoker_process() log_msg = ( 'ExecutorID {} | JobID {} - Starting function invocation: {}() - Total: {} ' 'activations'.format(job.executor_id, job.job_id, job.function_name, job.total_calls)) logger.info(log_msg) if not self.log_level: print(log_msg) if self.ongoing_activations < self.workers: callids = range(job.total_calls) total_direct = self.workers - self.ongoing_activations callids_to_invoke_direct = callids[:total_direct] callids_to_invoke_nondirect = callids[total_direct:] self.ongoing_activations += len(callids_to_invoke_direct) logger.debug( 'ExecutorID {} | JobID {} - Free workers: {} - Going to invoke {} function activations' .format(job.executor_id, job.job_id, total_direct, len(callids_to_invoke_direct))) call_futures = [] executor = ThreadPoolExecutor( max_workers=job.invoke_pool_threads) for i in callids_to_invoke_direct: call_id = "{:05d}".format(i) future = executor.submit(self._invoke, job, call_id) call_futures.append(future) time.sleep(0.1) # Put into the queue the rest of the callids to invoke within the process if callids_to_invoke_nondirect: logger.debug( 'ExecutorID {} | JobID {} - Putting remaining {} function invocations into pending queue' .format(job.executor_id, job.job_id, len(callids_to_invoke_nondirect))) for i in callids_to_invoke_nondirect: call_id = "{:05d}".format(i) self.pending_calls_q.put((job, call_id)) else: logger.debug( 'ExecutorID {} | JobID {} - Ongoing activations reached {} workers, ' 'putting {} function invocations into pending queue'. format(job.executor_id, job.job_id, self.workers, job.total_calls)) for i in range(job.total_calls): call_id = "{:05d}".format(i) self.pending_calls_q.put((job, call_id)) self.job_monitor.start_job_monitoring(job) except (KeyboardInterrupt, Exception) as e: self.stop() raise e # Create all futures futures = [] for i in range(job.total_calls): call_id = "{:05d}".format(i) fut = ResponseFuture(call_id, job_description, job.metadata.copy(), self.storage_config) fut._set_state(ResponseFuture.State.Invoked) futures.append(fut) return futures
def invoke_with_keys(self, func_key, data_key, output_key, status_key, executor_id, callgroup_id, call_id, extra_env, extra_meta, data_byte_range, host_job_meta, job_max_runtime, overwrite_invoke_args=None): storage_config = self.storage_handler.get_storage_config() arg_dict = { 'config': self.config, 'storage_config': storage_config, 'func_key': func_key, 'data_key': data_key, 'output_key': output_key, 'status_key': status_key, 'job_max_runtime': job_max_runtime, 'data_byte_range': data_byte_range, 'executor_id': executor_id, 'callgroup_id': callgroup_id, 'call_id': call_id, 'pywren_version': version.__version__ } if extra_env is not None: logger.debug("Extra environment vars {}".format(extra_env)) arg_dict['extra_env'] = extra_env if extra_meta is not None: # sanity for k, v in extra_meta.items(): if k in arg_dict: raise ValueError("Key {} already in dict".format(k)) arg_dict[k] = v host_submit_time = time.time() arg_dict['host_submit_time'] = host_submit_time logger.debug("call_async {} {} cf invoke ".format( executor_id, call_id)) # overwrite explicit args, mostly used for testing via injection if overwrite_invoke_args is not None: arg_dict.update(overwrite_invoke_args) lambda_invoke_time_start = time.time() # do the invocation activation_id = self.invoker.invoke(arg_dict) host_job_meta['cf_activation_id'] = activation_id host_job_meta['cf_invoke_timestamp'] = lambda_invoke_time_start host_job_meta['cf_invoke_time'] = time.time( ) - lambda_invoke_time_start host_job_meta.update(self.invoker.config()) logger.debug("call_async {} {} cf invoke complete".format( executor_id, call_id)) host_job_meta.update(arg_dict) fut = ResponseFuture(call_id, callgroup_id, executor_id, activation_id, host_job_meta, storage_config) fut._set_state(JobState.invoked) return fut