def invoke(executor_id, job_id, call_id, func_key, invoke_metadata, data_key, data_byte_range): output_key = create_output_key(self.storage_config['prefix'], executor_id, job_id, call_id) status_key = create_status_key(self.storage_config['prefix'], executor_id, job_id, call_id) payload = { 'config': self.config, 'log_level': self.log_level, 'func_key': func_key, 'data_key': data_key, 'output_key': output_key, 'status_key': status_key, 'task_execution_timeout': job.task_execution_timeout, 'data_byte_range': data_byte_range, 'executor_id': executor_id, 'job_id': job_id, 'call_id': call_id, 'pywren_version': __version__ } if job.extra_env is not None: logger.debug("Extra environment vars {}".format(job.extra_env)) payload['extra_env'] = job.extra_env if job.extra_meta is not None: # sanity for k, v in job.extra_meta.items(): if k in payload: raise ValueError("Key {} already in dict".format(k)) payload[k] = v # overwrite explicit args, mostly used for testing via injection if job.overwrite_invoke_args is not None: payload.update(job.overwrite_invoke_args) host_submit_time = time.time() payload['host_submit_time'] = host_submit_time # do the invocation activation_id = self.internal_compute.invoke( job.runtime_name, job.runtime_memory, payload) if not activation_id: raise Exception( "ExecutorID {} - Activation {} failed, therefore job is failed" .format(executor_id, call_id)) invoke_metadata['activation_id'] = activation_id invoke_metadata['invoke_time'] = time.time() - host_submit_time invoke_metadata.update(payload) del invoke_metadata['config'] fut = ResponseFuture(call_id, job_id, executor_id, activation_id, self.storage_config, invoke_metadata) fut._set_state(JobState.invoked) return fut
def _future_timeout_checker_thread(running_futures, internal_storage, throw_except): should_run = True while should_run: try: while True: current_time = time.time() for fut in running_futures: if fut.running and fut._call_status: fut_timeout = fut._call_status['start_time'] + fut.execution_timeout + 5 if current_time > fut_timeout: msg = 'The function did not run as expected.' raise TimeoutError('HANDLER', msg) time.sleep(5) except TimeoutError: # generate fake TimeoutError call status pickled_exception = str(pickle.dumps(sys.exc_info())) call_status = {'type': '__end__', 'exception': True, 'exc_info': pickled_exception, 'executor_id': fut.executor_id, 'job_id': fut.job_id, 'call_id': fut.call_id, 'activation_id': fut.activation_id} status_key = create_status_key(JOBS_PREFIX, fut.executor_id, fut.job_id, fut.call_id) dmpd_response_status = json.dumps(call_status) internal_storage.put_data(status_key, dmpd_response_status) if throw_except: should_run = False except Exception: pass
def get_call_status(self, executor_id, job_id, call_id): """ Get status of a call. :param executor_id: executor ID of the call :param call_id: call ID of the call :return: A dictionary containing call's status, or None if no updated status """ status_key = create_status_key(JOBS_PREFIX, executor_id, job_id, call_id) try: data = self.storage_handler.get_object(self.bucket, status_key) return json.loads(data.decode('ascii')) except StorageNoSuchKeyError: return None
def invoke(executor_id, job_id, call_id, func_key, invoke_metadata, data_key, data_byte_range): output_key = create_output_key(self.storage_config['prefix'], executor_id, job_id, call_id) status_key = create_status_key(self.storage_config['prefix'], executor_id, job_id, call_id) payload = { 'config': self.config, 'log_level': self.log_level, 'func_key': func_key, 'data_key': data_key, 'output_key': output_key, 'status_key': status_key, 'execution_timeout': job.execution_timeout, 'data_byte_range': data_byte_range, 'executor_id': executor_id, 'job_id': job_id, 'call_id': call_id, 'pywren_version': __version__ } if job.extra_env is not None: logger.debug("Extra environment vars {}".format(job.extra_env)) payload['extra_env'] = job.extra_env host_submit_time = time.time() payload['host_submit_time'] = host_submit_time # do the invocation activation_id = self.compute.invoke(job.runtime_name, job.runtime_memory, payload) if not activation_id: raise Exception( "ExecutorID {} | JobID {} - Retrying mechanism finished with no success. " "Failed to invoke the job".format(executor_id, job_id)) invoke_metadata['activation_id'] = activation_id invoke_metadata['invoke_time'] = time.time() - host_submit_time invoke_metadata.update(payload) del invoke_metadata['config'] fut = ResponseFuture(call_id, job_id, executor_id, activation_id, self.storage_config, invoke_metadata) fut._set_state(CallState.invoked) return fut
def _send_status_os(self): """ Send the status event to the Object Storage """ executor_id = self.response['executor_id'] job_id = self.response['job_id'] call_id = self.response['call_id'] if self.response['type'] == '__init__': init_key = create_init_key(JOBS_PREFIX, executor_id, job_id, call_id) self.internal_storage.put_data(init_key, '') elif self.response['type'] == '__end__': status_key = create_status_key(JOBS_PREFIX, executor_id, job_id, call_id) dmpd_response_status = json.dumps(self.response) drs = sizeof_fmt(len(dmpd_response_status)) logger.info("Storing execution stats - status.json - Size: {}".format(drs)) self.internal_storage.put_data(status_key, dmpd_response_status)
def _invoke(self, job, call_id): """ Method used to perform the actual invocation against the Compute Backend """ output_key = create_output_key(JOBS_PREFIX, job.executor_id, job.job_id, call_id) status_key = create_status_key(JOBS_PREFIX, job.executor_id, job.job_id, call_id) payload = { 'config': self.config, 'log_level': self.log_level, 'func_key': job.func_key, 'data_key': job.data_key, 'output_key': output_key, 'status_key': status_key, 'extra_env': job.extra_env, 'execution_timeout': job.execution_timeout, 'data_byte_range': job.data_ranges[int(call_id)], 'executor_id': job.executor_id, 'job_id': job.job_id, 'call_id': call_id, 'host_submit_time': time.time(), 'pywren_version': __version__ } # do the invocation start = time.time() compute_handler = random.choice(self.compute_handlers) activation_id = compute_handler.invoke(job.runtime_name, job.runtime_memory, payload) roundtrip = time.time() - start resp_time = format(round(roundtrip, 3), '.3f') if not activation_id: self.pending_calls_q.put((job, call_id)) return logger.debug( 'ExecutorID {} | JobID {} - Function invocation {} done! ({}s) - Activation' ' ID: {}'.format(job.executor_id, job.job_id, call_id, resp_time, activation_id)) return call_id
def invoke(executor_id, job_id, call_id, func_key, job_metadata, data_key, data_byte_range): output_key = create_output_key(self.storage_config['prefix'], executor_id, job_id, call_id) status_key = create_status_key(self.storage_config['prefix'], executor_id, job_id, call_id) payload = { 'config': self.pywren_config, 'log_level': self.log_level, 'func_key': func_key, 'data_key': data_key, 'output_key': output_key, 'status_key': status_key, 'extra_env': job.extra_env, 'execution_timeout': job.execution_timeout, 'data_byte_range': data_byte_range, 'executor_id': executor_id, 'job_id': job_id, 'call_id': call_id, 'host_submit_time': time.time(), 'pywren_version': __version__ } # do the invocation compute_handler = random.choice(self.compute_handlers) activation_id = compute_handler.invoke(job.runtime_name, job.runtime_memory, payload) if not activation_id: raise Exception( "ExecutorID {} | JobID {} - Retrying mechanism finished with no success. " "Failed to invoke the job".format(executor_id, job_id)) job_metadata['activation_id'] = activation_id fut = ResponseFuture(executor_id, job_id, call_id, self.storage_config, job_metadata) fut._set_state(ResponseFuture.State.Invoked) return fut