def invoke(executor_id, job_id, call_id, func_key, invoke_metadata,
data_key, data_byte_range):
output_key = create_output_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
status_key = create_status_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
payload = {
'config': self.config,
'log_level': self.log_level,
'func_key': func_key,
'data_key': data_key,
'output_key': output_key,
'status_key': status_key,
'task_execution_timeout': job.task_execution_timeout,
'data_byte_range': data_byte_range,
'executor_id': executor_id,
'job_id': job_id,
'call_id': call_id,
'pywren_version': __version__
}
if job.extra_env is not None:
logger.debug("Extra environment vars {}".format(job.extra_env))
payload['extra_env'] = job.extra_env
if job.extra_meta is not None:
# sanity
for k, v in job.extra_meta.items():
if k in payload:
raise ValueError("Key {} already in dict".format(k))
payload[k] = v
# overwrite explicit args, mostly used for testing via injection
if job.overwrite_invoke_args is not None:
payload.update(job.overwrite_invoke_args)
host_submit_time = time.time()
payload['host_submit_time'] = host_submit_time
# do the invocation
activation_id = self.internal_compute.invoke(
job.runtime_name, job.runtime_memory, payload)
if not activation_id:
raise Exception(
"ExecutorID {} - Activation {} failed, therefore job is failed"
.format(executor_id, call_id))
invoke_metadata['activation_id'] = activation_id
invoke_metadata['invoke_time'] = time.time() - host_submit_time
invoke_metadata.update(payload)
del invoke_metadata['config']
fut = ResponseFuture(call_id, job_id, executor_id, activation_id,
self.storage_config, invoke_metadata)
fut._set_state(JobState.invoked)
return fut
def _future_timeout_checker_thread(running_futures, internal_storage, throw_except):
should_run = True
while should_run:
try:
while True:
current_time = time.time()
for fut in running_futures:
if fut.running and fut._call_status:
fut_timeout = fut._call_status['start_time'] + fut.execution_timeout + 5
if current_time > fut_timeout:
msg = 'The function did not run as expected.'
raise TimeoutError('HANDLER', msg)
time.sleep(5)
except TimeoutError:
# generate fake TimeoutError call status
pickled_exception = str(pickle.dumps(sys.exc_info()))
call_status = {'type': '__end__',
'exception': True,
'exc_info': pickled_exception,
'executor_id': fut.executor_id,
'job_id': fut.job_id,
'call_id': fut.call_id,
'activation_id': fut.activation_id}
status_key = create_status_key(JOBS_PREFIX, fut.executor_id, fut.job_id, fut.call_id)
dmpd_response_status = json.dumps(call_status)
internal_storage.put_data(status_key, dmpd_response_status)
if throw_except:
should_run = False
except Exception:
pass
def get_call_status(self, executor_id, job_id, call_id):
"""
Get status of a call.
:param executor_id: executor ID of the call
:param call_id: call ID of the call
:return: A dictionary containing call's status, or None if no updated status
"""
status_key = create_status_key(JOBS_PREFIX, executor_id, job_id, call_id)
try:
data = self.storage_handler.get_object(self.bucket, status_key)
return json.loads(data.decode('ascii'))
except StorageNoSuchKeyError:
return None
def invoke(executor_id, job_id, call_id, func_key, invoke_metadata,
data_key, data_byte_range):
output_key = create_output_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
status_key = create_status_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
payload = {
'config': self.config,
'log_level': self.log_level,
'func_key': func_key,
'data_key': data_key,
'output_key': output_key,
'status_key': status_key,
'execution_timeout': job.execution_timeout,
'data_byte_range': data_byte_range,
'executor_id': executor_id,
'job_id': job_id,
'call_id': call_id,
'pywren_version': __version__
}
if job.extra_env is not None:
logger.debug("Extra environment vars {}".format(job.extra_env))
payload['extra_env'] = job.extra_env
host_submit_time = time.time()
payload['host_submit_time'] = host_submit_time
# do the invocation
activation_id = self.compute.invoke(job.runtime_name,
job.runtime_memory, payload)
if not activation_id:
raise Exception(
"ExecutorID {} | JobID {} - Retrying mechanism finished with no success. "
"Failed to invoke the job".format(executor_id, job_id))
invoke_metadata['activation_id'] = activation_id
invoke_metadata['invoke_time'] = time.time() - host_submit_time
invoke_metadata.update(payload)
del invoke_metadata['config']
fut = ResponseFuture(call_id, job_id, executor_id, activation_id,
self.storage_config, invoke_metadata)
fut._set_state(CallState.invoked)
return fut
def _send_status_os(self):
"""
Send the status event to the Object Storage
"""
executor_id = self.response['executor_id']
job_id = self.response['job_id']
call_id = self.response['call_id']
if self.response['type'] == '__init__':
init_key = create_init_key(JOBS_PREFIX, executor_id, job_id, call_id)
self.internal_storage.put_data(init_key, '')
elif self.response['type'] == '__end__':
status_key = create_status_key(JOBS_PREFIX, executor_id, job_id, call_id)
dmpd_response_status = json.dumps(self.response)
drs = sizeof_fmt(len(dmpd_response_status))
logger.info("Storing execution stats - status.json - Size: {}".format(drs))
self.internal_storage.put_data(status_key, dmpd_response_status)
def _invoke(self, job, call_id):
"""
Method used to perform the actual invocation against the Compute Backend
"""
output_key = create_output_key(JOBS_PREFIX, job.executor_id,
job.job_id, call_id)
status_key = create_status_key(JOBS_PREFIX, job.executor_id,
job.job_id, call_id)
payload = {
'config': self.config,
'log_level': self.log_level,
'func_key': job.func_key,
'data_key': job.data_key,
'output_key': output_key,
'status_key': status_key,
'extra_env': job.extra_env,
'execution_timeout': job.execution_timeout,
'data_byte_range': job.data_ranges[int(call_id)],
'executor_id': job.executor_id,
'job_id': job.job_id,
'call_id': call_id,
'host_submit_time': time.time(),
'pywren_version': __version__
}
# do the invocation
start = time.time()
compute_handler = random.choice(self.compute_handlers)
activation_id = compute_handler.invoke(job.runtime_name,
job.runtime_memory, payload)
roundtrip = time.time() - start
resp_time = format(round(roundtrip, 3), '.3f')
if not activation_id:
self.pending_calls_q.put((job, call_id))
return
logger.debug(
'ExecutorID {} | JobID {} - Function invocation {} done! ({}s) - Activation'
' ID: {}'.format(job.executor_id, job.job_id, call_id, resp_time,
activation_id))
return call_id
def invoke(executor_id, job_id, call_id, func_key, job_metadata,
data_key, data_byte_range):
output_key = create_output_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
status_key = create_status_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
payload = {
'config': self.pywren_config,
'log_level': self.log_level,
'func_key': func_key,
'data_key': data_key,
'output_key': output_key,
'status_key': status_key,
'extra_env': job.extra_env,
'execution_timeout': job.execution_timeout,
'data_byte_range': data_byte_range,
'executor_id': executor_id,
'job_id': job_id,
'call_id': call_id,
'host_submit_time': time.time(),
'pywren_version': __version__
}
# do the invocation
compute_handler = random.choice(self.compute_handlers)
activation_id = compute_handler.invoke(job.runtime_name,
job.runtime_memory, payload)
if not activation_id:
raise Exception(
"ExecutorID {} | JobID {} - Retrying mechanism finished with no success. "
"Failed to invoke the job".format(executor_id, job_id))
job_metadata['activation_id'] = activation_id
fut = ResponseFuture(executor_id, job_id, call_id,
self.storage_config, job_metadata)
fut._set_state(ResponseFuture.State.Invoked)
return fut
