def invoke(executor_id, job_id, call_id, func_key, invoke_metadata,
data_key, data_byte_range):
output_key = create_output_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
status_key = create_status_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
payload = {
'config': self.config,
'log_level': self.log_level,
'func_key': func_key,
'data_key': data_key,
'output_key': output_key,
'status_key': status_key,
'task_execution_timeout': job.task_execution_timeout,
'data_byte_range': data_byte_range,
'executor_id': executor_id,
'job_id': job_id,
'call_id': call_id,
'pywren_version': __version__
}
if job.extra_env is not None:
logger.debug("Extra environment vars {}".format(job.extra_env))
payload['extra_env'] = job.extra_env
if job.extra_meta is not None:
# sanity
for k, v in job.extra_meta.items():
if k in payload:
raise ValueError("Key {} already in dict".format(k))
payload[k] = v
# overwrite explicit args, mostly used for testing via injection
if job.overwrite_invoke_args is not None:
payload.update(job.overwrite_invoke_args)
host_submit_time = time.time()
payload['host_submit_time'] = host_submit_time
# do the invocation
activation_id = self.internal_compute.invoke(
job.runtime_name, job.runtime_memory, payload)
if not activation_id:
raise Exception(
"ExecutorID {} - Activation {} failed, therefore job is failed"
.format(executor_id, call_id))
invoke_metadata['activation_id'] = activation_id
invoke_metadata['invoke_time'] = time.time() - host_submit_time
invoke_metadata.update(payload)
del invoke_metadata['config']
fut = ResponseFuture(call_id, job_id, executor_id, activation_id,
self.storage_config, invoke_metadata)
fut._set_state(JobState.invoked)
return fut
def invoke(executor_id, job_id, call_id, func_key, invoke_metadata,
data_key, data_byte_range):
output_key = create_output_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
status_key = create_status_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
payload = {
'config': self.config,
'log_level': self.log_level,
'func_key': func_key,
'data_key': data_key,
'output_key': output_key,
'status_key': status_key,
'execution_timeout': job.execution_timeout,
'data_byte_range': data_byte_range,
'executor_id': executor_id,
'job_id': job_id,
'call_id': call_id,
'pywren_version': __version__
}
if job.extra_env is not None:
logger.debug("Extra environment vars {}".format(job.extra_env))
payload['extra_env'] = job.extra_env
host_submit_time = time.time()
payload['host_submit_time'] = host_submit_time
# do the invocation
activation_id = self.compute.invoke(job.runtime_name,
job.runtime_memory, payload)
if not activation_id:
raise Exception(
"ExecutorID {} | JobID {} - Retrying mechanism finished with no success. "
"Failed to invoke the job".format(executor_id, job_id))
invoke_metadata['activation_id'] = activation_id
invoke_metadata['invoke_time'] = time.time() - host_submit_time
invoke_metadata.update(payload)
del invoke_metadata['config']
fut = ResponseFuture(call_id, job_id, executor_id, activation_id,
self.storage_config, invoke_metadata)
fut._set_state(CallState.invoked)
return fut
def invoke(executor_id, job_id, call_id, func_key, job_metadata,
data_key, data_byte_range):
output_key = create_output_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
status_key = create_status_key(self.storage_config['prefix'],
executor_id, job_id, call_id)
payload = {
'config': self.pywren_config,
'log_level': self.log_level,
'func_key': func_key,
'data_key': data_key,
'output_key': output_key,
'status_key': status_key,
'extra_env': job.extra_env,
'execution_timeout': job.execution_timeout,
'data_byte_range': data_byte_range,
'executor_id': executor_id,
'job_id': job_id,
'call_id': call_id,
'host_submit_time': time.time(),
'pywren_version': __version__
}
# do the invocation
compute_handler = random.choice(self.compute_handlers)
activation_id = compute_handler.invoke(job.runtime_name,
job.runtime_memory, payload)
if not activation_id:
raise Exception(
"ExecutorID {} | JobID {} - Retrying mechanism finished with no success. "
"Failed to invoke the job".format(executor_id, job_id))
job_metadata['activation_id'] = activation_id
fut = ResponseFuture(executor_id, job_id, call_id,
self.storage_config, job_metadata)
fut._set_state(ResponseFuture.State.Invoked)
return fut
def invoke_with_keys(self, func_key, data_key, output_key,
status_key, executor_id, callgroup_id,
call_id, extra_env,
extra_meta, data_byte_range,
host_job_meta, job_max_runtime,
overwrite_invoke_args=None):
log_level = 'INFO' if not self.log_level else self.log_level
arg_dict = {
'config': self.config,
'log_level': log_level,
'func_key': func_key,
'data_key': data_key,
'output_key': output_key,
'status_key': status_key,
'job_max_runtime': job_max_runtime,
'data_byte_range': data_byte_range,
'executor_id': executor_id,
'callgroup_id': callgroup_id,
'call_id': call_id,
'pywren_version': version.__version__}
if extra_env is not None:
logger.debug("Extra environment vars {}".format(extra_env))
arg_dict['extra_env'] = extra_env
if extra_meta is not None:
# sanity
for k, v in extra_meta.items():
if k in arg_dict:
raise ValueError("Key {} already in dict".format(k))
arg_dict[k] = v
host_submit_time = time.time()
arg_dict['host_submit_time'] = host_submit_time
# logger.debug("Executor ID {} Activation {} invoke".format(executor_id, call_id))
# overwrite explicit args, mostly used for testing via injection
if overwrite_invoke_args is not None:
arg_dict.update(overwrite_invoke_args)
cf_invoke_time_start = time.time()
# do the invocation
activation_id = self.invoker.invoke(arg_dict)
if not activation_id:
raise Exception("Executor ID {} Activation {} failed, therefore job is failed".format(executor_id, call_id))
host_job_meta['cf_activation_id'] = activation_id
host_job_meta['cf_invoke_timestamp'] = cf_invoke_time_start
host_job_meta['cf_invoke_time'] = time.time() - cf_invoke_time_start
# logger.debug("Executor ID {} Activation {} complete".format(executor_id, call_id))
host_job_meta.update(self.invoker.config())
host_job_meta.update(arg_dict)
del host_job_meta['config']
storage_config = self.internal_storage.get_storage_config()
fut = ResponseFuture(call_id, callgroup_id, executor_id, activation_id, host_job_meta, storage_config)
fut._set_state(JobState.invoked)
return fut
def run(self, job_description):
"""
Run a job described in job_description
"""
job = SimpleNamespace(**job_description)
try:
while True:
self.token_bucket_q.get_nowait()
self.ongoing_activations -= 1
except Exception:
pass
if self.remote_invoker and job.total_calls > 1:
old_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
self.select_runtime(job.job_id, 1024)
sys.stdout = old_stdout
log_msg = (
'ExecutorID {} | JobID {} - Starting remote function invocation: {}() '
'- Total: {} activations'.format(job.executor_id, job.job_id,
job.func_name,
job.total_calls))
logger.info(log_msg)
if not self.log_level:
print(log_msg)
th = Thread(target=self._remote_invoke, args=(job_description, ))
th.daemon = True
th.start()
else:
log_msg = (
'ExecutorID {} | JobID {} - Starting function invocation: {}() - Total: {} '
'activations'.format(job.executor_id, job.job_id,
job.func_name, job.total_calls))
logger.info(log_msg)
if not self.log_level:
print(log_msg)
if self.ongoing_activations < self.workers:
callids = range(job.total_calls)
total_direct = self.workers - self.ongoing_activations
callids_to_invoke_direct = callids[:total_direct]
callids_to_invoke_nondirect = callids[total_direct:]
self.ongoing_activations += len(callids_to_invoke_direct)
call_futures = []
with ThreadPoolExecutor(
max_workers=job.invoke_pool_threads) as executor:
for i in callids_to_invoke_direct:
call_id = "{:05d}".format(i)
future = executor.submit(self._invoke, job, call_id)
call_futures.append(future)
# Block until all direct invocations have finished
callids_invoked = [ft.result() for ft in call_futures]
# Put into the queue the rest of the callids to invoke within the process
for i in callids_to_invoke_nondirect:
call_id = "{:05d}".format(i)
self.pending_calls_q.put((job, call_id))
self.start_job_status_checker(job)
else:
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
self.pending_calls_q.put((job, call_id))
self.start_job_status_checker(job)
# Create all futures
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(self.executor_id, job.job_id, call_id,
self.storage_config, job.metadata)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures
def run(self, job_description):
"""
Run a job described in job_description
"""
job = SimpleNamespace(**job_description)
try:
while True:
self.token_bucket_q.get_nowait()
self.ongoing_activations -= 1
except Exception:
pass
if self.remote_invoker:
old_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
self.select_runtime(job.job_id, REMOTE_INVOKER_MEMORY)
sys.stdout = old_stdout
log_msg = (
'ExecutorID {} | JobID {} - Starting remote function invocation: {}() '
'- Total: {} activations'.format(job.executor_id, job.job_id,
job.function_name,
job.total_calls))
logger.info(log_msg)
if not self.log_level:
print(log_msg)
th = Thread(target=self._invoke_remote, args=(job_description, ))
th.daemon = True
th.start()
time.sleep(0.1)
else:
try:
if self.running_flag.value == 0:
self.ongoing_activations = 0
self.running_flag.value = 1
self._start_invoker_process()
log_msg = (
'ExecutorID {} | JobID {} - Starting function invocation: {}() - Total: {} '
'activations'.format(job.executor_id, job.job_id,
job.function_name, job.total_calls))
logger.info(log_msg)
if not self.log_level:
print(log_msg)
if self.ongoing_activations < self.workers:
callids = range(job.total_calls)
total_direct = self.workers - self.ongoing_activations
callids_to_invoke_direct = callids[:total_direct]
callids_to_invoke_nondirect = callids[total_direct:]
self.ongoing_activations += len(callids_to_invoke_direct)
logger.debug(
'ExecutorID {} | JobID {} - Free workers: {} - Going to invoke {} function activations'
.format(job.executor_id, job.job_id, total_direct,
len(callids_to_invoke_direct)))
call_futures = []
executor = ThreadPoolExecutor(
max_workers=job.invoke_pool_threads)
for i in callids_to_invoke_direct:
call_id = "{:05d}".format(i)
future = executor.submit(self._invoke, job, call_id)
call_futures.append(future)
time.sleep(0.1)
# Put into the queue the rest of the callids to invoke within the process
if callids_to_invoke_nondirect:
logger.debug(
'ExecutorID {} | JobID {} - Putting remaining {} function invocations into pending queue'
.format(job.executor_id, job.job_id,
len(callids_to_invoke_nondirect)))
for i in callids_to_invoke_nondirect:
call_id = "{:05d}".format(i)
self.pending_calls_q.put((job, call_id))
else:
logger.debug(
'ExecutorID {} | JobID {} - Ongoing activations reached {} workers, '
'putting {} function invocations into pending queue'.
format(job.executor_id, job.job_id, self.workers,
job.total_calls))
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
self.pending_calls_q.put((job, call_id))
self.job_monitor.start_job_monitoring(job)
except (KeyboardInterrupt, Exception) as e:
self.stop()
raise e
# Create all futures
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job_description, job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures
def invoke_with_keys(self,
func_key,
data_key,
output_key,
status_key,
executor_id,
callgroup_id,
call_id,
extra_env,
extra_meta,
data_byte_range,
host_job_meta,
job_max_runtime,
overwrite_invoke_args=None):
storage_config = self.storage_handler.get_storage_config()
arg_dict = {
'config': self.config,
'storage_config': storage_config,
'func_key': func_key,
'data_key': data_key,
'output_key': output_key,
'status_key': status_key,
'job_max_runtime': job_max_runtime,
'data_byte_range': data_byte_range,
'executor_id': executor_id,
'callgroup_id': callgroup_id,
'call_id': call_id,
'pywren_version': version.__version__
}
if extra_env is not None:
logger.debug("Extra environment vars {}".format(extra_env))
arg_dict['extra_env'] = extra_env
if extra_meta is not None:
# sanity
for k, v in extra_meta.items():
if k in arg_dict:
raise ValueError("Key {} already in dict".format(k))
arg_dict[k] = v
host_submit_time = time.time()
arg_dict['host_submit_time'] = host_submit_time
logger.debug("call_async {} {} cf invoke ".format(
executor_id, call_id))
# overwrite explicit args, mostly used for testing via injection
if overwrite_invoke_args is not None:
arg_dict.update(overwrite_invoke_args)
lambda_invoke_time_start = time.time()
# do the invocation
activation_id = self.invoker.invoke(arg_dict)
host_job_meta['cf_activation_id'] = activation_id
host_job_meta['cf_invoke_timestamp'] = lambda_invoke_time_start
host_job_meta['cf_invoke_time'] = time.time(
) - lambda_invoke_time_start
host_job_meta.update(self.invoker.config())
logger.debug("call_async {} {} cf invoke complete".format(
executor_id, call_id))
host_job_meta.update(arg_dict)
fut = ResponseFuture(call_id, callgroup_id, executor_id, activation_id,
host_job_meta, storage_config)
fut._set_state(JobState.invoked)
return fut