def _run_job(self, job):
"""
Run a job
"""
if self.customized_runtime:
logger.debug(
'ExecutorID {} | JobID {} - Customized runtime activated'.
format(job.executor_id, job.job_id))
job.runtime_name = self.runtime_name
extend_runtime(job, self.compute_handler, self.internal_storage)
self.runtime_name = job.runtime_name
logger.info('ExecutorID {} | JobID {} - Starting function '
'invocation: {}() - Total: {} activations'.format(
job.executor_id, job.job_id, job.function_name,
job.total_calls))
logger.debug('ExecutorID {} | JobID {} - Chunksize:'
' {} - Worker processes: {}'.format(
job.executor_id, job.job_id, job.chunksize,
job.worker_processes))
self.prometheus.send_metric(name='job_total_calls',
value=job.total_calls,
type='counter',
labels=(('job_id_', job.job_key),
('function_name',
job.function_name)))
self.prometheus.send_metric(name='job_runtime_memory',
value=job.runtime_memory or 0,
type='counter',
labels=(('job_id_', job.job_key),
('function_name',
job.function_name)))
try:
job.runtime_name = self.runtime_name
self._invoke_job(job)
except (KeyboardInterrupt, Exception) as e:
self.stop()
raise e
log_file = os.path.join(LOGS_DIR, job.job_key + '.log')
logger.info(
"ExecutorID {} | JobID {} - View execution logs at {}".format(
job.executor_id, job.job_id, log_file))
# Create all futures
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job, job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
job.futures = futures
return futures
def run(self, job_description):
"""
Run a job
"""
job_description['runtime_name'] = self.runtime_name
job_description['runtime_memory'] = None
job_description['runtime_timeout'] = None
job = SimpleNamespace(**job_description)
payload = {
'config': self.config,
'log_level': logging.getLevelName(logger.getEffectiveLevel()),
'executor_id': job.executor_id,
'job_id': job.job_id,
'job_description': job_description,
'lithops_version': __version__
}
self.compute_handler.run_job(payload)
log_msg = ('ExecutorID {} | JobID {} - Invocation done'.format(
job.executor_id, job.job_id))
logger.info(log_msg)
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job_description, job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures
def run(self, job):
"""
Run a job
"""
job.runtime_name = self.runtime_name
payload = {
'config': self.config,
'log_level': logging.getLevelName(logger.getEffectiveLevel()),
'executor_id': job.executor_id,
'job_id': job.job_id,
'job_description': job.__dict__,
'lithops_version': __version__
}
self.compute_handler.run_job(payload)
log_msg = ('ExecutorID {} | JobID {} - {}() Invocation done'.format(
job.executor_id, job.job_id, job.function_name))
logger.info(log_msg)
if not self.log_active:
print(log_msg)
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job, job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures
def run(self, job):
"""
Run a job
"""
logger.info('ExecutorID {} | JobID {} - Starting function'
' invocation: {}() - Total: {} activations'.format(
job.executor_id, job.job_id, job.function_name,
job.total_calls))
logger.debug(
'ExecutorID {} | JobID {} - Chunksize: {} - Worker processes: {}'.
format(job.executor_id, job.job_id, job.chunksize,
job.worker_processes))
job.runtime_name = self.runtime_name
self.prometheus.send_metric(name='job_total_calls',
value=job.total_calls,
labels=(('executor_id', job.executor_id),
('job_id',
job.job_id), ('function_name',
job.function_name)))
payload = self._create_payload(job)
payload['call_ids'] = [
"{:05d}".format(i) for i in range(job.total_calls)
]
log_file = os.path.join(LOGS_DIR, payload['job_key'] + '.log')
logger.info(
"ExecutorID {} | JobID {} - View execution logs at {}".format(
job.executor_id, job.job_id, log_file))
self.compute_handler.run_job(payload)
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job, job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures
def run(self, job):
"""
Run a job
"""
job.runtime_name = self.runtime_name
self.prometheus.send_metric(name='job_total_calls',
value=job.total_calls,
labels=(
('job_id', job.job_id),
('function_name', job.function_name)
))
payload = {'config': self.config,
'log_level': self.log_level,
'executor_id': job.executor_id,
'job_id': job.job_id,
'job_description': job.__dict__,
'lithops_version': __version__}
self.compute_handler.run_job(payload)
log_msg = ('ExecutorID {} | JobID {} - {}() Invocation done - Total: {} activations'
.format(job.executor_id, job.job_id, job.function_name, job.total_calls))
logger.info(log_msg)
if not self.log_active:
print(log_msg)
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job,
job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures
def run(self, job):
"""
Run a job described in job_description
"""
job.runtime_name = self.runtime_name
try:
while True:
self.token_bucket_q.get_nowait()
self.ongoing_activations -= 1
except Exception:
pass
self.prometheus.send_metric(name='job_total_calls',
value=job.total_calls,
labels=(
('job_id', job.job_id),
('function_name', job.function_name)
))
if self.remote_invoker:
"""
Remote Invocation
Use a single cloud function to perform all the function invocations
"""
old_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
self.select_runtime(job.job_id, self.REMOTE_INVOKER_MEMORY)
sys.stdout = old_stdout
log_msg = ('ExecutorID {} | JobID {} - Starting remote function '
'invocation: {}() - Total: {} activations'
.format(job.executor_id, job.job_id,
job.function_name, job.total_calls))
logger.info(log_msg)
if not self.log_active:
print(log_msg)
th = Thread(target=self._invoke_remote, args=(job,), daemon=True)
th.start()
time.sleep(0.1)
else:
"""
Normal Invocation
Use local threads to perform all the function invocations
"""
try:
if self.running_flag.value == 0:
self.ongoing_activations = 0
self.running_flag.value = 1
self._start_invoker_process()
log_msg = ('ExecutorID {} | JobID {} - Starting function '
'invocation: {}() - Total: {} activations'
.format(job.executor_id, job.job_id,
job.function_name, job.total_calls))
logger.info(log_msg)
if not self.log_active:
print(log_msg)
if self.ongoing_activations < self.workers:
callids = range(job.total_calls)
total_direct = self.workers-self.ongoing_activations
callids_to_invoke_direct = callids[:total_direct]
callids_to_invoke_nondirect = callids[total_direct:]
self.ongoing_activations += len(callids_to_invoke_direct)
logger.debug('ExecutorID {} | JobID {} - Free workers: '
'{} - Going to invoke {} function activations'
.format(job.executor_id, job.job_id, total_direct,
len(callids_to_invoke_direct)))
def _callback(future):
future.result()
executor = ThreadPoolExecutor(job.invoke_pool_threads)
for i in callids_to_invoke_direct:
call_id = "{:05d}".format(i)
future = executor.submit(self._invoke, job, call_id)
future.add_done_callback(_callback)
time.sleep(0.1)
# Put into the queue the rest of the callids to invoke within the process
if callids_to_invoke_nondirect:
logger.debug('ExecutorID {} | JobID {} - Putting remaining '
'{} function invocations into pending queue'
.format(job.executor_id, job.job_id,
len(callids_to_invoke_nondirect)))
for i in callids_to_invoke_nondirect:
call_id = "{:05d}".format(i)
self.pending_calls_q.put((job, call_id))
else:
logger.debug('ExecutorID {} | JobID {} - Ongoing activations '
'reached {} workers, queuing {} function invocations'
.format(job.executor_id, job.job_id, self.workers,
job.total_calls))
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
self.pending_calls_q.put((job, call_id))
self.job_monitor.start_job_monitoring(job)
except (KeyboardInterrupt, Exception) as e:
self.stop()
raise e
# Create all futures
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job,
job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures
def run(self, job):
"""
Run a job described in job_description
"""
job.runtime_name = self.runtime_name
try:
while True:
self.token_bucket_q.get_nowait()
self.ongoing_activations -= 1
except Exception:
pass
self.prometheus.send_metric(name='job_total_calls',
value=job.total_calls,
labels=(('executor_id', job.executor_id),
('job_id',
job.job_id), ('function_name',
job.function_name)))
if self.remote_invoker:
"""
Remote Invocation
Use a single cloud function to perform all the function invocations
"""
log_level = logger.getEffectiveLevel()
logging.getLogger('lithops').setLevel(logging.CRITICAL)
self.select_runtime(job.job_id, self.REMOTE_INVOKER_MEMORY)
logging.getLogger('lithops').setLevel(log_level)
log_msg = ('ExecutorID {} | JobID {} - Starting function '
'invocation: {}() - Total: {} activations'.format(
job.executor_id, job.job_id, job.function_name,
job.total_calls))
logger.info(log_msg)
self._invoke_remote(job)
# th = Thread(target=self._invoke_remote, args=(job,), daemon=True)
# th.start()
else:
"""
Normal Invocation
Use local threads to perform all the function invocations
"""
try:
if self.running_flag.value == 0:
self.running_workers = 0
self.running_flag.value = 1
self._start_invoker_process()
log_msg = ('ExecutorID {} | JobID {} - Starting function '
'invocation: {}() - Total: {} activations'.format(
job.executor_id, job.job_id, job.function_name,
job.total_calls))
logger.info(log_msg)
logger.debug('ExecutorID {} | JobID {} - Chunksize:'
' {} - Worker processes: {}'.format(
job.executor_id, job.job_id, job.chunksize,
job.worker_processes))
if self.running_workers < self.workers:
free_workers = self.workers - self.running_workers
total_direct = free_workers * job.chunksize
callids = range(job.total_calls)
callids_to_invoke_direct = callids[:total_direct]
callids_to_invoke_nondirect = callids[total_direct:]
ci = len(callids_to_invoke_direct)
cz = job.chunksize
consumed_workers = ci // cz + (ci % cz > 0)
self.running_workers += consumed_workers
logger.debug(
'ExecutorID {} | JobID {} - Free workers:'
' {} - Going to run {} activations in {} workers'.
format(job.executor_id, job.job_id, free_workers,
len(callids_to_invoke_direct),
consumed_workers))
def _callback(future):
future.result()
executor = ThreadPoolExecutor(job.invoke_pool_threads)
for call_ids_range in iterchunks(callids_to_invoke_direct,
job.chunksize):
future = executor.submit(self._invoke, job,
call_ids_range)
future.add_done_callback(_callback)
time.sleep(0.1)
# Put into the queue the rest of the callids to invoke within the process
if callids_to_invoke_nondirect:
logger.debug(
'ExecutorID {} | JobID {} - Putting remaining '
'{} function activations into pending queue'.
format(job.executor_id, job.job_id,
len(callids_to_invoke_nondirect)))
for call_ids_range in iterchunks(
callids_to_invoke_nondirect, job.chunksize):
self.pending_calls_q.put((job, call_ids_range))
else:
logger.debug(
'ExecutorID {} | JobID {} - Reached maximun {} '
'workers, queuing {} function activations'.format(
job.executor_id, job.job_id, self.workers,
job.total_calls))
for call_ids_range in iterchunks(job.total_calls,
job.chunksize):
self.pending_calls_q.put((job, call_ids_range))
self.job_monitor.start_job_monitoring(job)
except (KeyboardInterrupt, Exception) as e:
self.stop()
raise e
log_file = os.path.join(LOGS_DIR, job.job_key + '.log')
logger.info(
"ExecutorID {} | JobID {} - View execution logs at {}".format(
job.executor_id, job.job_id, log_file))
# Create all futures
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job, job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures
def run(self, job_description):
"""
Run a job described in job_description
"""
job_description['runtime_name'] = self.config['serverless']['runtime']
job_description['runtime_memory'] = self.config['serverless'][
'runtime_memory']
job_description['runtime_timeout'] = self.config['serverless'][
'runtime_timeout']
execution_timeout = job_description['execution_timeout']
runtime_timeout = self.config['serverless']['runtime_timeout']
if execution_timeout >= runtime_timeout:
job_description['execution_timeout'] = runtime_timeout - 5
job = SimpleNamespace(**job_description)
try:
while True:
self.token_bucket_q.get_nowait()
self.ongoing_activations -= 1
except Exception:
pass
if self.remote_invoker:
old_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
self.select_runtime(job.job_id, REMOTE_INVOKER_MEMORY)
sys.stdout = old_stdout
log_msg = (
'ExecutorID {} | JobID {} - Starting remote function invocation: {}() '
'- Total: {} activations'.format(job.executor_id, job.job_id,
job.function_name,
job.total_calls))
logger.info(log_msg)
if not self.log_active:
print(log_msg)
th = Thread(target=self._invoke_remote, args=(job_description, ))
th.daemon = True
th.start()
time.sleep(0.1)
else:
try:
if self.running_flag.value == 0:
self.ongoing_activations = 0
self.running_flag.value = 1
self._start_invoker_process()
log_msg = (
'ExecutorID {} | JobID {} - Starting function invocation: {}() - Total: {} '
'activations'.format(job.executor_id, job.job_id,
job.function_name, job.total_calls))
logger.info(log_msg)
if not self.log_active:
print(log_msg)
if self.ongoing_activations < self.workers:
callids = range(job.total_calls)
total_direct = self.workers - self.ongoing_activations
callids_to_invoke_direct = callids[:total_direct]
callids_to_invoke_nondirect = callids[total_direct:]
self.ongoing_activations += len(callids_to_invoke_direct)
logger.debug(
'ExecutorID {} | JobID {} - Free workers: {} - Going to invoke {} function activations'
.format(job.executor_id, job.job_id, total_direct,
len(callids_to_invoke_direct)))
with ThreadPoolExecutor(
max_workers=job.invoke_pool_threads) as executor:
for i in callids_to_invoke_direct:
call_id = "{:05d}".format(i)
executor.submit(self._invoke, job, call_id)
# Put into the queue the rest of the callids to invoke within the process
if callids_to_invoke_nondirect:
logger.debug(
'ExecutorID {} | JobID {} - Putting remaining {} function invocations into pending queue'
.format(job.executor_id, job.job_id,
len(callids_to_invoke_nondirect)))
for i in callids_to_invoke_nondirect:
call_id = "{:05d}".format(i)
self.pending_calls_q.put((job, call_id))
else:
logger.debug(
'ExecutorID {} | JobID {} - Ongoing activations reached {} workers, '
'putting {} function invocations into pending queue'.
format(job.executor_id, job.job_id, self.workers,
job.total_calls))
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
self.pending_calls_q.put((job, call_id))
self.job_monitor.start_job_monitoring(job)
except (KeyboardInterrupt, Exception) as e:
self.stop()
raise e
# Create all futures
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job_description, job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
return futures