def __init__(self, job, jobrunner_conn, internal_storage):
self.jobrunner_conn = jobrunner_conn
self.internal_storage = internal_storage
self.lithops_config = job.config
self.executor_id = job.executor_id
self.call_id = job.call_id
self.job_id = job.job_id
self.func_key = job.func_key
self.data_key = job.data_key
self.data_byte_range = job.data_byte_range
self.output_key = create_output_key(JOBS_PREFIX, self.executor_id,
self.job_id, self.call_id)
# Setup stats class
self.stats = stats(job.jr_stats_file)
# Setup prometheus for live metrics
prom_enabled = self.lithops_config['lithops'].get('monitoring')
prom_config = self.lithops_config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
mode = self.lithops_config['lithops']['mode']
self.customized_runtime = self.lithops_config[mode].get(
'customized_runtime', False)
def __init__(self, config, executor_id, internal_storage, compute_handler,
job_monitor):
log_level = logger.getEffectiveLevel()
self.log_active = log_level != logging.WARNING
self.log_level = LOGGER_LEVEL if not self.log_active else log_level
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_handler = compute_handler
self.is_lithops_worker = is_lithops_worker()
self.job_monitor = job_monitor
self.workers = self.config['lithops'].get('workers')
if self.workers:
logger.debug('ExecutorID {} - Total workers: {}'.format(
self.executor_id, self.workers))
prom_enabled = self.config['lithops'].get('telemetry', False)
prom_config = self.config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
self.mode = self.config['lithops']['mode']
self.backend = self.config['lithops']['backend']
self.runtime_name = self.config[self.backend]['runtime']
self.customized_runtime = self.config[self.mode].get(
'customized_runtime', False)
def __init__(self, job, jobrunner_conn, internal_storage):
self.job = job
self.jobrunner_conn = jobrunner_conn
self.internal_storage = internal_storage
self.lithops_config = job.config
self.output_key = create_output_key(job.executor_id, job.job_id,
job.call_id)
# Setup stats class
self.stats = JobStats(self.job.stats_file)
# Setup prometheus for live metrics
prom_enabled = self.lithops_config['lithops'].get('telemetry')
prom_config = self.lithops_config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
def __init__(self, task, jobrunner_conn, internal_storage):
self.task = task
self.jobrunner_conn = jobrunner_conn
self.internal_storage = internal_storage
self.lithops_config = task.config
self.output_key = create_output_key(JOBS_PREFIX, self.task.executor_id,
self.task.job_id, self.task.id)
# Setup stats class
self.stats = stats(self.task.stats_file)
# Setup prometheus for live metrics
prom_enabled = self.lithops_config['lithops'].get('monitoring')
prom_config = self.lithops_config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
def __init__(self, jr_config, jobrunner_conn, internal_storage):
self.jr_config = jr_config
self.jobrunner_conn = jobrunner_conn
self.internal_storage = internal_storage
self.lithops_config = self.jr_config['lithops_config']
self.call_id = self.jr_config['call_id']
self.job_id = self.jr_config['job_id']
self.func_key = self.jr_config['func_key']
self.data_key = self.jr_config['data_key']
self.data_byte_range = self.jr_config['data_byte_range']
self.output_key = self.jr_config['output_key']
self.stats = stats(self.jr_config['stats_filename'])
prom_enabled = self.lithops_config['lithops'].get('monitoring')
prom_config = self.lithops_config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
class JobRunner:
def __init__(self, job, jobrunner_conn, internal_storage):
self.job = job
self.jobrunner_conn = jobrunner_conn
self.internal_storage = internal_storage
self.lithops_config = job.config
self.output_key = create_output_key(job.executor_id, job.job_id,
job.call_id)
# Setup stats class
self.stats = JobStats(self.job.stats_file)
# Setup prometheus for live metrics
prom_enabled = self.lithops_config['lithops'].get('telemetry')
prom_config = self.lithops_config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
def _fill_optional_args(self, function, data):
"""
Fills in those reserved, optional parameters that might be write to the function signature
"""
func_sig = inspect.signature(function)
if len(data) == 1 and 'future' in data:
# Function chaining feature
out = [
data.pop('future').result(
internal_storage=self.internal_storage)
]
data.update(verify_args(function, out, None)[0])
if 'ibm_cos' in func_sig.parameters:
if 'ibm_cos' in self.lithops_config:
if self.internal_storage.backend == 'ibm_cos':
ibm_boto3_client = self.internal_storage.get_client()
else:
ibm_boto3_client = Storage(config=self.lithops_config,
backend='ibm_cos').get_client()
data['ibm_cos'] = ibm_boto3_client
else:
raise Exception(
'Cannot create the ibm_cos client: missing configuration')
if 'storage' in func_sig.parameters:
data['storage'] = self.internal_storage.storage
if 'rabbitmq' in func_sig.parameters:
if 'rabbitmq' in self.lithops_config:
rabbit_amqp_url = self.lithops_config['rabbitmq'].get(
'amqp_url')
params = pika.URLParameters(rabbit_amqp_url)
connection = pika.BlockingConnection(params)
data['rabbitmq'] = connection
else:
raise Exception(
'Cannot create the rabbitmq client: missing configuration')
if 'id' in func_sig.parameters:
data['id'] = int(self.job.call_id)
def _wait_futures(self, data):
logger.info('Reduce function: waiting for map results')
fut_list = data['results']
wait(fut_list, self.internal_storage, download_results=True)
results = [f.result() for f in fut_list if f.done and not f.futures]
fut_list.clear()
data['results'] = results
def _load_object(self, data):
"""
Loads the object in /tmp in case of object processing
"""
extra_get_args = {}
obj = data['obj']
if hasattr(obj, 'bucket') and not hasattr(obj, 'path'):
logger.info('Getting dataset from {}://{}/{}'.format(
obj.backend, obj.bucket, obj.key))
if obj.backend == self.internal_storage.backend:
storage = self.internal_storage.storage
else:
storage = Storage(config=self.lithops_config,
backend=obj.backend)
if obj.data_byte_range is not None:
extra_get_args['Range'] = 'bytes={}-{}'.format(
*obj.data_byte_range)
logger.info('Chunk: {} - Range: {}'.format(
obj.part, extra_get_args['Range']))
sb = storage.get_object(obj.bucket,
obj.key,
stream=True,
extra_get_args=extra_get_args)
wsb = WrappedStreamingBodyPartition(sb, obj.chunk_size,
obj.data_byte_range)
obj.data_stream = wsb
else:
sb = storage.get_object(obj.bucket,
obj.key,
stream=True,
extra_get_args=extra_get_args)
obj.data_stream = sb
elif hasattr(obj, 'url'):
logger.info('Getting dataset from {}'.format(obj.url))
if obj.data_byte_range is not None:
range_str = 'bytes={}-{}'.format(*obj.data_byte_range)
extra_get_args['Range'] = range_str
logger.info('Chunk: {} - Range: {}'.format(
obj.part, extra_get_args['Range']))
resp = requests.get(obj.url, headers=extra_get_args, stream=True)
obj.data_stream = resp.raw
elif hasattr(obj, 'path'):
logger.info('Getting dataset from {}'.format(obj.path))
with open(obj.path, "rb") as f:
if obj.data_byte_range is not None:
extra_get_args['Range'] = 'bytes={}-{}'.format(
*obj.data_byte_range)
logger.info('Chunk: {} - Range: {}'.format(
obj.part, extra_get_args['Range']))
first_byte, last_byte = obj.data_byte_range
f.seek(first_byte)
buffer = io.BytesIO(f.read(last_byte - first_byte + 1))
sb = WrappedStreamingBodyPartition(buffer, obj.chunk_size,
obj.data_byte_range)
else:
sb = io.BytesIO(f.read())
obj.data_stream = sb
# Decorator to execute pre-run and post-run functions provided via environment variables
def prepost(func):
def call(envVar):
if envVar in os.environ:
method = locate(os.environ[envVar])
method()
def wrapper_decorator(*args, **kwargs):
call('PRE_RUN')
value = func(*args, **kwargs)
call('POST_RUN')
return value
return wrapper_decorator
@prepost
def run(self):
"""
Runs the function
"""
# self.stats.write('worker_jobrunner_start_tstamp', time.time())
logger.debug("Process started")
result = None
exception = False
fn_name = None
try:
func = pickle.loads(self.job.func)
data = pickle.loads(self.job.data)
if strtobool(os.environ.get('__LITHOPS_REDUCE_JOB', 'False')):
self._wait_futures(data)
elif is_object_processing_function(func):
self._load_object(data)
self._fill_optional_args(func, data)
fn_name = func.__name__ if inspect.isfunction(func) \
or inspect.ismethod(func) else type(func).__name__
self.prometheus.send_metric(name='function_start',
value=time.time(),
type='gauge',
labels=(('job_id', self.job.job_key),
('call_id', '-'.join([
self.job.job_key,
self.job.call_id
])), ('function_name', fn_name
or 'undefined')))
logger.info("Going to execute '{}()'".format(str(fn_name)))
print('---------------------- FUNCTION LOG ----------------------')
function_start_tstamp = time.time()
result = func(**data)
function_end_tstamp = time.time()
print('----------------------------------------------------------')
logger.info("Success function execution")
self.stats.write('worker_func_start_tstamp', function_start_tstamp)
self.stats.write('worker_func_end_tstamp', function_end_tstamp)
self.stats.write(
'worker_func_exec_time',
round(function_end_tstamp - function_start_tstamp, 8))
# Check for new futures
if result is not None:
if isinstance(result, ResponseFuture) or isinstance(result, FuturesList) \
or (type(result) == list and len(result) > 0 and isinstance(result[0], ResponseFuture)):
self.stats.write('new_futures', pickle.dumps(result))
result = None
else:
self.stats.write("result", True)
logger.debug("Pickling result")
output_dict = {'result': result}
pickled_output = pickle.dumps(output_dict)
self.stats.write('func_result_size', len(pickled_output))
if result is None:
logger.debug("No result to store")
self.stats.write("result", False)
self.stats.write('func_result_size', 0)
# self.stats.write('worker_jobrunner_end_tstamp', time.time())
except Exception:
exception = True
self.stats.write("exception", True)
exc_type, exc_value, exc_traceback = sys.exc_info()
print('----------------------- EXCEPTION !-----------------------')
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------')
try:
logger.debug("Pickling exception")
pickled_exc = pickle.dumps(
(exc_type, exc_value, exc_traceback))
pickle.loads(
pickled_exc
) # this is just to make sure they can be unpickled
self.stats.write("exc_info", str(pickled_exc))
except Exception as pickle_exception:
# Shockingly often, modules like subprocess don't properly
# call the base Exception.__init__, which results in them
# being unpickleable. As a result, we actually wrap this in a try/catch block
# and more-carefully handle the exceptions if any part of this save / test-reload
# fails
self.stats.write("exc_pickle_fail", True)
pickled_exc = pickle.dumps({
'exc_type': str(exc_type),
'exc_value': str(exc_value),
'exc_traceback': exc_traceback,
'pickle_exception': pickle_exception
})
pickle.loads(pickled_exc
) # this is just to make sure it can be unpickled
self.stats.write("exc_info", str(pickled_exc))
finally:
self.prometheus.send_metric(name='function_end',
value=time.time(),
type='gauge',
labels=(('job_id', self.job.job_key),
('call_id', '-'.join([
self.job.job_key,
self.job.call_id
])), ('function_name', fn_name
or 'undefined')))
store_result = strtobool(os.environ.get('STORE_RESULT', 'True'))
if result is not None and store_result and not exception:
output_upload_start_tstamp = time.time()
logger.info("Storing function result - Size: {}".format(
sizeof_fmt(len(pickled_output))))
self.internal_storage.put_data(self.output_key, pickled_output)
output_upload_end_tstamp = time.time()
self.stats.write(
"worker_result_upload_time",
round(
output_upload_end_tstamp - output_upload_start_tstamp,
8))
self.jobrunner_conn.send("Finished")
logger.info("Process finished")
class JobRunner:
def __init__(self, job, jobrunner_conn, internal_storage):
self.jobrunner_conn = jobrunner_conn
self.internal_storage = internal_storage
self.lithops_config = job.config
self.executor_id = job.executor_id
self.call_id = job.call_id
self.job_id = job.job_id
self.func_key = job.func_key
self.data_key = job.data_key
self.data_byte_range = job.data_byte_range
self.output_key = create_output_key(JOBS_PREFIX, self.executor_id,
self.job_id, self.call_id)
# Setup stats class
self.stats = stats(job.jr_stats_file)
# Setup prometheus for live metrics
prom_enabled = self.lithops_config['lithops'].get('monitoring')
prom_config = self.lithops_config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
mode = self.lithops_config['lithops']['mode']
self.customized_runtime = self.lithops_config[mode].get(
'customized_runtime', False)
def _get_function_and_modules(self):
"""
Gets and unpickles function and modules from storage
"""
logger.debug("Getting function and modules")
func_download_start_tstamp = time.time()
func_obj = None
if self.customized_runtime:
func_obj = self._get_func()
else:
func_obj = self.internal_storage.get_func(self.func_key)
loaded_func_all = pickle.loads(func_obj)
func_download_end_tstamp = time.time()
self.stats.write(
'worker_func_download_time',
round(func_download_end_tstamp - func_download_start_tstamp, 8))
return loaded_func_all
def _get_func(self):
func_path = '/'.join([LITHOPS_TEMP_DIR, self.func_key])
with open(func_path, "rb") as f:
return f.read()
def _save_modules(self, module_data):
"""
Save modules, before we unpickle actual function
"""
if module_data:
logger.debug("Writing Function dependencies to local disk")
module_path = os.path.join(PYTHON_MODULE_PATH, self.executor_id,
self.job_id, self.call_id)
# shutil.rmtree(PYTHON_MODULE_PATH, True) # delete old modules
os.makedirs(module_path, exist_ok=True)
sys.path.append(module_path)
for m_filename, m_data in module_data.items():
m_path = os.path.dirname(m_filename)
if len(m_path) > 0 and m_path[0] == "/":
m_path = m_path[1:]
to_make = os.path.join(module_path, m_path)
try:
os.makedirs(to_make)
except OSError as e:
if e.errno == 17:
pass
else:
raise e
full_filename = os.path.join(to_make,
os.path.basename(m_filename))
with open(full_filename, 'wb') as fid:
fid.write(b64str_to_bytes(m_data))
def _unpickle_function(self, pickled_func):
"""
Unpickle function; it will expect modules to be there
"""
return pickle.loads(pickled_func)
def _load_data(self):
"""
Loads iteradata
"""
extra_get_args = {}
if self.data_byte_range is not None:
range_str = 'bytes={}-{}'.format(*self.data_byte_range)
extra_get_args['Range'] = range_str
logger.debug("Getting function data")
data_download_start_tstamp = time.time()
data_obj = self.internal_storage.get_data(
self.data_key, extra_get_args=extra_get_args)
loaded_data = pickle.loads(data_obj)
data_download_end_tstamp = time.time()
self.stats.write(
'worker_data_download_time',
round(data_download_end_tstamp - data_download_start_tstamp, 8))
return loaded_data
def _fill_optional_args(self, function, data):
"""
Fills in those reserved, optional parameters that might be write to the function signature
"""
func_sig = inspect.signature(function)
if 'ibm_cos' in func_sig.parameters:
if 'ibm_cos' in self.lithops_config:
if self.internal_storage.backend == 'ibm_cos':
ibm_boto3_client = self.internal_storage.get_client()
else:
ibm_boto3_client = Storage(config=self.lithops_config,
backend='ibm_cos').get_client()
data['ibm_cos'] = ibm_boto3_client
else:
raise Exception(
'Cannot create the ibm_cos client: missing configuration')
if 'storage' in func_sig.parameters:
data['storage'] = self.internal_storage.storage
if 'rabbitmq' in func_sig.parameters:
if 'rabbitmq' in self.lithops_config:
rabbit_amqp_url = self.lithops_config['rabbitmq'].get(
'amqp_url')
params = pika.URLParameters(rabbit_amqp_url)
connection = pika.BlockingConnection(params)
data['rabbitmq'] = connection
else:
raise Exception(
'Cannot create the rabbitmq client: missing configuration')
if 'id' in func_sig.parameters:
data['id'] = int(self.call_id)
def _wait_futures(self, data):
logger.info('Reduce function: waiting for map results')
fut_list = data['results']
wait_storage(fut_list, self.internal_storage, download_results=True)
results = [f.result() for f in fut_list if f.done and not f.futures]
fut_list.clear()
data['results'] = results
def _load_object(self, data):
"""
Loads the object in /tmp in case of object processing
"""
extra_get_args = {}
if 'url' in data:
url = data['url']
logger.info('Getting dataset from {}'.format(url.path))
if url.data_byte_range is not None:
range_str = 'bytes={}-{}'.format(*url.data_byte_range)
extra_get_args['Range'] = range_str
logger.info('Chunk: {} - Range: {}'.format(
url.part, extra_get_args['Range']))
resp = requests.get(url.path, headers=extra_get_args, stream=True)
url.data_stream = resp.raw
if 'obj' in data:
obj = data['obj']
logger.info('Getting dataset from {}://{}/{}'.format(
obj.backend, obj.bucket, obj.key))
if obj.backend == self.internal_storage.backend:
storage = self.internal_storage.storage
else:
storage = Storage(config=self.lithops_config,
backend=obj.backend)
if obj.data_byte_range is not None:
extra_get_args['Range'] = 'bytes={}-{}'.format(
*obj.data_byte_range)
logger.info('Chunk: {} - Range: {}'.format(
obj.part, extra_get_args['Range']))
sb = storage.get_object(obj.bucket,
obj.key,
stream=True,
extra_get_args=extra_get_args)
wsb = WrappedStreamingBodyPartition(sb, obj.chunk_size,
obj.data_byte_range)
obj.data_stream = wsb
else:
sb = storage.get_object(obj.bucket,
obj.key,
stream=True,
extra_get_args=extra_get_args)
obj.data_stream = sb
# Decorator to execute pre-run and post-run functions provided via environment variables
def prepost(func):
def call(envVar):
if envVar in os.environ:
method = locate(os.environ[envVar])
method()
def wrapper_decorator(*args, **kwargs):
call('PRE_RUN')
value = func(*args, **kwargs)
call('POST_RUN')
return value
return wrapper_decorator
@prepost
def run(self):
"""
Runs the function
"""
# self.stats.write('worker_jobrunner_start_tstamp', time.time())
logger.debug("Process started")
result = None
exception = False
try:
function = None
if self.customized_runtime:
function = self._get_function_and_modules()
else:
loaded_func_all = self._get_function_and_modules()
self._save_modules(loaded_func_all['module_data'])
function = self._unpickle_function(loaded_func_all['func'])
data = self._load_data()
if strtobool(os.environ.get('__LITHOPS_REDUCE_JOB', 'False')):
self._wait_futures(data)
elif is_object_processing_function(function):
self._load_object(data)
self._fill_optional_args(function, data)
self.prometheus.send_metric(name='function_start',
value=time.time(),
labels=(('job_id', self.job_id),
('call_id', self.call_id),
('function_name',
function.__name__)))
logger.info("Going to execute '{}()'".format(str(
function.__name__)))
print('---------------------- FUNCTION LOG ----------------------')
function_start_tstamp = time.time()
result = function(**data)
function_end_tstamp = time.time()
print('----------------------------------------------------------')
logger.info("Success function execution")
self.prometheus.send_metric(name='function_end',
value=time.time(),
labels=(('job_id', self.job_id),
('call_id', self.call_id),
('function_name',
function.__name__)))
self.stats.write('worker_func_start_tstamp', function_start_tstamp)
self.stats.write('worker_func_end_tstamp', function_end_tstamp)
self.stats.write(
'worker_func_exec_time',
round(function_end_tstamp - function_start_tstamp, 8))
# Check for new futures
if result is not None:
self.stats.write("result", True)
if isinstance(result, ResponseFuture) or \
(type(result) == list and len(result) > 0 and isinstance(result[0], ResponseFuture)):
self.stats.write('new_futures', True)
logger.debug("Pickling result")
output_dict = {'result': result}
pickled_output = pickle.dumps(output_dict)
else:
logger.debug("No result to store")
self.stats.write("result", False)
# self.stats.write('worker_jobrunner_end_tstamp', time.time())
except Exception:
exception = True
self.stats.write("exception", True)
exc_type, exc_value, exc_traceback = sys.exc_info()
print('----------------------- EXCEPTION !-----------------------')
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------')
try:
logger.debug("Pickling exception")
pickled_exc = pickle.dumps(
(exc_type, exc_value, exc_traceback))
pickle.loads(
pickled_exc
) # this is just to make sure they can be unpickled
self.stats.write("exc_info", str(pickled_exc))
except Exception as pickle_exception:
# Shockingly often, modules like subprocess don't properly
# call the base Exception.__init__, which results in them
# being unpickleable. As a result, we actually wrap this in a try/catch block
# and more-carefully handle the exceptions if any part of this save / test-reload
# fails
self.stats.write("exc_pickle_fail", True)
pickled_exc = pickle.dumps({
'exc_type': str(exc_type),
'exc_value': str(exc_value),
'exc_traceback': exc_traceback,
'pickle_exception': pickle_exception
})
pickle.loads(pickled_exc
) # this is just to make sure it can be unpickled
self.stats.write("exc_info", str(pickled_exc))
finally:
store_result = strtobool(os.environ.get('STORE_RESULT', 'True'))
if result is not None and store_result and not exception:
output_upload_start_tstamp = time.time()
logger.info("Storing function result - Size: {}".format(
sizeof_fmt(len(pickled_output))))
self.internal_storage.put_data(self.output_key, pickled_output)
output_upload_end_tstamp = time.time()
self.stats.write(
"worker_result_upload_time",
round(
output_upload_end_tstamp - output_upload_start_tstamp,
8))
self.jobrunner_conn.send("Finished")
logger.info("Process finished")
class Invoker:
"""
Abstract invoker class
"""
def __init__(self, config, executor_id, internal_storage, compute_handler,
job_monitor):
log_level = logger.getEffectiveLevel()
self.log_active = log_level != logging.WARNING
self.log_level = LOGGER_LEVEL if not self.log_active else log_level
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_handler = compute_handler
self.is_lithops_worker = is_lithops_worker()
self.job_monitor = job_monitor
self.workers = self.config['lithops'].get('workers')
if self.workers:
logger.debug('ExecutorID {} - Total workers: {}'.format(
self.executor_id, self.workers))
prom_enabled = self.config['lithops'].get('telemetry', False)
prom_config = self.config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
self.mode = self.config['lithops']['mode']
self.backend = self.config['lithops']['backend']
self.runtime_name = self.config[self.backend]['runtime']
self.customized_runtime = self.config[self.mode].get(
'customized_runtime', False)
def select_runtime(self, job_id, runtime_memory):
"""
Return the runtime metadata
"""
if self.mode == SERVERLESS:
runtime_memory = runtime_memory or self.config[self.backend].get(
'runtime_memory')
runtime_timeout = self.config[self.backend].get('runtime_timeout')
elif self.mode == STANDALONE:
runtime_memory = None
runtime_timeout = self.config[STANDALONE]['hard_dismantle_timeout']
elif self.mode == LOCALHOST:
runtime_memory = None
runtime_timeout = None
msg = ('ExecutorID {} | JobID {} - Selected Runtime: {} '.format(
self.executor_id, job_id, self.runtime_name))
msg = msg + '- {}MB'.format(runtime_memory) if runtime_memory else msg
logger.info(msg)
runtime_key = self.compute_handler.get_runtime_key(
self.runtime_name, runtime_memory)
runtime_meta = self.internal_storage.get_runtime_meta(runtime_key)
if not runtime_meta:
msg = 'Runtime {}'.format(self.runtime_name)
msg = msg + ' with {}MB'.format(
runtime_memory) if runtime_memory else msg
logger.info(msg + ' is not yet installed')
runtime_meta = self.compute_handler.create_runtime(
self.runtime_name, runtime_memory, runtime_timeout)
runtime_meta['runtime_timeout'] = runtime_timeout
self.internal_storage.put_runtime_meta(runtime_key, runtime_meta)
# Verify python version and lithops version
if lithops_version != runtime_meta['lithops_version']:
raise Exception(
"Lithops version mismatch. Host version: {} - Runtime version: {}"
.format(lithops_version, runtime_meta['lithops_version']))
py_local_version = version_str(sys.version_info)
py_remote_version = runtime_meta['python_version']
if py_local_version != py_remote_version:
raise Exception(
("The indicated runtime '{}' is running Python {} and it "
"is not compatible with the local Python version {}").format(
self.runtime_name, py_remote_version, py_local_version))
return runtime_meta
def _create_payload(self, job):
"""
Creates the default pyload dictionary
"""
payload = {
'config': self.config,
'chunksize': job.chunksize,
'log_level': self.log_level,
'func_key': job.func_key,
'data_key': job.data_key,
'extra_env': job.extra_env,
'total_calls': job.total_calls,
'execution_timeout': job.execution_timeout,
'data_byte_ranges': job.data_byte_ranges,
'executor_id': job.executor_id,
'job_id': job.job_id,
'job_key': job.job_key,
'workers': self.workers,
'call_ids': None,
'host_submit_tstamp': time.time(),
'lithops_version': lithops_version,
'runtime_name': job.runtime_name,
'runtime_memory': job.runtime_memory,
'worker_processes': job.worker_processes
}
return payload
def _run_job(self, job):
"""
Run a job
"""
if self.customized_runtime:
logger.debug(
'ExecutorID {} | JobID {} - Customized runtime activated'.
format(job.executor_id, job.job_id))
job.runtime_name = self.runtime_name
extend_runtime(job, self.compute_handler, self.internal_storage)
self.runtime_name = job.runtime_name
logger.info('ExecutorID {} | JobID {} - Starting function '
'invocation: {}() - Total: {} activations'.format(
job.executor_id, job.job_id, job.function_name,
job.total_calls))
logger.debug('ExecutorID {} | JobID {} - Chunksize:'
' {} - Worker processes: {}'.format(
job.executor_id, job.job_id, job.chunksize,
job.worker_processes))
self.prometheus.send_metric(name='job_total_calls',
value=job.total_calls,
type='counter',
labels=(('job_id_', job.job_key),
('function_name',
job.function_name)))
self.prometheus.send_metric(name='job_runtime_memory',
value=job.runtime_memory or 0,
type='counter',
labels=(('job_id_', job.job_key),
('function_name',
job.function_name)))
try:
job.runtime_name = self.runtime_name
self._invoke_job(job)
except (KeyboardInterrupt, Exception) as e:
self.stop()
raise e
log_file = os.path.join(LOGS_DIR, job.job_key + '.log')
logger.info(
"ExecutorID {} | JobID {} - View execution logs at {}".format(
job.executor_id, job.job_id, log_file))
# Create all futures
futures = []
for i in range(job.total_calls):
call_id = "{:05d}".format(i)
fut = ResponseFuture(call_id, job, job.metadata.copy(),
self.storage_config)
fut._set_state(ResponseFuture.State.Invoked)
futures.append(fut)
job.futures = futures
return futures
def stop(self):
"""
Stop invoker-related processes
"""
pass