def main(action, payload_decoded):
logger.info(
"Welcome to Lithops-Code-Engine entry point. Action {}".format(action))
payload = binary_to_dict(payload_decoded)
log_level = payload['log_level']
default_logging_config(log_level)
logger.info(payload)
if (action == 'preinstals'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = os.environ['JOB_INDEX']
logger.info("Action {}. Job Index {}".format(action, job_index))
os.environ['__PW_ACTIVATION_ID'] = payload['activation_id']
payload['JOB_INDEX'] = job_index
if 'remote_invoker' in payload:
logger.info("Lithops v{} - Remote Invoker. Starting execution".format(
__version__))
#function_invoker(payload)
payload['data_byte_range'] = payload['job_description']['data_ranges'][
int(job_index)]
for key in payload['job_description']:
payload[key] = payload['job_description'][key]
payload['host_submit_tstamp'] = payload['metadata'][
'host_job_create_tstamp']
payload['call_id'] = "{:05d}".format(int(job_index))
function_handler(payload)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(payload)
return {"Execution": "Finished"}
def _process_runner(self, worker_id):
logger.debug('Localhost worker process {} started'.format(worker_id))
while True:
with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr(
buf):
try:
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
event = self.queue.get(block=True)
if isinstance(event, ShutdownSentinel):
break
log_level = event['log_level']
default_logging_config(log_level)
logger.info(
"Lithops v{} - Starting execution".format(__version__))
event['extra_env']['__LITHOPS_LOCAL_EXECUTION'] = 'True'
function_handler(event)
except KeyboardInterrupt:
break
header = "Activation: '{}' ({})\n[\n".format(
event['runtime_name'], act_id)
tail = ']\n\n'
output = buf.getvalue()
output = output.replace('\n', '\n ', output.count('\n') - 1)
with open(FN_LOG_FILE, 'a') as lf:
lf.write(header + ' ' + output + tail)
def run():
def error():
response = flask.jsonify({
'error':
'The action did not receive a dictionary as an argument.'
})
response.status_code = 404
return complete(response)
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
log_level = message['log_level']
default_logging_config(log_level)
if 'remote_invoker' in message:
logger.info(
"Lithops v{} - Starting Knative invoker".format(__version__))
function_invoker(message)
else:
logger.info(
"Lithops v{} - Starting Knative execution".format(__version__))
function_handler(message)
response = flask.jsonify({"activationId": act_id})
response.status_code = 202
return complete(response)
def lambda_handler(event, context):
os.environ['__PW_ACTIVATION_ID'] = context.aws_request_id
os.environ['__OW_ACTIVATION_ID'] = context.aws_request_id
log_level = event['log_level']
default_logging_config(log_level)
if 'remote_invoker' in event:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(event)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(event)
return {"Execution": "Finished"}
def __init__(self,
type=None,
config=None,
backend=None,
storage=None,
runtime=None,
runtime_memory=None,
rabbitmq_monitor=None,
workers=None,
remote_invoker=None,
log_level=None):
if type is None:
config = default_config(copy.deepcopy(config))
type = config['lithops']['executor']
if log_level:
default_logging_config(log_level)
config_ow = {'lithops': {'executor': type}, type: {}}
if runtime is not None:
config_ow[type]['runtime'] = runtime
if backend is not None:
config_ow[type]['backend'] = backend
if runtime_memory is not None:
config_ow[type]['runtime_memory'] = int(runtime_memory)
if remote_invoker is not None:
config_ow[type]['remote_invoker'] = remote_invoker
if storage is not None:
config_ow['lithops']['storage'] = storage
if workers is not None:
config_ow['lithops']['workers'] = workers
if rabbitmq_monitor is not None:
config_ow['lithops']['rabbitmq_monitor'] = rabbitmq_monitor
self.config = default_config(copy.deepcopy(config), config_ow)
self.log_active = logger.getEffectiveLevel() != logging.WARNING
self.is_lithops_worker = is_lithops_worker()
self.executor_id = create_executor_id()
self.data_cleaner = self.config['lithops'].get('data_cleaner', True)
self.rabbitmq_monitor = self.config['lithops'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config[
'rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception("You cannot use rabbitmq_mnonitor since "
"'amqp_url' is not present in configuration")
self.storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(self.storage_config)
self.storage = self.internal_storage.storage
self.futures = []
self.total_jobs = 0
self.cleaned_jobs = set()
self.last_call = None
if type == 'localhost':
localhost_config = extract_localhost_config(self.config)
self.compute_handler = LocalhostHandler(localhost_config)
self.invoker = StandaloneInvoker(self.config, self.executor_id,
self.internal_storage,
self.compute_handler)
elif type == 'serverless':
serverless_config = extract_serverless_config(self.config)
self.compute_handler = ServerlessHandler(serverless_config,
self.storage_config)
self.invoker = ServerlessInvoker(self.config, self.executor_id,
self.internal_storage,
self.compute_handler)
elif type == 'standalone':
standalone_config = extract_standalone_config(self.config)
self.compute_handler = StandaloneHandler(standalone_config)
self.invoker = StandaloneInvoker(self.config, self.executor_id,
self.internal_storage,
self.compute_handler)
else:
raise Exception("Function executor type must be one of "
"'localhost', 'serverless' or 'standalone'")
logger.info('{} Executor created with ID: {}'.format(
type.capitalize(), self.executor_id))
def __init__(self, config=None, runtime=None, runtime_memory=None, compute_backend=None,
compute_backend_region=None, storage_backend=None, storage_backend_region=None,
workers=None, rabbitmq_monitor=None, remote_invoker=None, log_level=None):
"""
Initialize a FunctionExecutor class.
:param config: Settings passed in here will override those in config file. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime. Default None.
:param compute_backend: Name of the compute backend to use. Default None.
:param compute_backend_region: Name of the compute backend region to use. Default None.
:param storage_backend: Name of the storage backend to use. Default None.
:param storage_backend_region: Name of the storage backend region to use. Default None.
:param workers: Max number of concurrent workers.
:param rabbitmq_monitor: use rabbitmq as the monitoring system. Default None.
:param log_level: log level to use during the execution. Default None.
:return `FunctionExecutor` object.
"""
self.is_lithops_function = is_lithops_function()
if log_level:
default_logging_config(log_level)
self.log_active = logger.getEffectiveLevel() != logging.WARNING
# Overwrite lithops config parameters
pw_config_ow = {}
if runtime is not None:
pw_config_ow['runtime'] = runtime
if runtime_memory is not None:
pw_config_ow['runtime_memory'] = int(runtime_memory)
if compute_backend is not None:
pw_config_ow['compute_backend'] = compute_backend
if compute_backend_region is not None:
pw_config_ow['compute_backend_region'] = compute_backend_region
if storage_backend is not None:
pw_config_ow['storage_backend'] = storage_backend
if storage_backend_region is not None:
pw_config_ow['storage_backend_region'] = storage_backend_region
if workers is not None:
pw_config_ow['workers'] = workers
if rabbitmq_monitor is not None:
pw_config_ow['rabbitmq_monitor'] = rabbitmq_monitor
if remote_invoker is not None:
pw_config_ow['remote_invoker'] = remote_invoker
self.config = default_config(copy.deepcopy(config), pw_config_ow)
self.executor_id = create_executor_id()
logger.debug('FunctionExecutor created with ID: {}'.format(self.executor_id))
self.data_cleaner = self.config['lithops'].get('data_cleaner', True)
self.auto_dismantle = self.config['lithops'].get('auto_dismantle', True)
self.rabbitmq_monitor = self.config['lithops'].get('rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception("You cannot use rabbitmq_mnonitor since 'amqp_url'"
" is not present in configuration")
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.storage = self.internal_storage.storage
self.invoker = FunctionInvoker(self.config, self.executor_id, self.internal_storage)
self.futures = []
self.total_jobs = 0
self.cleaned_jobs = set()
self.last_call = None
