def update(name, config, backend):
""" Update a serverless runtime """
verify_runtime_name(name)
setup_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1],
timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise (
"Unable to upload 'preinstalled-modules' file into {}".format(
internal_storage.backend))
def clean(mode, config, debug):
log_level = 'INFO' if not debug else 'DEBUG'
setup_logger(log_level)
logger.info('Cleaning all Lithops information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def create(name, backend, memory, timeout, config):
""" Create a serverless runtime """
setup_logger(logging.DEBUG)
logger.info('Creating new lithops runtime: {}'.format(name))
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise ("Unable to upload 'preinstalled-modules' file into {}".format(
internal_storage.backend))
def main_job(action, encoded_payload):
logger.info(
"Lithops v{} - Starting Code Engine execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_logger(payload['log_level'])
if (action == 'preinstalls'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = os.environ['JOB_INDEX']
payload['JOB_INDEX'] = job_index
logger.info("Action {}. Job Index {}".format(action, job_index))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
payload['data_byte_range'] = payload['data_byte_range'][int(job_index)]
payload['call_id'] = "{:05d}".format(int(job_index))
function_handler(payload)
return {"Execution": "Finished"}
def main(action, payload_decoded):
logger.info(
"Welcome to Lithops-Code-Engine entry point. Action {}".format(action))
payload = binary_to_dict(payload_decoded)
setup_logger(payload['log_level'])
logger.info(payload)
if (action == 'preinstals'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = os.environ['JOB_INDEX']
logger.info("Action {}. Job Index {}".format(action, job_index))
os.environ['__PW_ACTIVATION_ID'] = payload['activation_id']
payload['JOB_INDEX'] = job_index
if 'remote_invoker' in payload:
logger.info("Lithops v{} - Remote Invoker. Starting execution".format(
__version__))
#function_invoker(payload)
payload['data_byte_range'] = payload['job_description']['data_ranges'][
int(job_index)]
for key in payload['job_description']:
payload[key] = payload['job_description'][key]
payload['host_submit_tstamp'] = payload['metadata'][
'host_job_create_tstamp']
payload['call_id'] = "{:05d}".format(int(job_index))
function_handler(payload)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(payload)
return {"Execution": "Finished"}
def create(name, mode, memory, timeout, config):
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
runtime_key = compute_handler.get_runtime_key(name)
runtime_meta = compute_handler.create_runtime(name)
elif mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
runtime_key = compute_handler.get_runtime_key(name)
runtime_meta = compute_handler.create_runtime(name)
else:
raise Exception('Unknown execution mode {}'.format(mode))
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def run():
def error():
response = flask.jsonify({
'error':
'The action did not receive a dictionary as an argument.'
})
response.status_code = 404
return complete(response)
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
setup_logger(message['log_level'])
if 'remote_invoker' in message:
logger.info(
"Lithops v{} - Starting Knative invoker".format(__version__))
function_invoker(message)
else:
logger.info(
"Lithops v{} - Starting Knative execution".format(__version__))
function_handler(message)
response = flask.jsonify({"activationId": act_id})
response.status_code = 202
return complete(response)
def _process_runner(self, worker_id):
logger.debug('Localhost worker process {} started'.format(worker_id))
os.environ['__LITHOPS_LOCAL_EXECUTION'] = 'True'
p_logger = logging.getLogger('lithops')
while True:
with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr(buf):
event = self.queue.get(block=True)
if isinstance(event, ShutdownSentinel):
break
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
executor_id = event['executor_id']
job_id = event['job_id']
setup_logger(event['log_level'])
p_logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(event)
log_output = buf.getvalue()
job_key = create_job_key(executor_id, job_id)
log_file = os.path.join(LOGS_DIR, job_key+'.log')
header = "Activation: '{}' ({})\n[\n".format(event['runtime_name'], act_id)
tail = ']\n\n'
output = log_output.replace('\n', '\n ', log_output.count('\n')-1)
with open(log_file, 'a') as lf:
lf.write(header+' '+output+tail)
with open(FN_LOG_FILE, 'a') as lf:
lf.write(header+' '+output+tail)
def build(name, file, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def verify(test, config, mode, debug):
if debug:
setup_logger(logging.DEBUG)
if test == 'help':
print_help()
else:
run_tests(test, mode, config)
def preinstalls_task():
setup_logger(logging.INFO)
logger.info("Lithops v{} - Generating metadata".format(__version__))
runtime_meta = get_runtime_preinstalls()
response = flask.jsonify(runtime_meta)
response.status_code = 200
logger.info("Done!")
return complete(response)
def main(event, context):
args = json.loads(event)
os.environ['__LITHOPS_ACTIVATION_ID'] = context.request_id
setup_logger(args['log_level'])
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def main(event, context):
logger.info("Starting GCP Functions function execution")
args = json.loads(base64.b64decode(event['data']).decode('utf-8'))
os.environ['__PW_ACTIVATION_ID'] = uuid.uuid4().hex
setup_logger(args['log_level'])
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def main(args):
os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID']
setup_logger(args['log_level'], sys.stdout, LOGGER_FORMAT_SHORT)
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def lambda_handler(event, context):
os.environ['__PW_ACTIVATION_ID'] = context.aws_request_id
os.environ['__OW_ACTIVATION_ID'] = context.aws_request_id
setup_logger(event['log_level'])
if 'remote_invoker' in event:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(event)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(event)
return {"Execution": "Finished"}
def delete(name, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def build(name, file, config, backend):
""" build a serverless runtime. """
verify_runtime_name(name)
setup_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def main(args):
os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID']
setup_logger(args['log_level'], sys.stdout, LOGGER_FORMAT_SHORT)
if 'get_preinstalls' in args:
logger.info("Lithops v{} - Generating metadata".format(__version__))
return get_runtime_preinstalls()
elif 'remote_invoker' in args:
logger.info("Lithops v{} - Starting IBM CF invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting IBM CF execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def main(msgIn: func.QueueMessage):
try:
args = json.loads(msgIn.get_body())
except Exception:
args = msgIn.get_json()
os.environ['__PW_ACTIVATION_ID'] = str(msgIn.id)
setup_logger(args['log_level'])
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def test_function(config, debug):
if debug:
setup_logger(logging.DEBUG)
def hello(name):
return 'Hello {}!'.format(name)
fexec = lithops.FunctionExecutor(config=config)
fexec.call_async(hello, 'World')
result = fexec.get_result()
print()
if result == 'Hello World!':
print(result, 'Lithops is working as expected :)')
else:
print(result, 'Something went wrong :(')
print()
def main(msgIn: func.QueueMessage, msgOut: func.Out[func.QueueMessage]):
try:
args = json.loads(msgIn.get_body())
except Exception:
args = msgIn.get_json()
os.environ['__LITHOPS_ACTIVATION_ID'] = str(msgIn.id)
setup_logger(args['log_level'])
if 'get_preinstalls' in args:
logger.info("Lithops v{} - Generating metadata".format(__version__))
runtime_meta = get_runtime_preinstalls()
msgOut.set(json.dumps(runtime_meta))
elif 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
def delete(name, config, backend):
""" delete a serverless runtime """
verify_runtime_name(name)
setup_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def update(name, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
timeout = config['lithops']['runtime_timeout']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
# from multiprocessing import Pool
from lithops.multiprocessing import Pool
from lithops.utils import setup_logger
import time
import logging
import os
setup_logger(logging.CRITICAL)
def hello(name):
return 'Hello {}!'.format(name)
def square(x):
return x * x
def divide(x, y):
return x / y
def sleep_seconds(s):
time.sleep(s)
if __name__ == '__main__':
with Pool() as pool:
# Synchronously execute function square remotely
def __init__(self, type=None, mode=None, config=None, backend=None, storage=None,
runtime=None, runtime_memory=None, rabbitmq_monitor=None,
workers=None, remote_invoker=None, log_level=None):
mode = mode or type
if mode is None:
config = default_config(copy.deepcopy(config))
mode = config['lithops']['mode']
if mode not in [LOCALHOST, SERVERLESS, STANDALONE]:
raise Exception("Function executor mode must be one of '{}', '{}' "
"or '{}'".format(LOCALHOST, SERVERLESS, STANDALONE))
if log_level:
setup_logger(log_level)
if type is not None:
logger.warning("'type' parameter is deprecated and it will be removed"
"in future releases. Use 'mode' parameter instead")
config_ow = {'lithops': {'mode': mode}, mode: {}}
if runtime is not None:
config_ow[mode]['runtime'] = runtime
if backend is not None:
config_ow[mode]['backend'] = backend
if runtime_memory is not None:
config_ow[mode]['runtime_memory'] = int(runtime_memory)
if remote_invoker is not None:
config_ow[mode]['remote_invoker'] = remote_invoker
if storage is not None:
config_ow['lithops']['storage'] = storage
if workers is not None:
config_ow['lithops']['workers'] = workers
if rabbitmq_monitor is not None:
config_ow['lithops']['rabbitmq_monitor'] = rabbitmq_monitor
self.config = default_config(copy.deepcopy(config), config_ow)
self.log_active = logger.getEffectiveLevel() != logging.WARNING
self.is_lithops_worker = is_lithops_worker()
self.executor_id = create_executor_id()
self.data_cleaner = self.config['lithops'].get('data_cleaner', True)
if self.data_cleaner and not self.is_lithops_worker:
spawn_cleaner = int(self.executor_id.split('-')[1]) == 0
atexit.register(self.clean, spawn_cleaner=spawn_cleaner,
clean_cloudobjects=False)
self.rabbitmq_monitor = self.config['lithops'].get('rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception("You cannot use rabbitmq_mnonitor since "
"'amqp_url' is not present in configuration")
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.storage = self.internal_storage.storage
self.futures = []
self.cleaned_jobs = set()
self.total_jobs = 0
self.last_call = None
if mode == LOCALHOST:
localhost_config = extract_localhost_config(self.config)
self.compute_handler = LocalhostHandler(localhost_config)
self.invoker = StandaloneInvoker(self.config,
self.executor_id,
self.internal_storage,
self.compute_handler)
elif mode == SERVERLESS:
serverless_config = extract_serverless_config(self.config)
self.compute_handler = ServerlessHandler(serverless_config,
storage_config)
self.invoker = ServerlessInvoker(self.config,
self.executor_id,
self.internal_storage,
self.compute_handler)
elif mode == STANDALONE:
standalone_config = extract_standalone_config(self.config)
self.compute_handler = StandaloneHandler(standalone_config)
self.invoker = StandaloneInvoker(self.config,
self.executor_id,
self.internal_storage,
self.compute_handler)
logger.info('{} Executor created with ID: {}'
.format(mode.capitalize(), self.executor_id))
