def delete_object(bucket, key, backend, debug):
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(backend=backend)
logger.info('Deleting object "{}" from bucket "{}"'.format(key, bucket))
storage.delete_object(bucket, key)
logger.info('Object deleted successfully')
def main(event, context):
# pub/sub event data is b64 encoded
args = json.loads(base64.b64decode(event['data']).decode('utf-8'))
setup_lithops_logger(args.get('log_level', 'INFO'))
os.environ['__LITHOPS_ACTIVATION_ID'] = uuid.uuid4().hex
os.environ['__LITHOPS_BACKEND'] = 'Google Cloud Functions'
if 'get_preinstalls' in args:
logger.info("Lithops v{} - Generating metadata".format(__version__))
internal_storage = InternalStorage(
args['get_preinstalls']['storage_config'])
object_key = '/'.join(
[JOBS_PREFIX, args['get_preinstalls']['runtime_name'] + '.meta'])
logger.info("Runtime metadata key {}".format(object_key))
runtime_meta = get_runtime_preinstalls()
runtime_meta_json = json.dumps(runtime_meta)
internal_storage.put_data(object_key, runtime_meta_json)
elif 'remote_invoker' in args:
logger.info(
"Lithops v{} - Starting Google Cloud Functions invoker".format(
__version__))
function_invoker(args)
else:
logger.info(
"Lithops v{} - Starting Google Cloud Functions execution".format(
__version__))
function_handler(args)
return {"Execution": "Finished"}
def create(name, storage, backend, memory, timeout, config):
""" Create a serverless runtime """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
logger.info('Creating new lithops runtime: {}'.format(name))
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise ("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def download_file(bucket, key, out, backend, debug, config):
if config:
config = load_yaml_config(config)
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(config=config, backend=backend)
def download_file():
logger.info(
f'Downloading file {storage.backend}://{bucket}/{key} to {out or key}'
)
if storage.download_file(bucket, key, out):
file_size = os.path.getsize(out or key)
logger.info(
f'Download File {key} - Size: {sizeof_fmt(file_size)} - Ok')
else:
logger.error(f'Download File {key} - Error')
with ThreadPoolExecutor() as ex:
future = ex.submit(download_file)
cy = cycle(r"-\|/")
while not future.done():
print("Downloading file " + next(cy), end="\r")
time.sleep(0.1)
future.result()
def run_job(encoded_payload):
logger.info(
"Lithops v{} - Starting kubernetes execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
job_key = payload['job_key']
idgiver_ip = os.environ['IDGIVER_POD_IP']
res = requests.get('http://{}:{}/getid/{}'.format(idgiver_ip, IDGIVER_PORT,
job_key))
job_index = int(res.text)
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
os.environ['__LITHOPS_BACKEND'] = 'k8s'
logger.info("Activation ID: {} - Job Index: {}".format(act_id, job_index))
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
call_ids = call_ids_ranges[job_index]
data_byte_ranges = [
payload['data_byte_ranges'][int(call_id)] for call_id in call_ids
]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = data_byte_ranges
function_handler(payload)
def main_job(action, encoded_payload):
logger.info(
"Lithops v{} - Starting Code Engine execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
if (action == 'preinstalls'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = int(os.environ['JOB_INDEX'])
payload['JOB_INDEX'] = job_index
logger.info("Action {}. Job Index {}".format(action, job_index))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
call_ids = call_ids_ranges[job_index]
data_byte_ranges = [
payload['data_byte_ranges'][int(call_id)] for call_id in call_ids
]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = data_byte_ranges
function_handler(payload)
return {"Execution": "Finished"}
def delete(name, config, backend, storage, debug):
""" delete a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime delete" command is only valid for serverless backends')
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def delete(name, config, backend, storage):
""" delete a serverless runtime """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def list_runtimes(config, backend, debug):
""" list all deployed serverless runtime. """
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, runtime_name='None')
config = default_config(config, config_ow, load_storage_config=False)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime list" command is only valid for serverless backends')
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, None)
runtimes = compute_handler.list_runtimes()
if runtimes:
width = max([len(runtime[0]) for runtime in runtimes])
print('\n{:{width}} \t {}'.format('Runtime Name', 'Memory Size (MB)', width=width))
print('-' * width, '\t', '-' * 20)
for runtime in runtimes:
name = runtime[0]
mem = runtime[1]
print('{:{width}} \t {}'.format(name, mem, width=width))
print()
print('Total runtimes: {}'.format(len(runtimes)))
else:
width = 10
print('\n{:{width}} \t {}'.format('Runtime Name', 'Memory Size (MB)', width=width))
print('-' * width, '\t', '-' * 20)
print('\nNo runtimes deployed')
def update(name, config, backend, storage, debug):
""" Update a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime update" command is only valid for serverless backends')
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.deploy_runtime(runtime[0], runtime[1], timeout)
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def list_bucket(prefix, bucket, backend, debug, config):
if config:
config = load_yaml_config(config)
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(config=config, backend=backend)
logger.info('Listing objects in bucket {}'.format(bucket))
objects = storage.list_objects(bucket, prefix=prefix)
if objects:
width = max([len(obj['Key']) for obj in objects])
print('\n{:{width}} \t {} \t\t {:>9}'.format('Key', 'Last modified', 'Size', width=width))
print('-' * width, '\t', '-' * 20, '\t', '-' * 9)
for obj in objects:
key = obj['Key']
date = obj['LastModified'].strftime("%b %d %Y %H:%M:%S")
size = sizeof_fmt(obj['Size'])
print('{:{width}} \t {} \t {:>9}'.format(key, date, size, width=width))
print()
print('Total objects: {}'.format(len(objects)))
else:
width = 10
print('\n{:{width}} \t {} \t\t {:>9}'.format('Key', 'Last modified', 'Size', width=width))
print('-' * width, '\t', '-' * 20, '\t', '-' * 9)
print('\nThe bucket is empty')
def deploy(name, storage, backend, memory, timeout, config, debug):
""" deploy a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime create" command is only valid for serverless backends')
logger.info('Creating new lithops runtime: {}'.format(name))
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.deploy_runtime(name, mem, timeout=to)
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def test_function(config, backend, storage, debug):
if config:
config = load_yaml_config(config)
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
try:
import getpass
username = getpass.getuser()
except Exception:
username = '******'
def hello(name):
return 'Hello {}!'.format(name)
fexec = lithops.FunctionExecutor(config=config, backend=backend, storage=storage)
fexec.call_async(hello, username)
result = fexec.get_result()
print()
if result == 'Hello {}!'.format(username):
print(result, 'Lithops is working as expected :)')
else:
print(result, 'Something went wrong :(')
print()
def run():
def error():
response = flask.jsonify({
'error':
'The action did not receive a dictionary as an argument.'
})
response.status_code = 404
return complete(response)
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
os.environ['__LITHOPS_BACKEND'] = 'Knative'
setup_lithops_logger(message['log_level'])
if 'remote_invoker' in message:
logger.info(
"Lithops v{} - Starting Knative invoker".format(__version__))
function_invoker(message)
else:
logger.info(
"Lithops v{} - Starting Knative execution".format(__version__))
function_handler(message)
response = flask.jsonify({"activationId": act_id})
response.status_code = 202
return complete(response)
def process_runner(worker_id, job_queue):
logger.debug('Localhost worker process {} started'.format(worker_id))
os.environ['__LITHOPS_LOCAL_EXECUTION'] = 'True'
p_logger = logging.getLogger('lithops')
while True:
with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr(buf):
event = job_queue.get(block=True)
if isinstance(event, ShutdownSentinel):
break
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
executor_id = event['executor_id']
job_id = event['job_id']
setup_lithops_logger(event['log_level'])
p_logger.info(
"Lithops v{} - Starting execution".format(__version__))
function_handler(event)
log_output = buf.getvalue()
job_key = create_job_key(executor_id, job_id)
log_file = os.path.join(LOGS_DIR, job_key + '.log')
header = "Activation: '{}' ({})\n[\n".format(event['runtime_name'],
act_id)
tail = ']\n\n'
output = log_output.replace('\n', '\n ', log_output.count('\n') - 1)
with open(log_file, 'a') as lf:
lf.write(header + ' ' + output + tail)
with open(FN_LOG_FILE, 'a') as lf:
lf.write(header + ' ' + output + tail)
def main_job(action, encoded_payload):
logger.info(
"Lithops v{} - Starting Code Engine execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
if (action == 'preinstalls'):
runtime_packages(payload)
return {"Execution": "Finished"}
job_index = os.environ['JOB_INDEX']
payload['JOB_INDEX'] = job_index
logger.info("Action {}. Job Index {}".format(action, job_index))
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
payload['data_byte_range'] = payload['data_byte_range'][int(job_index)]
payload['call_id'] = "{:05d}".format(int(job_index))
function_handler(payload)
return {"Execution": "Finished"}
def update(name, config, backend, storage):
""" Update a serverless runtime """
if config:
config = load_yaml_config(config)
verify_runtime_name(name)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def verify(test, config, mode, backend, storage, debug):
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
if test == 'help':
print_help()
else:
run_tests(test, config, mode, backend, storage)
def put_object(filename, bucket, backend, debug):
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(backend=backend)
logger.info('Uploading file {} to bucket {}'.format(filename, bucket))
with open(filename, 'rb') as in_file:
storage.put_object(bucket, filename, in_file)
logger.info('File uploaded successfully')
def function_handler(payload):
job = SimpleNamespace(**payload)
setup_lithops_logger(job.log_level)
processes = min(job.worker_processes, len(job.call_ids))
logger.info('Tasks received: {} - Concurrent processes: {}'.format(
len(job.call_ids), processes))
env = job.extra_env
env['LITHOPS_WORKER'] = 'True'
env['PYTHONUNBUFFERED'] = 'True'
os.environ.update(env)
storage_config = extract_storage_config(job.config)
internal_storage = InternalStorage(storage_config)
job.func = get_function_and_modules(job, internal_storage)
job_data = get_function_data(job, internal_storage)
if processes == 1:
job_queue = queue.Queue()
for call_id in job.call_ids:
data = job_data.pop(0)
job_queue.put((job, call_id, data))
job_queue.put(ShutdownSentinel())
process_runner(job_queue)
else:
manager = SyncManager()
manager.start()
job_queue = manager.Queue()
job_runners = []
for call_id in job.call_ids:
data = job_data.pop(0)
job_queue.put((job, call_id, data))
for i in range(processes):
job_queue.put(ShutdownSentinel())
for runner_id in range(processes):
p = mp.Process(target=process_runner, args=(job_queue, ))
job_runners.append(p)
p.start()
logger.info('Worker process {} started'.format(runner_id))
for runner in job_runners:
runner.join()
manager.shutdown()
# Delete modules path from syspath
module_path = os.path.join(MODULES_DIR, job.job_key)
if module_path in sys.path:
sys.path.remove(module_path)
# Unset specific job env vars
for key in job.extra_env:
os.environ.pop(key, None)
os.environ.pop('__LITHOPS_TOTAL_EXECUTORS', None)
def preinstalls_task():
setup_lithops_logger(logging.INFO)
logger.info("Lithops v{} - Generating metadata".format(__version__))
runtime_meta = get_runtime_preinstalls()
response = flask.jsonify(runtime_meta)
response.status_code = 200
logger.info("Done!")
return complete(response)
def empty_bucket(bucket, backend, debug):
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(backend=backend)
logger.info('Deleting all objects in bucket "{}"'.format(bucket))
keys = storage.list_keys(bucket)
logger.info('Total objects found: {}'.format(len(keys)))
storage.delete_objects(bucket, keys)
logger.info('All objects deleted successfully')
def get_object(bucket, key, backend, debug):
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(backend=backend)
logger.info('Downloading object {} from bucket {}'.format(key, bucket))
data_stream = storage.get_object(bucket, key, stream=True)
with open(key, 'wb') as out:
shutil.copyfileobj(data_stream, out)
logger.info('Object downloaded successfully')
def main(event, context):
args = json.loads(event)
os.environ['__LITHOPS_ACTIVATION_ID'] = context.request_id
setup_lithops_logger(args['log_level'])
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def main(event, context):
logger.info("Starting GCP Functions function execution")
args = json.loads(base64.b64decode(event['data']).decode('utf-8'))
os.environ['__PW_ACTIVATION_ID'] = uuid.uuid4().hex
setup_lithops_logger(args['log_level'])
if 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def extract_runtime_meta(encoded_payload):
logger.info("Lithops v{} - Generating metadata".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
runtime_meta = get_runtime_preinstalls()
internal_storage = InternalStorage(payload)
status_key = '/'.join([JOBS_PREFIX, payload['runtime_name'] + '.meta'])
logger.info("Runtime metadata key {}".format(status_key))
dmpd_response_status = json.dumps(runtime_meta)
internal_storage.put_data(status_key, dmpd_response_status)
def delete_object(bucket, key, prefix, backend, debug):
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(backend=backend)
if key:
logger.info('Deleting object "{}" from bucket "{}"'.format(key, bucket))
storage.delete_object(bucket, key)
logger.info('Object deleted successfully')
elif prefix:
objs = storage.list_keys(bucket, prefix)
logger.info('Deleting {} objects with prefix "{}" from bucket "{}"'.format(len(objs), prefix, bucket))
storage.delete_objects(bucket, objs)
logger.info('Object deleted successfully')
def build(name, file, config, backend):
""" build a serverless runtime. """
verify_runtime_name(name)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def lambda_handler(event, context):
os.environ['__LITHOPS_ACTIVATION_ID'] = context.aws_request_id
setup_lithops_logger(event.get('log_level', logging.INFO))
if 'get_preinstalls' in event:
logger.info("Lithops v{} - Generating metadata".format(__version__))
return get_runtime_preinstalls()
elif 'remote_invoker' in event:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(event)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(event)
return {"Execution": "Finished"}
def run_job(encoded_payload):
logger.info(
"Lithops v{} - Starting kubernetes execution".format(__version__))
payload = b64str_to_dict(encoded_payload)
setup_lithops_logger(payload['log_level'])
total_calls = payload['total_calls']
job_key = payload['job_key']
master_ip = os.environ['MASTER_POD_IP']
chunksize = payload['chunksize']
call_ids_ranges = [
call_ids_range
for call_ids_range in iterchunks(payload['call_ids'], chunksize)
]
data_byte_ranges = payload['data_byte_ranges']
job_finished = False
while not job_finished:
job_index = None
while job_index is None:
try:
url = f'http://{master_ip}:{MASTER_PORT}/getid/{job_key}/{total_calls}'
res = requests.get(url)
job_index = int(res.text)
except Exception:
time.sleep(0.1)
if job_index == -1:
job_finished = True
continue
act_id = str(uuid.uuid4()).replace('-', '')[:12]
os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
os.environ['__LITHOPS_BACKEND'] = 'k8s'
logger.info("Activation ID: {} - Job Index: {}".format(
act_id, job_index))
call_ids = call_ids_ranges[job_index]
dbr = [data_byte_ranges[int(call_id)] for call_id in call_ids]
payload['call_ids'] = call_ids
payload['data_byte_ranges'] = dbr
function_handler(payload)
