def delete_object(bucket, key, backend, debug): log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) storage = Storage(backend=backend) logger.info('Deleting object "{}" from bucket "{}"'.format(key, bucket)) storage.delete_object(bucket, key) logger.info('Object deleted successfully')
def main(event, context): # pub/sub event data is b64 encoded args = json.loads(base64.b64decode(event['data']).decode('utf-8')) setup_lithops_logger(args.get('log_level', 'INFO')) os.environ['__LITHOPS_ACTIVATION_ID'] = uuid.uuid4().hex os.environ['__LITHOPS_BACKEND'] = 'Google Cloud Functions' if 'get_preinstalls' in args: logger.info("Lithops v{} - Generating metadata".format(__version__)) internal_storage = InternalStorage( args['get_preinstalls']['storage_config']) object_key = '/'.join( [JOBS_PREFIX, args['get_preinstalls']['runtime_name'] + '.meta']) logger.info("Runtime metadata key {}".format(object_key)) runtime_meta = get_runtime_preinstalls() runtime_meta_json = json.dumps(runtime_meta) internal_storage.put_data(object_key, runtime_meta_json) elif 'remote_invoker' in args: logger.info( "Lithops v{} - Starting Google Cloud Functions invoker".format( __version__)) function_invoker(args) else: logger.info( "Lithops v{} - Starting Google Cloud Functions execution".format( __version__)) function_handler(args) return {"Execution": "Finished"}
def create(name, storage, backend, memory, timeout, config): """ Create a serverless runtime """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] logger.info('Creating new lithops runtime: {}'.format(name)) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise ("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def download_file(bucket, key, out, backend, debug, config): if config: config = load_yaml_config(config) log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) storage = Storage(config=config, backend=backend) def download_file(): logger.info( f'Downloading file {storage.backend}://{bucket}/{key} to {out or key}' ) if storage.download_file(bucket, key, out): file_size = os.path.getsize(out or key) logger.info( f'Download File {key} - Size: {sizeof_fmt(file_size)} - Ok') else: logger.error(f'Download File {key} - Error') with ThreadPoolExecutor() as ex: future = ex.submit(download_file) cy = cycle(r"-\|/") while not future.done(): print("Downloading file " + next(cy), end="\r") time.sleep(0.1) future.result()
def run_job(encoded_payload): logger.info( "Lithops v{} - Starting kubernetes execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) job_key = payload['job_key'] idgiver_ip = os.environ['IDGIVER_POD_IP'] res = requests.get('http://{}:{}/getid/{}'.format(idgiver_ip, IDGIVER_PORT, job_key)) job_index = int(res.text) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id os.environ['__LITHOPS_BACKEND'] = 'k8s' logger.info("Activation ID: {} - Job Index: {}".format(act_id, job_index)) chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] call_ids = call_ids_ranges[job_index] data_byte_ranges = [ payload['data_byte_ranges'][int(call_id)] for call_id in call_ids ] payload['call_ids'] = call_ids payload['data_byte_ranges'] = data_byte_ranges function_handler(payload)
def main_job(action, encoded_payload): logger.info( "Lithops v{} - Starting Code Engine execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) if (action == 'preinstalls'): runtime_packages(payload) return {"Execution": "Finished"} job_index = int(os.environ['JOB_INDEX']) payload['JOB_INDEX'] = job_index logger.info("Action {}. Job Index {}".format(action, job_index)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] call_ids = call_ids_ranges[job_index] data_byte_ranges = [ payload['data_byte_ranges'][int(call_id)] for call_id in call_ids ] payload['call_ids'] = call_ids payload['data_byte_ranges'] = data_byte_ranges function_handler(payload) return {"Execution": "Finished"}
def delete(name, config, backend, storage, debug): """ delete a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime delete" command is only valid for serverless backends') storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def delete(name, config, backend, storage): """ delete a serverless runtime """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def list_runtimes(config, backend, debug): """ list all deployed serverless runtime. """ log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, runtime_name='None') config = default_config(config, config_ow, load_storage_config=False) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime list" command is only valid for serverless backends') compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, None) runtimes = compute_handler.list_runtimes() if runtimes: width = max([len(runtime[0]) for runtime in runtimes]) print('\n{:{width}} \t {}'.format('Runtime Name', 'Memory Size (MB)', width=width)) print('-' * width, '\t', '-' * 20) for runtime in runtimes: name = runtime[0] mem = runtime[1] print('{:{width}} \t {}'.format(name, mem, width=width)) print() print('Total runtimes: {}'.format(len(runtimes))) else: width = 10 print('\n{:{width}} \t {}'.format('Runtime Name', 'Memory Size (MB)', width=width)) print('-' * width, '\t', '-' * 20) print('\nNo runtimes deployed')
def update(name, config, backend, storage, debug): """ Update a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime update" command is only valid for serverless backends') storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.deploy_runtime(runtime[0], runtime[1], timeout) internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def list_bucket(prefix, bucket, backend, debug, config): if config: config = load_yaml_config(config) log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) storage = Storage(config=config, backend=backend) logger.info('Listing objects in bucket {}'.format(bucket)) objects = storage.list_objects(bucket, prefix=prefix) if objects: width = max([len(obj['Key']) for obj in objects]) print('\n{:{width}} \t {} \t\t {:>9}'.format('Key', 'Last modified', 'Size', width=width)) print('-' * width, '\t', '-' * 20, '\t', '-' * 9) for obj in objects: key = obj['Key'] date = obj['LastModified'].strftime("%b %d %Y %H:%M:%S") size = sizeof_fmt(obj['Size']) print('{:{width}} \t {} \t {:>9}'.format(key, date, size, width=width)) print() print('Total objects: {}'.format(len(objects))) else: width = 10 print('\n{:{width}} \t {} \t\t {:>9}'.format('Key', 'Last modified', 'Size', width=width)) print('-' * width, '\t', '-' * 20, '\t', '-' * 9) print('\nThe bucket is empty')
def deploy(name, storage, backend, memory, timeout, config, debug): """ deploy a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime create" command is only valid for serverless backends') logger.info('Creating new lithops runtime: {}'.format(name)) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.deploy_runtime(name, mem, timeout=to) internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def test_function(config, backend, storage, debug): if config: config = load_yaml_config(config) log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) try: import getpass username = getpass.getuser() except Exception: username = '******' def hello(name): return 'Hello {}!'.format(name) fexec = lithops.FunctionExecutor(config=config, backend=backend, storage=storage) fexec.call_async(hello, username) result = fexec.get_result() print() if result == 'Hello {}!'.format(username): print(result, 'Lithops is working as expected :)') else: print(result, 'Something went wrong :(') print()
def run(): def error(): response = flask.jsonify({ 'error': 'The action did not receive a dictionary as an argument.' }) response.status_code = 404 return complete(response) message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id os.environ['__LITHOPS_BACKEND'] = 'Knative' setup_lithops_logger(message['log_level']) if 'remote_invoker' in message: logger.info( "Lithops v{} - Starting Knative invoker".format(__version__)) function_invoker(message) else: logger.info( "Lithops v{} - Starting Knative execution".format(__version__)) function_handler(message) response = flask.jsonify({"activationId": act_id}) response.status_code = 202 return complete(response)
def process_runner(worker_id, job_queue): logger.debug('Localhost worker process {} started'.format(worker_id)) os.environ['__LITHOPS_LOCAL_EXECUTION'] = 'True' p_logger = logging.getLogger('lithops') while True: with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr(buf): event = job_queue.get(block=True) if isinstance(event, ShutdownSentinel): break act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id executor_id = event['executor_id'] job_id = event['job_id'] setup_lithops_logger(event['log_level']) p_logger.info( "Lithops v{} - Starting execution".format(__version__)) function_handler(event) log_output = buf.getvalue() job_key = create_job_key(executor_id, job_id) log_file = os.path.join(LOGS_DIR, job_key + '.log') header = "Activation: '{}' ({})\n[\n".format(event['runtime_name'], act_id) tail = ']\n\n' output = log_output.replace('\n', '\n ', log_output.count('\n') - 1) with open(log_file, 'a') as lf: lf.write(header + ' ' + output + tail) with open(FN_LOG_FILE, 'a') as lf: lf.write(header + ' ' + output + tail)
def main_job(action, encoded_payload): logger.info( "Lithops v{} - Starting Code Engine execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) if (action == 'preinstalls'): runtime_packages(payload) return {"Execution": "Finished"} job_index = os.environ['JOB_INDEX'] payload['JOB_INDEX'] = job_index logger.info("Action {}. Job Index {}".format(action, job_index)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id payload['data_byte_range'] = payload['data_byte_range'][int(job_index)] payload['call_id'] = "{:05d}".format(int(job_index)) function_handler(payload) return {"Execution": "Finished"}
def update(name, config, backend, storage): """ Update a serverless runtime """ if config: config = load_yaml_config(config) verify_runtime_name(name) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def verify(test, config, mode, backend, storage, debug): log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) if test == 'help': print_help() else: run_tests(test, config, mode, backend, storage)
def put_object(filename, bucket, backend, debug): log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) storage = Storage(backend=backend) logger.info('Uploading file {} to bucket {}'.format(filename, bucket)) with open(filename, 'rb') as in_file: storage.put_object(bucket, filename, in_file) logger.info('File uploaded successfully')
def function_handler(payload): job = SimpleNamespace(**payload) setup_lithops_logger(job.log_level) processes = min(job.worker_processes, len(job.call_ids)) logger.info('Tasks received: {} - Concurrent processes: {}'.format( len(job.call_ids), processes)) env = job.extra_env env['LITHOPS_WORKER'] = 'True' env['PYTHONUNBUFFERED'] = 'True' os.environ.update(env) storage_config = extract_storage_config(job.config) internal_storage = InternalStorage(storage_config) job.func = get_function_and_modules(job, internal_storage) job_data = get_function_data(job, internal_storage) if processes == 1: job_queue = queue.Queue() for call_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, call_id, data)) job_queue.put(ShutdownSentinel()) process_runner(job_queue) else: manager = SyncManager() manager.start() job_queue = manager.Queue() job_runners = [] for call_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, call_id, data)) for i in range(processes): job_queue.put(ShutdownSentinel()) for runner_id in range(processes): p = mp.Process(target=process_runner, args=(job_queue, )) job_runners.append(p) p.start() logger.info('Worker process {} started'.format(runner_id)) for runner in job_runners: runner.join() manager.shutdown() # Delete modules path from syspath module_path = os.path.join(MODULES_DIR, job.job_key) if module_path in sys.path: sys.path.remove(module_path) # Unset specific job env vars for key in job.extra_env: os.environ.pop(key, None) os.environ.pop('__LITHOPS_TOTAL_EXECUTORS', None)
def preinstalls_task(): setup_lithops_logger(logging.INFO) logger.info("Lithops v{} - Generating metadata".format(__version__)) runtime_meta = get_runtime_preinstalls() response = flask.jsonify(runtime_meta) response.status_code = 200 logger.info("Done!") return complete(response)
def empty_bucket(bucket, backend, debug): log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) storage = Storage(backend=backend) logger.info('Deleting all objects in bucket "{}"'.format(bucket)) keys = storage.list_keys(bucket) logger.info('Total objects found: {}'.format(len(keys))) storage.delete_objects(bucket, keys) logger.info('All objects deleted successfully')
def get_object(bucket, key, backend, debug): log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) storage = Storage(backend=backend) logger.info('Downloading object {} from bucket {}'.format(key, bucket)) data_stream = storage.get_object(bucket, key, stream=True) with open(key, 'wb') as out: shutil.copyfileobj(data_stream, out) logger.info('Object downloaded successfully')
def main(event, context): args = json.loads(event) os.environ['__LITHOPS_ACTIVATION_ID'] = context.request_id setup_lithops_logger(args['log_level']) if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def main(event, context): logger.info("Starting GCP Functions function execution") args = json.loads(base64.b64decode(event['data']).decode('utf-8')) os.environ['__PW_ACTIVATION_ID'] = uuid.uuid4().hex setup_lithops_logger(args['log_level']) if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def extract_runtime_meta(encoded_payload): logger.info("Lithops v{} - Generating metadata".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) runtime_meta = get_runtime_preinstalls() internal_storage = InternalStorage(payload) status_key = '/'.join([JOBS_PREFIX, payload['runtime_name'] + '.meta']) logger.info("Runtime metadata key {}".format(status_key)) dmpd_response_status = json.dumps(runtime_meta) internal_storage.put_data(status_key, dmpd_response_status)
def delete_object(bucket, key, prefix, backend, debug): log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) storage = Storage(backend=backend) if key: logger.info('Deleting object "{}" from bucket "{}"'.format(key, bucket)) storage.delete_object(bucket, key) logger.info('Object deleted successfully') elif prefix: objs = storage.list_keys(bucket, prefix) logger.info('Deleting {} objects with prefix "{}" from bucket "{}"'.format(len(objs), prefix, bucket)) storage.delete_objects(bucket, objs) logger.info('Object deleted successfully')
def build(name, file, config, backend): """ build a serverless runtime. """ verify_runtime_name(name) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) compute_handler.build_runtime(name, file)
def lambda_handler(event, context): os.environ['__LITHOPS_ACTIVATION_ID'] = context.aws_request_id setup_lithops_logger(event.get('log_level', logging.INFO)) if 'get_preinstalls' in event: logger.info("Lithops v{} - Generating metadata".format(__version__)) return get_runtime_preinstalls() elif 'remote_invoker' in event: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(event) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(event) return {"Execution": "Finished"}
def run_job(encoded_payload): logger.info( "Lithops v{} - Starting kubernetes execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_lithops_logger(payload['log_level']) total_calls = payload['total_calls'] job_key = payload['job_key'] master_ip = os.environ['MASTER_POD_IP'] chunksize = payload['chunksize'] call_ids_ranges = [ call_ids_range for call_ids_range in iterchunks(payload['call_ids'], chunksize) ] data_byte_ranges = payload['data_byte_ranges'] job_finished = False while not job_finished: job_index = None while job_index is None: try: url = f'http://{master_ip}:{MASTER_PORT}/getid/{job_key}/{total_calls}' res = requests.get(url) job_index = int(res.text) except Exception: time.sleep(0.1) if job_index == -1: job_finished = True continue act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id os.environ['__LITHOPS_BACKEND'] = 'k8s' logger.info("Activation ID: {} - Job Index: {}".format( act_id, job_index)) call_ids = call_ids_ranges[job_index] dbr = [data_byte_ranges[int(call_id)] for call_id in call_ids] payload['call_ids'] = call_ids payload['data_byte_ranges'] = dbr function_handler(payload)