def update(name, config, backend): """ Update a serverless runtime """ verify_runtime_name(name) setup_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise ( "Unable to upload 'preinstalled-modules' file into {}".format( internal_storage.backend))
def clean(mode, config, debug): log_level = 'INFO' if not debug else 'DEBUG' setup_logger(log_level) logger.info('Cleaning all Lithops information') config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def create(name, backend, memory, timeout, config): """ Create a serverless runtime """ setup_logger(logging.DEBUG) logger.info('Creating new lithops runtime: {}'.format(name)) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise ("Unable to upload 'preinstalled-modules' file into {}".format( internal_storage.backend))
def main_job(action, encoded_payload): logger.info( "Lithops v{} - Starting Code Engine execution".format(__version__)) payload = b64str_to_dict(encoded_payload) setup_logger(payload['log_level']) if (action == 'preinstalls'): runtime_packages(payload) return {"Execution": "Finished"} job_index = os.environ['JOB_INDEX'] payload['JOB_INDEX'] = job_index logger.info("Action {}. Job Index {}".format(action, job_index)) act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id payload['data_byte_range'] = payload['data_byte_range'][int(job_index)] payload['call_id'] = "{:05d}".format(int(job_index)) function_handler(payload) return {"Execution": "Finished"}
def main(action, payload_decoded): logger.info( "Welcome to Lithops-Code-Engine entry point. Action {}".format(action)) payload = binary_to_dict(payload_decoded) setup_logger(payload['log_level']) logger.info(payload) if (action == 'preinstals'): runtime_packages(payload) return {"Execution": "Finished"} job_index = os.environ['JOB_INDEX'] logger.info("Action {}. Job Index {}".format(action, job_index)) os.environ['__PW_ACTIVATION_ID'] = payload['activation_id'] payload['JOB_INDEX'] = job_index if 'remote_invoker' in payload: logger.info("Lithops v{} - Remote Invoker. Starting execution".format( __version__)) #function_invoker(payload) payload['data_byte_range'] = payload['job_description']['data_ranges'][ int(job_index)] for key in payload['job_description']: payload[key] = payload['job_description'][key] payload['host_submit_tstamp'] = payload['metadata'][ 'host_job_create_tstamp'] payload['call_id'] = "{:05d}".format(int(job_index)) function_handler(payload) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(payload) return {"Execution": "Finished"}
def create(name, mode, memory, timeout, config): setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) runtime_key = compute_handler.get_runtime_key(name) runtime_meta = compute_handler.create_runtime(name) elif mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) runtime_key = compute_handler.get_runtime_key(name) runtime_meta = compute_handler.create_runtime(name) else: raise Exception('Unknown execution mode {}'.format(mode)) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def run(): def error(): response = flask.jsonify({ 'error': 'The action did not receive a dictionary as an argument.' }) response.status_code = 404 return complete(response) message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id setup_logger(message['log_level']) if 'remote_invoker' in message: logger.info( "Lithops v{} - Starting Knative invoker".format(__version__)) function_invoker(message) else: logger.info( "Lithops v{} - Starting Knative execution".format(__version__)) function_handler(message) response = flask.jsonify({"activationId": act_id}) response.status_code = 202 return complete(response)
def _process_runner(self, worker_id): logger.debug('Localhost worker process {} started'.format(worker_id)) os.environ['__LITHOPS_LOCAL_EXECUTION'] = 'True' p_logger = logging.getLogger('lithops') while True: with io.StringIO() as buf, redirect_stdout(buf), redirect_stderr(buf): event = self.queue.get(block=True) if isinstance(event, ShutdownSentinel): break act_id = str(uuid.uuid4()).replace('-', '')[:12] os.environ['__LITHOPS_ACTIVATION_ID'] = act_id executor_id = event['executor_id'] job_id = event['job_id'] setup_logger(event['log_level']) p_logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(event) log_output = buf.getvalue() job_key = create_job_key(executor_id, job_id) log_file = os.path.join(LOGS_DIR, job_key+'.log') header = "Activation: '{}' ({})\n[\n".format(event['runtime_name'], act_id) tail = ']\n\n' output = log_output.replace('\n', '\n ', log_output.count('\n')-1) with open(log_file, 'a') as lf: lf.write(header+' '+output+tail) with open(FN_LOG_FILE, 'a') as lf: lf.write(header+' '+output+tail)
def build(name, file, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) compute_handler.build_runtime(name, file)
def verify(test, config, mode, debug): if debug: setup_logger(logging.DEBUG) if test == 'help': print_help() else: run_tests(test, mode, config)
def preinstalls_task(): setup_logger(logging.INFO) logger.info("Lithops v{} - Generating metadata".format(__version__)) runtime_meta = get_runtime_preinstalls() response = flask.jsonify(runtime_meta) response.status_code = 200 logger.info("Done!") return complete(response)
def main(event, context): args = json.loads(event) os.environ['__LITHOPS_ACTIVATION_ID'] = context.request_id setup_logger(args['log_level']) if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def main(event, context): logger.info("Starting GCP Functions function execution") args = json.loads(base64.b64decode(event['data']).decode('utf-8')) os.environ['__PW_ACTIVATION_ID'] = uuid.uuid4().hex setup_logger(args['log_level']) if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def main(args): os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID'] setup_logger(args['log_level'], sys.stdout, LOGGER_FORMAT_SHORT) if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def lambda_handler(event, context): os.environ['__PW_ACTIVATION_ID'] = context.aws_request_id os.environ['__OW_ACTIVATION_ID'] = context.aws_request_id setup_logger(event['log_level']) if 'remote_invoker' in event: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(event) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(event) return {"Execution": "Finished"}
def delete(name, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def build(name, file, config, backend): """ build a serverless runtime. """ verify_runtime_name(name) setup_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) compute_handler.build_runtime(name, file)
def main(args): os.environ['__LITHOPS_ACTIVATION_ID'] = os.environ['__OW_ACTIVATION_ID'] setup_logger(args['log_level'], sys.stdout, LOGGER_FORMAT_SHORT) if 'get_preinstalls' in args: logger.info("Lithops v{} - Generating metadata".format(__version__)) return get_runtime_preinstalls() elif 'remote_invoker' in args: logger.info("Lithops v{} - Starting IBM CF invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting IBM CF execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def main(msgIn: func.QueueMessage): try: args = json.loads(msgIn.get_body()) except Exception: args = msgIn.get_json() os.environ['__PW_ACTIVATION_ID'] = str(msgIn.id) setup_logger(args['log_level']) if 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args) return {"Execution": "Finished"}
def test_function(config, debug): if debug: setup_logger(logging.DEBUG) def hello(name): return 'Hello {}!'.format(name) fexec = lithops.FunctionExecutor(config=config) fexec.call_async(hello, 'World') result = fexec.get_result() print() if result == 'Hello World!': print(result, 'Lithops is working as expected :)') else: print(result, 'Something went wrong :(') print()
def main(msgIn: func.QueueMessage, msgOut: func.Out[func.QueueMessage]): try: args = json.loads(msgIn.get_body()) except Exception: args = msgIn.get_json() os.environ['__LITHOPS_ACTIVATION_ID'] = str(msgIn.id) setup_logger(args['log_level']) if 'get_preinstalls' in args: logger.info("Lithops v{} - Generating metadata".format(__version__)) runtime_meta = get_runtime_preinstalls() msgOut.set(json.dumps(runtime_meta)) elif 'remote_invoker' in args: logger.info("Lithops v{} - Starting invoker".format(__version__)) function_invoker(args) else: logger.info("Lithops v{} - Starting execution".format(__version__)) function_handler(args)
def delete(name, config, backend): """ delete a serverless runtime """ verify_runtime_name(name) setup_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def update(name, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) timeout = config['lithops']['runtime_timeout'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
# from multiprocessing import Pool from lithops.multiprocessing import Pool from lithops.utils import setup_logger import time import logging import os setup_logger(logging.CRITICAL) def hello(name): return 'Hello {}!'.format(name) def square(x): return x * x def divide(x, y): return x / y def sleep_seconds(s): time.sleep(s) if __name__ == '__main__': with Pool() as pool: # Synchronously execute function square remotely
def __init__(self, type=None, mode=None, config=None, backend=None, storage=None, runtime=None, runtime_memory=None, rabbitmq_monitor=None, workers=None, remote_invoker=None, log_level=None): mode = mode or type if mode is None: config = default_config(copy.deepcopy(config)) mode = config['lithops']['mode'] if mode not in [LOCALHOST, SERVERLESS, STANDALONE]: raise Exception("Function executor mode must be one of '{}', '{}' " "or '{}'".format(LOCALHOST, SERVERLESS, STANDALONE)) if log_level: setup_logger(log_level) if type is not None: logger.warning("'type' parameter is deprecated and it will be removed" "in future releases. Use 'mode' parameter instead") config_ow = {'lithops': {'mode': mode}, mode: {}} if runtime is not None: config_ow[mode]['runtime'] = runtime if backend is not None: config_ow[mode]['backend'] = backend if runtime_memory is not None: config_ow[mode]['runtime_memory'] = int(runtime_memory) if remote_invoker is not None: config_ow[mode]['remote_invoker'] = remote_invoker if storage is not None: config_ow['lithops']['storage'] = storage if workers is not None: config_ow['lithops']['workers'] = workers if rabbitmq_monitor is not None: config_ow['lithops']['rabbitmq_monitor'] = rabbitmq_monitor self.config = default_config(copy.deepcopy(config), config_ow) self.log_active = logger.getEffectiveLevel() != logging.WARNING self.is_lithops_worker = is_lithops_worker() self.executor_id = create_executor_id() self.data_cleaner = self.config['lithops'].get('data_cleaner', True) if self.data_cleaner and not self.is_lithops_worker: spawn_cleaner = int(self.executor_id.split('-')[1]) == 0 atexit.register(self.clean, spawn_cleaner=spawn_cleaner, clean_cloudobjects=False) self.rabbitmq_monitor = self.config['lithops'].get('rabbitmq_monitor', False) if self.rabbitmq_monitor: if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') else: raise Exception("You cannot use rabbitmq_mnonitor since " "'amqp_url' is not present in configuration") storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.storage = self.internal_storage.storage self.futures = [] self.cleaned_jobs = set() self.total_jobs = 0 self.last_call = None if mode == LOCALHOST: localhost_config = extract_localhost_config(self.config) self.compute_handler = LocalhostHandler(localhost_config) self.invoker = StandaloneInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) elif mode == SERVERLESS: serverless_config = extract_serverless_config(self.config) self.compute_handler = ServerlessHandler(serverless_config, storage_config) self.invoker = ServerlessInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) elif mode == STANDALONE: standalone_config = extract_standalone_config(self.config) self.compute_handler = StandaloneHandler(standalone_config) self.invoker = StandaloneInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) logger.info('{} Executor created with ID: {}' .format(mode.capitalize(), self.executor_id))