def update(name, config, backend, storage): """ Update a serverless runtime """ if config: config = load_yaml_config(config) verify_runtime_name(name) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def delete(name, config, backend, storage, debug): """ delete a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime delete" command is only valid for serverless backends') storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def create(name, storage, backend, memory, timeout, config): """ Create a serverless runtime """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] logger.info('Creating new lithops runtime: {}'.format(name)) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise ("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def run_worker(master_ip, job_key): """ Run a job """ global BUDGET_KEEPER while True: url = 'http://{}:{}/get-task/{}'.format(master_ip, STANDALONE_SERVICE_PORT, job_key) logger.info('Getting task from {}'.format(url)) resp = requests.get(url) if resp.status_code != 200: logger.info('All tasks completed'.format(url)) return job_payload = resp.json() logger.info(job_payload) logger.info('Got tasks {}'.format(', '.join(job_payload['call_ids']))) try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception: return BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running' pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime}) localhost_handler.invoke(job_payload) wait_job_completed(job_key)
def deploy(name, storage, backend, memory, timeout, config, debug): """ deploy a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime create" command is only valid for serverless backends') logger.info('Creating new lithops runtime: {}'.format(name)) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.deploy_runtime(name, mem, timeout=to) internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def update(name, config, backend, storage, debug): """ Update a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime update" command is only valid for serverless backends') storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.deploy_runtime(runtime[0], runtime[1], timeout) internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def run(): """ Run a job """ global BUDGET_KEEPER job_payload = flask.request.get_json(force=True, silent=True) if job_payload and not isinstance(job_payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running' pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) localhost_handler = LocalhostHandler({ 'runtime': runtime, 'pull_runtime': pull_runtime }) localhost_handler.run_job(job_payload) act_id = str(uuid.uuid4()).replace('-', '')[:12] response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def delete(name, config, backend, storage): """ delete a serverless runtime """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def build(name, file, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) compute_handler.build_runtime(name, file)
def run(): """ Run a job locally, in consume mode """ global budget_keeper global work_queues global exec_mode global localhost_manager_process job_payload = flask.request.get_json(force=True, silent=True) if job_payload and not isinstance(job_payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) job_key = job_payload['job_key'] logger.debug('Received job {}'.format(job_key)) budget_keeper.last_usage_time = time.time() budget_keeper.update_config(job_payload['config']['standalone']) budget_keeper.jobs[job_key] = 'running' exec_mode = job_payload['config']['standalone'].get('exec_mode', 'consume') if exec_mode == 'consume': # Consume mode runs jobs in this master VM work_queue_name = 'local' work_queue = work_queues.setdefault(work_queue_name, queue.Queue()) if not localhost_manager_process: logger.debug('Starting manager process for localhost jobs') lmp = Thread(target=run_job_local, args=(work_queue, ), daemon=True) lmp.start() localhost_manager_process = lmp logger.debug(f'Putting job {job_key} into master queue') work_queue.put(job_payload) elif exec_mode in ['create', 'reuse']: # Create and reuse mode runs jobs on woker VMs logger.debug(f'Starting process for job {job_key}') work_queue_name = job_key if exec_mode == 'create' else REUSE_WORK_QUEUE_NAME work_queue = work_queues.setdefault(work_queue_name, queue.Queue()) Thread(target=start_workers, args=(job_payload, work_queue_name)).start() Thread(target=run_job_worker, args=(job_payload, work_queue), daemon=True).start() act_id = str(uuid.uuid4()).replace('-', '')[:12] response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def run(): """ Run a job locally, in consume mode """ global BUDGET_KEEPER global WORK_QUEUES global JOB_PROCESSES job_payload = flask.request.get_json(force=True, silent=True) if job_payload and not isinstance(job_payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) job_key = job_payload['job_key'] logger.info('Received job {}'.format(job_key)) BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_key] = 'running' exec_mode = job_payload['config']['standalone'].get('exec_mode', 'consume') if exec_mode == 'consume': # Consume mode runs the job locally pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) try: localhost_handler = LocalhostHandler({ 'runtime': runtime, 'pull_runtime': pull_runtime }) localhost_handler.invoke(job_payload) except Exception as e: logger.error(e) elif exec_mode == 'create': # Create mode runs the job in worker VMs work_queue = MP_MANAGER.Queue() WORK_QUEUES[job_key] = work_queue jp = mp.Process(target=run_job_process, args=(job_payload, work_queue)) jp.daemon = True jp.start() JOB_PROCESSES[job_key] = jp act_id = str(uuid.uuid4()).replace('-', '')[:12] response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def run_worker(master_ip, work_queue): """ Run a job """ global budget_keeper global localhos_handler global last_job_key pull_runtime = stanbdalone_config.get('pull_runtime', False) localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime}) while True: url = f'http://{master_ip}:{SA_SERVICE_PORT}/get-task/{work_queue}' logger.debug(f'Getting task from {url}') try: resp = requests.get(url) except Exception: time.sleep(1) continue if resp.status_code != 200: if stanbdalone_config.get('exec_mode') == 'reuse': time.sleep(1) continue else: logger.debug('All tasks completed'.format(url)) return job_payload = resp.json() logger.debug(job_payload) logger.debug('Got tasks {}'.format(', '.join(job_payload['call_ids']))) try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception: return job_key = job_payload['job_key'] last_job_key = job_key budget_keeper.last_usage_time = time.time() budget_keeper.update_config(job_payload['config']['standalone']) budget_keeper.jobs[job_key] = 'running' try: localhos_handler.invoke(job_payload) except Exception as e: logger.error(e) wait_job_completed(job_key)
def delete(name, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def build(name, file, config, backend): """ build a serverless runtime. """ verify_runtime_name(name) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) compute_handler.build_runtime(name, file)
def build(ctx, name, file, config, backend): """ build a serverless runtime. """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, runtime_name=name) config = default_config(config, config_ow, load_storage_config=False) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops build" command is only valid for serverless backends') compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, None) compute_handler.build_runtime(name, file, ctx.args)
def preinstalls(): message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = message['runtime'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) localhost_handler = LocalhostHandler(message) runtime_meta = localhost_handler.create_runtime(runtime) response = flask.jsonify(runtime_meta) response.status_code = 200 return response
def preinstalls(): payload = flask.request.get_json(force=True, silent=True) if payload and not isinstance(payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = payload['runtime'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime}) runtime_meta = localhost_handler.create_runtime(runtime) response = flask.jsonify(runtime_meta) response.status_code = 200 return response
def run_create(): """ Runs a given job remotely in workers, in create mode """ global BUDGET_KEEPER logger.info('Running job on worker VMs') job_payload = flask.request.get_json(force=True, silent=True) if job_payload and not isinstance(job_payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) job_key = job_payload['job_key'] call_ids = job_payload['call_ids'] chunksize = job_payload['chunksize'] workers = job_payload['woreker_instances'] BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_key] = 'running' with ThreadPoolExecutor(len(workers)) as executor: for call_ids_range in iterchunks(call_ids, chunksize): worker_info = workers.pop(0) executor.submit(run_job_on_worker, worker_info, call_ids_range, copy.deepcopy(job_payload)) done = os.path.join(JOBS_DIR, job_key + '.done') Path(done).touch() act_id = str(uuid.uuid4()).replace('-', '')[:12] response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def delete(name, config, backend): """ delete a serverless runtime """ verify_runtime_name(name) setup_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def run(): """ Run a job """ global last_usage_time global backend_handler global jobs message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = message['job_description']['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) last_usage_time = time.time() standalone_config = message['config']['standalone'] backend_handler.auto_dismantle = standalone_config['auto_dismantle'] backend_handler.soft_dismantle_timeout = standalone_config[ 'soft_dismantle_timeout'] backend_handler.hard_dismantle_timeout = standalone_config[ 'hard_dismantle_timeout'] act_id = str(uuid.uuid4()).replace('-', '')[:12] executor_id = message['executor_id'] job_id = message['job_id'] job_key = create_job_key(executor_id, job_id) jobs[job_key] = 'running' localhost_handler = LocalhostHandler({'runtime': runtime}) localhost_handler.run_job(message) response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def preinstalls(): payload = flask.request.get_json(force=True, silent=True) if payload and not isinstance(payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = payload['runtime'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) pull_runtime = standalone_config.get('pull_runtime', False) lh = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime}) runtime_meta = lh.create_runtime(runtime) if 'lithops_version' in runtime_meta: logger.debug("Runtime metdata extracted correctly: Lithops " f"{runtime_meta['lithops_version']}") response = flask.jsonify(runtime_meta) response.status_code = 200 return response
def update(name, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) timeout = config['lithops']['runtime_timeout'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def build(name, file, config, backend): """ build a serverless runtime. """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) compute_handler.build_runtime(name, file)
def default_config(config_data=None, config_overwrite={}): """ First checks .lithops_config then checks LITHOPS_CONFIG_FILE environment variable then ~/.lithops/config """ logger.info('Lithops v{}'.format(__version__)) config_data = copy.deepcopy(config_data) or load_config() if 'lithops' not in config_data or not config_data['lithops']: config_data['lithops'] = {} if 'mode' not in config_data['lithops']: config_data['lithops']['mode'] = constants.MODE_DEFAULT if 'execution_timeout' not in config_data['lithops']: config_data['lithops'][ 'execution_timeout'] = constants.EXECUTION_TIMEOUT_DEFAULT if 'chunksize' not in config_data['lithops']: config_data['lithops']['chunksize'] = constants.CHUNKSIZE_DEFAULT if 'worker_processes' not in config_data['lithops']: config_data['lithops'][ 'worker_processes'] = constants.WORKER_PROCESSES_DEFAULT if 'monitoring' not in config_data['lithops']: config_data['lithops']['monitoring'] = constants.MONITORING_DEFAULT # overwrite values provided by the user if 'lithops' in config_overwrite: config_data['lithops'].update(config_overwrite['lithops']) if constants.LOCALHOST in config_overwrite: if constants.LOCALHOST not in config_data or \ config_data[constants.LOCALHOST] is None: config_data[constants.LOCALHOST] = {} config_data[constants.LOCALHOST].update( config_overwrite[constants.LOCALHOST]) if constants.SERVERLESS in config_overwrite: if constants.SERVERLESS not in config_data or \ config_data[constants.SERVERLESS] is None: config_data[constants.SERVERLESS] = {} config_data[constants.SERVERLESS].update( config_overwrite[constants.SERVERLESS]) if constants.STANDALONE in config_overwrite: if constants.STANDALONE not in config_data or \ config_data[constants.STANDALONE] is None: config_data[constants.STANDALONE] = {} config_data[constants.STANDALONE].update( config_overwrite[constants.STANDALONE]) if config_data['lithops']['mode'] == constants.SERVERLESS: if constants.SERVERLESS not in config_data or \ config_data[constants.SERVERLESS] is None: config_data[constants.SERVERLESS] = {} if 'backend' not in config_data[constants.SERVERLESS]: config_data[constants.SERVERLESS][ 'backend'] = constants.SERVERLESS_BACKEND_DEFAULT sb = config_data[constants.SERVERLESS]['backend'] logger.debug("Loading Serverless backend module: {}".format(sb)) cb_config = importlib.import_module( 'lithops.serverless.backends.{}.config'.format(sb)) cb_config.load_config(config_data) verify_runtime_name(config_data[constants.SERVERLESS]['runtime']) elif config_data['lithops']['mode'] == constants.STANDALONE: if constants.STANDALONE not in config_data or \ config_data[constants.STANDALONE] is None: config_data[constants.STANDALONE] = {} if 'auto_dismantle' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'auto_dismantle'] = constants.STANDALONE_AUTO_DISMANTLE_DEFAULT if 'soft_dismantle_timeout' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'soft_dismantle_timeout'] = constants.STANDALONE_SOFT_DISMANTLE_TIMEOUT_DEFAULT if 'hard_dismantle_timeout' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'hard_dismantle_timeout'] = constants.STANDALONE_HARD_DISMANTLE_TIMEOUT_DEFAULT if 'backend' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'backend'] = constants.STANDALONE_BACKEND_DEFAULT if 'runtime' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'runtime'] = constants.STANDALONE_RUNTIME_DEFAULT sb = config_data[constants.STANDALONE]['backend'] logger.debug("Loading Standalone backend module: {}".format(sb)) sb_config = importlib.import_module( 'lithops.standalone.backends.{}.config'.format(sb)) sb_config.load_config(config_data) verify_runtime_name(config_data[constants.STANDALONE]['runtime']) elif config_data['lithops']['mode'] == constants.LOCALHOST: config_data['lithops']['workers'] = 1 config_data['lithops']['worker_processes'] = CPU_COUNT if constants.LOCALHOST not in config_data or \ config_data[constants.LOCALHOST] is None: config_data[constants.LOCALHOST] = {} if 'runtime' not in config_data[constants.LOCALHOST]: config_data[constants.LOCALHOST][ 'runtime'] = constants.LOCALHOST_RUNTIME_DEFAULT logger.debug("Loading compute backend module: localhost") verify_runtime_name(config_data[constants.LOCALHOST]['runtime']) mode = config_data['lithops']['mode'] if mode in config_overwrite and 'runtime' in config_overwrite[mode]: config_data[mode]['runtime'] = config_overwrite[mode]['runtime'] return default_storage_config(config_data)
def default_config(config_data=None, config_overwrite={}, load_storage_config=True): """ First checks .lithops_config then checks LITHOPS_CONFIG_FILE environment variable then ~/.lithops/config """ logger.info('Lithops v{}'.format(__version__)) config_data = copy.deepcopy(config_data) or load_config() if 'lithops' not in config_data or not config_data['lithops']: config_data['lithops'] = {} # overwrite values provided by the user if 'lithops' in config_overwrite: config_data['lithops'].update(config_overwrite['lithops']) backend = config_data['lithops'].get('backend') mode = config_data['lithops'].get('mode') if mode and not backend: if mode in config_data and 'backend' in config_data[mode]: config_data['lithops']['backend'] = config_data[mode]['backend'] else: config_data['lithops']['backend'] = get_default_backend(mode) elif backend: config_data['lithops']['mode'] = get_mode(backend) elif not backend and not mode: mode = config_data['lithops']['mode'] = constants.MODE_DEFAULT config_data['lithops']['backend'] = get_default_backend(mode) backend = config_data['lithops'].get('backend') mode = config_data['lithops'].get('mode') if mode == constants.LOCALHOST: logger.debug("Loading compute backend module: localhost") config_data['lithops']['workers'] = 1 if 'storage' not in config_data['lithops']: config_data['lithops']['storage'] = constants.LOCALHOST if 'worker_processes' not in config_data['lithops']: config_data['lithops']['worker_processes'] = CPU_COUNT if constants.LOCALHOST not in config_data or \ config_data[constants.LOCALHOST] is None: config_data[constants.LOCALHOST] = {} if 'runtime' in config_overwrite: config_data[ constants.LOCALHOST]['runtime'] = config_overwrite['runtime'] if 'runtime' not in config_data[constants.LOCALHOST]: config_data[constants.LOCALHOST][ 'runtime'] = constants.LOCALHOST_RUNTIME_DEFAULT verify_runtime_name(config_data[constants.LOCALHOST]['runtime']) elif mode == constants.SERVERLESS: if constants.SERVERLESS not in config_data or \ config_data[constants.SERVERLESS] is None: config_data[constants.SERVERLESS] = {} if 'runtime' in config_overwrite: config_data[backend]['runtime'] = config_overwrite['runtime'] logger.debug("Loading Serverless backend module: {}".format(backend)) cb_config = importlib.import_module( 'lithops.serverless.backends.{}.config'.format(backend)) cb_config.load_config(config_data) if 'runtime' in config_overwrite: config_data[backend]['runtime'] = config_overwrite['runtime'] if 'runtime_memory' in config_overwrite: config_data[backend]['runtime_memory'] = config_overwrite[ 'runtime_memory'] if 'remote_invoker' in config_overwrite: config_data[constants.SERVERLESS][ 'remote_invoker'] = config_overwrite['remote_invoker'] verify_runtime_name(config_data[backend]['runtime']) elif mode == constants.STANDALONE: if constants.STANDALONE not in config_data or \ config_data[constants.STANDALONE] is None: config_data[constants.STANDALONE] = {} if 'auto_dismantle' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'auto_dismantle'] = constants.STANDALONE_AUTO_DISMANTLE_DEFAULT if 'soft_dismantle_timeout' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'soft_dismantle_timeout'] = constants.STANDALONE_SOFT_DISMANTLE_TIMEOUT_DEFAULT if 'hard_dismantle_timeout' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'hard_dismantle_timeout'] = constants.STANDALONE_HARD_DISMANTLE_TIMEOUT_DEFAULT logger.debug("Loading Standalone backend module: {}".format(backend)) sb_config = importlib.import_module( 'lithops.standalone.backends.{}.config'.format(backend)) sb_config.load_config(config_data) if 'runtime' in config_overwrite: config_data[ constants.STANDALONE]['runtime'] = config_overwrite['runtime'] if 'runtime' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE][ 'runtime'] = constants.STANDALONE_RUNTIME_DEFAULT verify_runtime_name(config_data[constants.STANDALONE]['runtime']) if 'execution_timeout' not in config_data['lithops']: config_data['lithops'][ 'execution_timeout'] = constants.EXECUTION_TIMEOUT_DEFAULT if 'chunksize' not in config_data['lithops']: config_data['lithops']['chunksize'] = constants.CHUNKSIZE_DEFAULT if 'worker_processes' not in config_data['lithops']: config_data['lithops'][ 'worker_processes'] = constants.WORKER_PROCESSES_DEFAULT if 'monitoring' not in config_data['lithops']: config_data['lithops']['monitoring'] = constants.MONITORING_DEFAULT if load_storage_config: config_data = default_storage_config(config_data) if config_data['lithops'][ 'storage'] == constants.LOCALHOST and mode != constants.LOCALHOST: raise Exception( 'Localhost storage backend cannot be used in {} mode'.format( mode)) return config_data
def default_config(config_data=None, config_overwrite={}): """ First checks .lithops_config then checks LITHOPS_CONFIG_FILE environment variable then ~/.lithops/config """ logger.info('Lithops v{}'.format(__version__)) logger.debug("Loading configuration") if not config_data: if 'LITHOPS_CONFIG' in os.environ: config_data = json.loads(os.environ.get('LITHOPS_CONFIG')) else: config_filename = get_default_config_filename() if config_filename: config_data = load_yaml_config(config_filename) else: logger.debug("No config file found. Running on Localhost mode") config_data = {'lithops': {'mode': constants.LOCALHOST}} if 'lithops' not in config_data: config_data['lithops'] = {} if 'executor' in config_data['lithops']: logging.warning("'executor' key in lithopos section is deprecated, use 'mode' key instead") config_data['lithops']['mode'] = config_data['lithops']['executor'] # overwrite values provided by the user if 'lithops' in config_overwrite: config_data['lithops'].update(config_overwrite['lithops']) if constants.LOCALHOST in config_overwrite: if constants.LOCALHOST not in config_data or \ config_data[constants.LOCALHOST] is None: config_data[constants.LOCALHOST] = {} config_data[constants.LOCALHOST].update(config_overwrite[constants.LOCALHOST]) if constants.SERVERLESS in config_overwrite: if constants.SERVERLESS not in config_data or \ config_data[constants.SERVERLESS] is None: config_data[constants.SERVERLESS] = {} config_data[constants.SERVERLESS].update(config_overwrite[constants.SERVERLESS]) if constants.STANDALONE in config_overwrite: if constants.STANDALONE not in config_data or \ config_data[constants.STANDALONE] is None: config_data[constants.STANDALONE] = {} config_data[constants.STANDALONE].update(config_overwrite[constants.STANDALONE]) if 'mode' not in config_data['lithops']: config_data['lithops']['mode'] = constants.MODE_DEFAULT if 'execution_timeout' not in config_data['lithops']: config_data['lithops']['execution_timeout'] = constants.EXECUTION_TIMEOUT_DEFAULT if config_data['lithops']['mode'] == constants.SERVERLESS: if 'storage_bucket' not in config_data['lithops']: raise Exception("storage_bucket is mandatory in " "lithops section of the configuration") if constants.SERVERLESS not in config_data or \ config_data[constants.SERVERLESS] is None: config_data[constants.SERVERLESS] = {} if 'backend' not in config_data[constants.SERVERLESS]: config_data[constants.SERVERLESS]['backend'] = constants.SERVERLESS_BACKEND_DEFAULT sb = config_data[constants.SERVERLESS]['backend'] logger.debug("Loading Serverless backend module: {}".format(sb)) cb_config = importlib.import_module('lithops.serverless.backends.{}.config'.format(sb)) cb_config.load_config(config_data) verify_runtime_name(config_data[constants.SERVERLESS]['runtime']) elif config_data['lithops']['mode'] == constants.STANDALONE: if 'storage_bucket' not in config_data['lithops']: raise Exception("storage_bucket is mandatory in " "lithops section of the configuration") if constants.STANDALONE not in config_data or \ config_data[constants.STANDALONE] is None: config_data[constants.STANDALONE] = {} if 'auto_dismantle' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE]['auto_dismantle'] = constants.STANDALONE_AUTO_DISMANTLE_DEFAULT if 'soft_dismantle_timeout' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE]['soft_dismantle_timeout'] = constants.STANDALONE_SOFT_DISMANTLE_TIMEOUT_DEFAULT if 'hard_dismantle_timeout' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE]['hard_dismantle_timeout'] = constants.STANDALONE_HARD_DISMANTLE_TIMEOUT_DEFAULT if 'backend' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE]['backend'] = constants.STANDALONE_BACKEND_DEFAULT if 'runtime' not in config_data[constants.STANDALONE]: config_data[constants.STANDALONE]['runtime'] = constants.STANDALONE_RUNTIME_DEFAULT sb = config_data['standalone']['backend'] logger.debug("Loading Standalone backend module: {}".format(sb)) sb_config = importlib.import_module('lithops.standalone.backends.{}.config'.format(sb)) sb_config.load_config(config_data) verify_runtime_name(config_data[constants.STANDALONE]['runtime']) elif config_data['lithops']['mode'] == constants.LOCALHOST: if 'storage' not in config_data['lithops']: config_data['lithops']['storage'] = 'localhost' config_data['lithops']['storage_bucket'] = 'storage' if 'workers' not in config_data['lithops']: config_data['lithops']['workers'] = mp.cpu_count() if constants.LOCALHOST not in config_data or \ config_data[constants.LOCALHOST] is None: config_data[constants.LOCALHOST] = {} if 'runtime' not in config_data[constants.LOCALHOST]: config_data[constants.LOCALHOST]['runtime'] = constants.LOCALHOST_RUNTIME_DEFAULT verify_runtime_name(config_data[constants.LOCALHOST]['runtime']) if 'storage' not in config_data['lithops']: config_data['lithops']['storage'] = constants.STORAGE_BACKEND_DEFAULT sb = config_data['lithops']['storage'] logger.debug("Loading Storage backend module: {}".format(sb)) sb_config = importlib.import_module('lithops.storage.backends.{}.config'.format(sb)) sb_config.load_config(config_data) return config_data