def update(name, config, backend, storage):
""" Update a serverless runtime """
if config:
config = load_yaml_config(config)
verify_runtime_name(name)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def delete(name, config, backend, storage, debug):
""" delete a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime delete" command is only valid for serverless backends')
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def create(name, storage, backend, memory, timeout, config):
""" Create a serverless runtime """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
logger.info('Creating new lithops runtime: {}'.format(name))
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise ("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def run_worker(master_ip, job_key):
"""
Run a job
"""
global BUDGET_KEEPER
while True:
url = 'http://{}:{}/get-task/{}'.format(master_ip, STANDALONE_SERVICE_PORT, job_key)
logger.info('Getting task from {}'.format(url))
resp = requests.get(url)
if resp.status_code != 200:
logger.info('All tasks completed'.format(url))
return
job_payload = resp.json()
logger.info(job_payload)
logger.info('Got tasks {}'.format(', '.join(job_payload['call_ids'])))
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception:
return
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running'
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime})
localhost_handler.invoke(job_payload)
wait_job_completed(job_key)
def deploy(name, storage, backend, memory, timeout, config, debug):
""" deploy a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime create" command is only valid for serverless backends')
logger.info('Creating new lithops runtime: {}'.format(name))
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.deploy_runtime(name, mem, timeout=to)
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def update(name, config, backend, storage, debug):
""" Update a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime update" command is only valid for serverless backends')
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.deploy_runtime(runtime[0], runtime[1], timeout)
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def run():
"""
Run a job
"""
global BUDGET_KEEPER
job_payload = flask.request.get_json(force=True, silent=True)
if job_payload and not isinstance(job_payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running'
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
localhost_handler = LocalhostHandler({
'runtime': runtime,
'pull_runtime': pull_runtime
})
localhost_handler.run_job(job_payload)
act_id = str(uuid.uuid4()).replace('-', '')[:12]
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def delete(name, config, backend, storage):
""" delete a serverless runtime """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def build(name, file, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def run():
"""
Run a job locally, in consume mode
"""
global budget_keeper
global work_queues
global exec_mode
global localhost_manager_process
job_payload = flask.request.get_json(force=True, silent=True)
if job_payload and not isinstance(job_payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
job_key = job_payload['job_key']
logger.debug('Received job {}'.format(job_key))
budget_keeper.last_usage_time = time.time()
budget_keeper.update_config(job_payload['config']['standalone'])
budget_keeper.jobs[job_key] = 'running'
exec_mode = job_payload['config']['standalone'].get('exec_mode', 'consume')
if exec_mode == 'consume':
# Consume mode runs jobs in this master VM
work_queue_name = 'local'
work_queue = work_queues.setdefault(work_queue_name, queue.Queue())
if not localhost_manager_process:
logger.debug('Starting manager process for localhost jobs')
lmp = Thread(target=run_job_local,
args=(work_queue, ),
daemon=True)
lmp.start()
localhost_manager_process = lmp
logger.debug(f'Putting job {job_key} into master queue')
work_queue.put(job_payload)
elif exec_mode in ['create', 'reuse']:
# Create and reuse mode runs jobs on woker VMs
logger.debug(f'Starting process for job {job_key}')
work_queue_name = job_key if exec_mode == 'create' else REUSE_WORK_QUEUE_NAME
work_queue = work_queues.setdefault(work_queue_name, queue.Queue())
Thread(target=start_workers,
args=(job_payload, work_queue_name)).start()
Thread(target=run_job_worker,
args=(job_payload, work_queue),
daemon=True).start()
act_id = str(uuid.uuid4()).replace('-', '')[:12]
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def run():
"""
Run a job locally, in consume mode
"""
global BUDGET_KEEPER
global WORK_QUEUES
global JOB_PROCESSES
job_payload = flask.request.get_json(force=True, silent=True)
if job_payload and not isinstance(job_payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
job_key = job_payload['job_key']
logger.info('Received job {}'.format(job_key))
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_key] = 'running'
exec_mode = job_payload['config']['standalone'].get('exec_mode', 'consume')
if exec_mode == 'consume':
# Consume mode runs the job locally
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
try:
localhost_handler = LocalhostHandler({
'runtime': runtime,
'pull_runtime': pull_runtime
})
localhost_handler.invoke(job_payload)
except Exception as e:
logger.error(e)
elif exec_mode == 'create':
# Create mode runs the job in worker VMs
work_queue = MP_MANAGER.Queue()
WORK_QUEUES[job_key] = work_queue
jp = mp.Process(target=run_job_process, args=(job_payload, work_queue))
jp.daemon = True
jp.start()
JOB_PROCESSES[job_key] = jp
act_id = str(uuid.uuid4()).replace('-', '')[:12]
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def run_worker(master_ip, work_queue):
"""
Run a job
"""
global budget_keeper
global localhos_handler
global last_job_key
pull_runtime = stanbdalone_config.get('pull_runtime', False)
localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime})
while True:
url = f'http://{master_ip}:{SA_SERVICE_PORT}/get-task/{work_queue}'
logger.debug(f'Getting task from {url}')
try:
resp = requests.get(url)
except Exception:
time.sleep(1)
continue
if resp.status_code != 200:
if stanbdalone_config.get('exec_mode') == 'reuse':
time.sleep(1)
continue
else:
logger.debug('All tasks completed'.format(url))
return
job_payload = resp.json()
logger.debug(job_payload)
logger.debug('Got tasks {}'.format(', '.join(job_payload['call_ids'])))
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception:
return
job_key = job_payload['job_key']
last_job_key = job_key
budget_keeper.last_usage_time = time.time()
budget_keeper.update_config(job_payload['config']['standalone'])
budget_keeper.jobs[job_key] = 'running'
try:
localhos_handler.invoke(job_payload)
except Exception as e:
logger.error(e)
wait_job_completed(job_key)
def delete(name, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def build(name, file, config, backend):
""" build a serverless runtime. """
verify_runtime_name(name)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def build(ctx, name, file, config, backend):
""" build a serverless runtime. """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, runtime_name=name)
config = default_config(config, config_ow, load_storage_config=False)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops build" command is only valid for serverless backends')
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, None)
compute_handler.build_runtime(name, file, ctx.args)
def preinstalls():
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = message['runtime']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
localhost_handler = LocalhostHandler(message)
runtime_meta = localhost_handler.create_runtime(runtime)
response = flask.jsonify(runtime_meta)
response.status_code = 200
return response
def preinstalls():
payload = flask.request.get_json(force=True, silent=True)
if payload and not isinstance(payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = payload['runtime']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime})
runtime_meta = localhost_handler.create_runtime(runtime)
response = flask.jsonify(runtime_meta)
response.status_code = 200
return response
def run_create():
"""
Runs a given job remotely in workers, in create mode
"""
global BUDGET_KEEPER
logger.info('Running job on worker VMs')
job_payload = flask.request.get_json(force=True, silent=True)
if job_payload and not isinstance(job_payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
job_key = job_payload['job_key']
call_ids = job_payload['call_ids']
chunksize = job_payload['chunksize']
workers = job_payload['woreker_instances']
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_key] = 'running'
with ThreadPoolExecutor(len(workers)) as executor:
for call_ids_range in iterchunks(call_ids, chunksize):
worker_info = workers.pop(0)
executor.submit(run_job_on_worker, worker_info, call_ids_range,
copy.deepcopy(job_payload))
done = os.path.join(JOBS_DIR, job_key + '.done')
Path(done).touch()
act_id = str(uuid.uuid4()).replace('-', '')[:12]
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def delete(name, config, backend):
""" delete a serverless runtime """
verify_runtime_name(name)
setup_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def run():
"""
Run a job
"""
global last_usage_time
global backend_handler
global jobs
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = message['job_description']['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
last_usage_time = time.time()
standalone_config = message['config']['standalone']
backend_handler.auto_dismantle = standalone_config['auto_dismantle']
backend_handler.soft_dismantle_timeout = standalone_config[
'soft_dismantle_timeout']
backend_handler.hard_dismantle_timeout = standalone_config[
'hard_dismantle_timeout']
act_id = str(uuid.uuid4()).replace('-', '')[:12]
executor_id = message['executor_id']
job_id = message['job_id']
job_key = create_job_key(executor_id, job_id)
jobs[job_key] = 'running'
localhost_handler = LocalhostHandler({'runtime': runtime})
localhost_handler.run_job(message)
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def preinstalls():
payload = flask.request.get_json(force=True, silent=True)
if payload and not isinstance(payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = payload['runtime']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
pull_runtime = standalone_config.get('pull_runtime', False)
lh = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime})
runtime_meta = lh.create_runtime(runtime)
if 'lithops_version' in runtime_meta:
logger.debug("Runtime metdata extracted correctly: Lithops "
f"{runtime_meta['lithops_version']}")
response = flask.jsonify(runtime_meta)
response.status_code = 200
return response
def update(name, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
timeout = config['lithops']['runtime_timeout']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def build(name, file, config, backend):
""" build a serverless runtime. """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
compute_handler.build_runtime(name, file)
def default_config(config_data=None, config_overwrite={}):
"""
First checks .lithops_config
then checks LITHOPS_CONFIG_FILE environment variable
then ~/.lithops/config
"""
logger.info('Lithops v{}'.format(__version__))
config_data = copy.deepcopy(config_data) or load_config()
if 'lithops' not in config_data or not config_data['lithops']:
config_data['lithops'] = {}
if 'mode' not in config_data['lithops']:
config_data['lithops']['mode'] = constants.MODE_DEFAULT
if 'execution_timeout' not in config_data['lithops']:
config_data['lithops'][
'execution_timeout'] = constants.EXECUTION_TIMEOUT_DEFAULT
if 'chunksize' not in config_data['lithops']:
config_data['lithops']['chunksize'] = constants.CHUNKSIZE_DEFAULT
if 'worker_processes' not in config_data['lithops']:
config_data['lithops'][
'worker_processes'] = constants.WORKER_PROCESSES_DEFAULT
if 'monitoring' not in config_data['lithops']:
config_data['lithops']['monitoring'] = constants.MONITORING_DEFAULT
# overwrite values provided by the user
if 'lithops' in config_overwrite:
config_data['lithops'].update(config_overwrite['lithops'])
if constants.LOCALHOST in config_overwrite:
if constants.LOCALHOST not in config_data or \
config_data[constants.LOCALHOST] is None:
config_data[constants.LOCALHOST] = {}
config_data[constants.LOCALHOST].update(
config_overwrite[constants.LOCALHOST])
if constants.SERVERLESS in config_overwrite:
if constants.SERVERLESS not in config_data or \
config_data[constants.SERVERLESS] is None:
config_data[constants.SERVERLESS] = {}
config_data[constants.SERVERLESS].update(
config_overwrite[constants.SERVERLESS])
if constants.STANDALONE in config_overwrite:
if constants.STANDALONE not in config_data or \
config_data[constants.STANDALONE] is None:
config_data[constants.STANDALONE] = {}
config_data[constants.STANDALONE].update(
config_overwrite[constants.STANDALONE])
if config_data['lithops']['mode'] == constants.SERVERLESS:
if constants.SERVERLESS not in config_data or \
config_data[constants.SERVERLESS] is None:
config_data[constants.SERVERLESS] = {}
if 'backend' not in config_data[constants.SERVERLESS]:
config_data[constants.SERVERLESS][
'backend'] = constants.SERVERLESS_BACKEND_DEFAULT
sb = config_data[constants.SERVERLESS]['backend']
logger.debug("Loading Serverless backend module: {}".format(sb))
cb_config = importlib.import_module(
'lithops.serverless.backends.{}.config'.format(sb))
cb_config.load_config(config_data)
verify_runtime_name(config_data[constants.SERVERLESS]['runtime'])
elif config_data['lithops']['mode'] == constants.STANDALONE:
if constants.STANDALONE not in config_data or \
config_data[constants.STANDALONE] is None:
config_data[constants.STANDALONE] = {}
if 'auto_dismantle' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'auto_dismantle'] = constants.STANDALONE_AUTO_DISMANTLE_DEFAULT
if 'soft_dismantle_timeout' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'soft_dismantle_timeout'] = constants.STANDALONE_SOFT_DISMANTLE_TIMEOUT_DEFAULT
if 'hard_dismantle_timeout' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'hard_dismantle_timeout'] = constants.STANDALONE_HARD_DISMANTLE_TIMEOUT_DEFAULT
if 'backend' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'backend'] = constants.STANDALONE_BACKEND_DEFAULT
if 'runtime' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'runtime'] = constants.STANDALONE_RUNTIME_DEFAULT
sb = config_data[constants.STANDALONE]['backend']
logger.debug("Loading Standalone backend module: {}".format(sb))
sb_config = importlib.import_module(
'lithops.standalone.backends.{}.config'.format(sb))
sb_config.load_config(config_data)
verify_runtime_name(config_data[constants.STANDALONE]['runtime'])
elif config_data['lithops']['mode'] == constants.LOCALHOST:
config_data['lithops']['workers'] = 1
config_data['lithops']['worker_processes'] = CPU_COUNT
if constants.LOCALHOST not in config_data or \
config_data[constants.LOCALHOST] is None:
config_data[constants.LOCALHOST] = {}
if 'runtime' not in config_data[constants.LOCALHOST]:
config_data[constants.LOCALHOST][
'runtime'] = constants.LOCALHOST_RUNTIME_DEFAULT
logger.debug("Loading compute backend module: localhost")
verify_runtime_name(config_data[constants.LOCALHOST]['runtime'])
mode = config_data['lithops']['mode']
if mode in config_overwrite and 'runtime' in config_overwrite[mode]:
config_data[mode]['runtime'] = config_overwrite[mode]['runtime']
return default_storage_config(config_data)
def default_config(config_data=None,
config_overwrite={},
load_storage_config=True):
"""
First checks .lithops_config
then checks LITHOPS_CONFIG_FILE environment variable
then ~/.lithops/config
"""
logger.info('Lithops v{}'.format(__version__))
config_data = copy.deepcopy(config_data) or load_config()
if 'lithops' not in config_data or not config_data['lithops']:
config_data['lithops'] = {}
# overwrite values provided by the user
if 'lithops' in config_overwrite:
config_data['lithops'].update(config_overwrite['lithops'])
backend = config_data['lithops'].get('backend')
mode = config_data['lithops'].get('mode')
if mode and not backend:
if mode in config_data and 'backend' in config_data[mode]:
config_data['lithops']['backend'] = config_data[mode]['backend']
else:
config_data['lithops']['backend'] = get_default_backend(mode)
elif backend:
config_data['lithops']['mode'] = get_mode(backend)
elif not backend and not mode:
mode = config_data['lithops']['mode'] = constants.MODE_DEFAULT
config_data['lithops']['backend'] = get_default_backend(mode)
backend = config_data['lithops'].get('backend')
mode = config_data['lithops'].get('mode')
if mode == constants.LOCALHOST:
logger.debug("Loading compute backend module: localhost")
config_data['lithops']['workers'] = 1
if 'storage' not in config_data['lithops']:
config_data['lithops']['storage'] = constants.LOCALHOST
if 'worker_processes' not in config_data['lithops']:
config_data['lithops']['worker_processes'] = CPU_COUNT
if constants.LOCALHOST not in config_data or \
config_data[constants.LOCALHOST] is None:
config_data[constants.LOCALHOST] = {}
if 'runtime' in config_overwrite:
config_data[
constants.LOCALHOST]['runtime'] = config_overwrite['runtime']
if 'runtime' not in config_data[constants.LOCALHOST]:
config_data[constants.LOCALHOST][
'runtime'] = constants.LOCALHOST_RUNTIME_DEFAULT
verify_runtime_name(config_data[constants.LOCALHOST]['runtime'])
elif mode == constants.SERVERLESS:
if constants.SERVERLESS not in config_data or \
config_data[constants.SERVERLESS] is None:
config_data[constants.SERVERLESS] = {}
if 'runtime' in config_overwrite:
config_data[backend]['runtime'] = config_overwrite['runtime']
logger.debug("Loading Serverless backend module: {}".format(backend))
cb_config = importlib.import_module(
'lithops.serverless.backends.{}.config'.format(backend))
cb_config.load_config(config_data)
if 'runtime' in config_overwrite:
config_data[backend]['runtime'] = config_overwrite['runtime']
if 'runtime_memory' in config_overwrite:
config_data[backend]['runtime_memory'] = config_overwrite[
'runtime_memory']
if 'remote_invoker' in config_overwrite:
config_data[constants.SERVERLESS][
'remote_invoker'] = config_overwrite['remote_invoker']
verify_runtime_name(config_data[backend]['runtime'])
elif mode == constants.STANDALONE:
if constants.STANDALONE not in config_data or \
config_data[constants.STANDALONE] is None:
config_data[constants.STANDALONE] = {}
if 'auto_dismantle' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'auto_dismantle'] = constants.STANDALONE_AUTO_DISMANTLE_DEFAULT
if 'soft_dismantle_timeout' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'soft_dismantle_timeout'] = constants.STANDALONE_SOFT_DISMANTLE_TIMEOUT_DEFAULT
if 'hard_dismantle_timeout' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'hard_dismantle_timeout'] = constants.STANDALONE_HARD_DISMANTLE_TIMEOUT_DEFAULT
logger.debug("Loading Standalone backend module: {}".format(backend))
sb_config = importlib.import_module(
'lithops.standalone.backends.{}.config'.format(backend))
sb_config.load_config(config_data)
if 'runtime' in config_overwrite:
config_data[
constants.STANDALONE]['runtime'] = config_overwrite['runtime']
if 'runtime' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE][
'runtime'] = constants.STANDALONE_RUNTIME_DEFAULT
verify_runtime_name(config_data[constants.STANDALONE]['runtime'])
if 'execution_timeout' not in config_data['lithops']:
config_data['lithops'][
'execution_timeout'] = constants.EXECUTION_TIMEOUT_DEFAULT
if 'chunksize' not in config_data['lithops']:
config_data['lithops']['chunksize'] = constants.CHUNKSIZE_DEFAULT
if 'worker_processes' not in config_data['lithops']:
config_data['lithops'][
'worker_processes'] = constants.WORKER_PROCESSES_DEFAULT
if 'monitoring' not in config_data['lithops']:
config_data['lithops']['monitoring'] = constants.MONITORING_DEFAULT
if load_storage_config:
config_data = default_storage_config(config_data)
if config_data['lithops'][
'storage'] == constants.LOCALHOST and mode != constants.LOCALHOST:
raise Exception(
'Localhost storage backend cannot be used in {} mode'.format(
mode))
return config_data
def default_config(config_data=None, config_overwrite={}):
"""
First checks .lithops_config
then checks LITHOPS_CONFIG_FILE environment variable
then ~/.lithops/config
"""
logger.info('Lithops v{}'.format(__version__))
logger.debug("Loading configuration")
if not config_data:
if 'LITHOPS_CONFIG' in os.environ:
config_data = json.loads(os.environ.get('LITHOPS_CONFIG'))
else:
config_filename = get_default_config_filename()
if config_filename:
config_data = load_yaml_config(config_filename)
else:
logger.debug("No config file found. Running on Localhost mode")
config_data = {'lithops': {'mode': constants.LOCALHOST}}
if 'lithops' not in config_data:
config_data['lithops'] = {}
if 'executor' in config_data['lithops']:
logging.warning("'executor' key in lithopos section is deprecated, use 'mode' key instead")
config_data['lithops']['mode'] = config_data['lithops']['executor']
# overwrite values provided by the user
if 'lithops' in config_overwrite:
config_data['lithops'].update(config_overwrite['lithops'])
if constants.LOCALHOST in config_overwrite:
if constants.LOCALHOST not in config_data or \
config_data[constants.LOCALHOST] is None:
config_data[constants.LOCALHOST] = {}
config_data[constants.LOCALHOST].update(config_overwrite[constants.LOCALHOST])
if constants.SERVERLESS in config_overwrite:
if constants.SERVERLESS not in config_data or \
config_data[constants.SERVERLESS] is None:
config_data[constants.SERVERLESS] = {}
config_data[constants.SERVERLESS].update(config_overwrite[constants.SERVERLESS])
if constants.STANDALONE in config_overwrite:
if constants.STANDALONE not in config_data or \
config_data[constants.STANDALONE] is None:
config_data[constants.STANDALONE] = {}
config_data[constants.STANDALONE].update(config_overwrite[constants.STANDALONE])
if 'mode' not in config_data['lithops']:
config_data['lithops']['mode'] = constants.MODE_DEFAULT
if 'execution_timeout' not in config_data['lithops']:
config_data['lithops']['execution_timeout'] = constants.EXECUTION_TIMEOUT_DEFAULT
if config_data['lithops']['mode'] == constants.SERVERLESS:
if 'storage_bucket' not in config_data['lithops']:
raise Exception("storage_bucket is mandatory in "
"lithops section of the configuration")
if constants.SERVERLESS not in config_data or \
config_data[constants.SERVERLESS] is None:
config_data[constants.SERVERLESS] = {}
if 'backend' not in config_data[constants.SERVERLESS]:
config_data[constants.SERVERLESS]['backend'] = constants.SERVERLESS_BACKEND_DEFAULT
sb = config_data[constants.SERVERLESS]['backend']
logger.debug("Loading Serverless backend module: {}".format(sb))
cb_config = importlib.import_module('lithops.serverless.backends.{}.config'.format(sb))
cb_config.load_config(config_data)
verify_runtime_name(config_data[constants.SERVERLESS]['runtime'])
elif config_data['lithops']['mode'] == constants.STANDALONE:
if 'storage_bucket' not in config_data['lithops']:
raise Exception("storage_bucket is mandatory in "
"lithops section of the configuration")
if constants.STANDALONE not in config_data or \
config_data[constants.STANDALONE] is None:
config_data[constants.STANDALONE] = {}
if 'auto_dismantle' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE]['auto_dismantle'] = constants.STANDALONE_AUTO_DISMANTLE_DEFAULT
if 'soft_dismantle_timeout' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE]['soft_dismantle_timeout'] = constants.STANDALONE_SOFT_DISMANTLE_TIMEOUT_DEFAULT
if 'hard_dismantle_timeout' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE]['hard_dismantle_timeout'] = constants.STANDALONE_HARD_DISMANTLE_TIMEOUT_DEFAULT
if 'backend' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE]['backend'] = constants.STANDALONE_BACKEND_DEFAULT
if 'runtime' not in config_data[constants.STANDALONE]:
config_data[constants.STANDALONE]['runtime'] = constants.STANDALONE_RUNTIME_DEFAULT
sb = config_data['standalone']['backend']
logger.debug("Loading Standalone backend module: {}".format(sb))
sb_config = importlib.import_module('lithops.standalone.backends.{}.config'.format(sb))
sb_config.load_config(config_data)
verify_runtime_name(config_data[constants.STANDALONE]['runtime'])
elif config_data['lithops']['mode'] == constants.LOCALHOST:
if 'storage' not in config_data['lithops']:
config_data['lithops']['storage'] = 'localhost'
config_data['lithops']['storage_bucket'] = 'storage'
if 'workers' not in config_data['lithops']:
config_data['lithops']['workers'] = mp.cpu_count()
if constants.LOCALHOST not in config_data or \
config_data[constants.LOCALHOST] is None:
config_data[constants.LOCALHOST] = {}
if 'runtime' not in config_data[constants.LOCALHOST]:
config_data[constants.LOCALHOST]['runtime'] = constants.LOCALHOST_RUNTIME_DEFAULT
verify_runtime_name(config_data[constants.LOCALHOST]['runtime'])
if 'storage' not in config_data['lithops']:
config_data['lithops']['storage'] = constants.STORAGE_BACKEND_DEFAULT
sb = config_data['lithops']['storage']
logger.debug("Loading Storage backend module: {}".format(sb))
sb_config = importlib.import_module('lithops.storage.backends.{}.config'.format(sb))
sb_config.load_config(config_data)
return config_data
