def clean_all(config=None):
logger.info('Cleaning all Lithops information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
default_executor = config['lithops']['executor']
if default_executor == 'localhost':
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif default_executor == 'serverless':
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif default_executor == 'standalone':
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(STORAGE_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def create(name, mode, memory, timeout, config):
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
runtime_key = compute_handler.get_runtime_key(name)
runtime_meta = compute_handler.create_runtime(name)
elif mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
runtime_key = compute_handler.get_runtime_key(name)
runtime_meta = compute_handler.create_runtime(name)
else:
raise Exception('Unknown execution mode {}'.format(mode))
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def create(name, storage, backend, memory, timeout, config):
""" Create a serverless runtime """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
logger.info('Creating new lithops runtime: {}'.format(name))
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise ("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def delete(name, config, backend, storage):
""" delete a serverless runtime """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def run_tests(tests, config=None, group=None, backend=None, storage=None, fail_fast=False,
keep_datasets=False):
global CONFIG, STORAGE_CONFIG, STORAGE
config_ow = {'lithops': {}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow['lithops']['backend'] = backend
CONFIG = default_config(config, config_ow)
STORAGE_CONFIG = extract_storage_config(CONFIG)
STORAGE = Storage(storage_config=STORAGE_CONFIG)
init_test_variables()
suite = unittest.TestSuite()
config_suite(suite, tests, group)
words_in_data_set = upload_data_sets() # uploads datasets and returns word count
main_util.init_config(CONFIG, STORAGE, STORAGE_CONFIG, words_in_data_set, TEST_FILES_URLS)
runner = unittest.TextTestRunner(verbosity=2, failfast=fail_fast)
tests_results = runner.run(suite)
# removes previously uploaded datasets from storage.
if not keep_datasets:
clean_tests(STORAGE, STORAGE_CONFIG, PREFIX)
if not tests_results.wasSuccessful(): # Fails github workflow action to reject merge to repository
sys.tracebacklimit = 0 # avoid displaying redundant stack track-back info
raise Exception("--------Test procedure failed. Merge rejected--------")
def update(name, config, backend, storage, debug):
""" Update a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime update" command is only valid for serverless backends')
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.deploy_runtime(runtime[0], runtime[1], timeout)
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def run_tests(test_to_run, config=None, mode=None, backend=None, storage=None):
global CONFIG, STORAGE_CONFIG, STORAGE
mode = mode or get_mode(backend, config)
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
CONFIG = default_config(config, config_ow)
STORAGE_CONFIG = extract_storage_config(CONFIG)
STORAGE = Storage(storage_config=STORAGE_CONFIG)
suite = unittest.TestSuite()
if test_to_run == 'all':
suite.addTest(unittest.makeSuite(TestLithops))
else:
try:
suite.addTest(TestLithops(test_to_run))
except ValueError:
print("unknown test, use: --help")
sys.exit()
runner = unittest.TextTestRunner()
runner.run(suite)
def delete(name, config, backend, storage, debug):
""" delete a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime delete" command is only valid for serverless backends')
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def list_runtimes(config, backend, debug):
""" list all deployed serverless runtime. """
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, runtime_name='None')
config = default_config(config, config_ow, load_storage_config=False)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime list" command is only valid for serverless backends')
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, None)
runtimes = compute_handler.list_runtimes()
if runtimes:
width = max([len(runtime[0]) for runtime in runtimes])
print('\n{:{width}} \t {}'.format('Runtime Name', 'Memory Size (MB)', width=width))
print('-' * width, '\t', '-' * 20)
for runtime in runtimes:
name = runtime[0]
mem = runtime[1]
print('{:{width}} \t {}'.format(name, mem, width=width))
print()
print('Total runtimes: {}'.format(len(runtimes)))
else:
width = 10
print('\n{:{width}} \t {}'.format('Runtime Name', 'Memory Size (MB)', width=width))
print('-' * width, '\t', '-' * 20)
print('\nNo runtimes deployed')
def deploy(name, storage, backend, memory, timeout, config, debug):
""" deploy a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime create" command is only valid for serverless backends')
logger.info('Creating new lithops runtime: {}'.format(name))
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.deploy_runtime(name, mem, timeout=to)
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def update(name, config, backend, storage):
""" Update a serverless runtime """
if config:
config = load_yaml_config(config)
verify_runtime_name(name)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def clean(mode, config, debug):
log_level = 'INFO' if not debug else 'DEBUG'
setup_logger(log_level)
logger.info('Cleaning all Lithops information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def build(name, file, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def __init__(self, ds_config, db_config, use_db_cache=True, use_ds_cache=True, hybrid_impl='auto'):
self.config = default_config()
self.ds_config = ds_config
self.db_config = db_config
self.use_db_cache = use_db_cache
self.use_ds_cache = use_ds_cache
if hybrid_impl == 'auto':
self.hybrid_impl = (
self.config['lithops']['mode'] == 'localhost'
or self.config['lithops']['mode'] == 'serverless' and 'ibm_vpc' in self.config
)
if self.hybrid_impl:
logger.info(f'Using the Hybrid implementation')
else:
logger.info(f'Using the pure Serverless implementation')
else:
self.hybrid_impl = hybrid_impl
lithops_bucket = self.config['lithops']['storage_bucket']
self.ds_bucket = self.config.get('storage', {}).get('ds_bucket', lithops_bucket)
self.lithops_executor = lithops.FunctionExecutor(config=self.config, runtime_memory=2048)
if self.hybrid_impl:
if self.config['lithops']['mode'] == 'localhost':
self.lithops_vm_executor = self.lithops_executor
else:
self.lithops_vm_executor = lithops.StandaloneExecutor(config=self.config)
self.storage = Storage(config=self.config)
cache_namespace = 'vm' if hybrid_impl else 'function'
self.cacher = PipelineCacher(
self.storage, lithops_bucket, cache_namespace, self.ds_config["name"], self.db_config["name"]
)
if not self.use_db_cache or not self.use_ds_cache:
self.cacher.clean(database=not self.use_db_cache, dataset=not self.use_ds_cache)
stats_path_cache_key = ':ds/:db/stats_path.cache'
if self.cacher.exists(stats_path_cache_key):
self.stats_path = self.cacher.load(stats_path_cache_key)
PipelineStats.path = self.stats_path
logger.info(f'Using cached {self.stats_path} for statistics')
else:
PipelineStats.init()
self.stats_path = PipelineStats.path
self.cacher.save(self.stats_path, stats_path_cache_key)
logger.info(f'Initialised {self.stats_path} for statistics')
self.ds_segm_size_mb = 128
self.image_gen_config = {
"q": 99,
"do_preprocessing": False,
"nlevels": 30,
"ppm": 3.0
}
def delete_runtime(name, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def __init__(self, config=None):
if isinstance(config, str):
config = load_yaml_config(config)
self._config = extract_storage_config(config)
elif isinstance(config, dict):
if 'lithops' in config:
self._config = extract_storage_config(config)
else:
self._config = config
else:
self._config = extract_storage_config(default_config())
super().__init__(self._config)
def delete(name, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def build(name, file, config, backend):
""" build a serverless runtime. """
verify_runtime_name(name)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def build(ctx, name, file, config, backend):
""" build a serverless runtime. """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, runtime_name=name)
config = default_config(config, config_ow, load_storage_config=False)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops build" command is only valid for serverless backends')
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, None)
compute_handler.build_runtime(name, file, ctx.args)
def __init__(self,
storage_config=None,
lithops_config=None,
storage_backend=None,
bucket=None):
if lithops_config is None:
self.lithops_config = config.default_config()
if storage_config is None:
self.storage_config = config.extract_storage_config(
self.lithops_config)
if storage_backend is None:
self.storage_backend = self.storage_config['backend']
if bucket is not None:
self.storage_config['bucket'] = bucket
super().__init__(storage_config=self.storage_config,
lithops_config=self.lithops_config,
storage_backend=self.storage_backend)
def run_tests(test_to_run, config=None):
global CONFIG, STORAGE_CONFIG, STORAGE
CONFIG = json.load(args.config) if config else default_config()
STORAGE_CONFIG = extract_storage_config(CONFIG)
STORAGE = InternalStorage(STORAGE_CONFIG).storage
suite = unittest.TestSuite()
if test_to_run == 'all':
suite.addTest(unittest.makeSuite(TestLithops))
else:
try:
suite.addTest(TestLithops(test_to_run))
except ValueError:
print("unknown test, use: --help")
sys.exit()
runner = unittest.TextTestRunner()
runner.run(suite)
def clean_all(config=None):
logger.info('Cleaning all Lithops information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
# Clean localhost executor temp dirs
shutil.rmtree(STORAGE_FOLDER, ignore_errors=True)
shutil.rmtree(DOCKER_FOLDER, ignore_errors=True)
# Clean object storage temp dirs
compute_handler.delete_all_runtimes()
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def attach(config, backend, start, debug):
"""Create or attach to a SSH session on Lithops master VM"""
if config:
config = load_yaml_config(config)
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
config_ow = set_config_ow(backend)
config = default_config(config, config_ow)
if config['lithops']['mode'] != STANDALONE:
raise Exception(
'lithops attach method is only available for standalone backends')
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.init()
if start:
compute_handler.backend.master.start()
master_ip = compute_handler.backend.master.get_public_ip()
user = compute_handler.backend.master.ssh_credentials['username']
key_file = compute_handler.backend.master.ssh_credentials[
'key_filename'] or '~/.ssh/id_rsa'
key_file = os.path.abspath(os.path.expanduser(key_file))
if not os.path.exists(key_file):
raise Exception(f'Private key file {key_file} does not exists')
print(f'Got master VM public IP address: {master_ip}')
print(f'Loading ssh private key from: {key_file}')
print('Creating SSH Connection to lithops master VM')
cmd = (
'ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" '
f'-i {key_file} {user}@{master_ip}')
compute_handler.backend.master.wait_ready()
sp.run(shlex.split(cmd))
def clean(config, mode, backend, storage, debug):
if config:
config = load_yaml_config(config)
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
logger.info('Cleaning all Lithops information')
mode = mode or get_mode(backend, config)
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def delete(name, config, backend):
""" delete a serverless runtime """
verify_runtime_name(name)
setup_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def update(name, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
timeout = config['lithops']['runtime_timeout']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def create_runtime(name, memory=None, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
memory = config['lithops']['runtime_memory'] if not memory else memory
timeout = config['lithops']['runtime_timeout']
logger.info('Creating runtime: {}, memory: {}'.format(name, memory))
runtime_key = compute_handler.get_runtime_key(name, memory)
runtime_meta = compute_handler.create_runtime(name,
memory,
timeout=timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise ("Unable to upload 'preinstalled-modules' file into {}".format(
internal_storage.backend))
def build(name, file, config, backend):
""" build a serverless runtime. """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
compute_handler.build_runtime(name, file)
def __init__(self,
mode=None,
config=None,
backend=None,
storage=None,
runtime=None,
runtime_memory=None,
monitoring=None,
workers=None,
remote_invoker=None,
log_level=False):
""" Create a FunctionExecutor Class """
if mode and mode not in [LOCALHOST, SERVERLESS, STANDALONE]:
raise Exception("Function executor mode must be one of '{}', '{}' "
"or '{}'".format(LOCALHOST, SERVERLESS,
STANDALONE))
self.is_lithops_worker = is_lithops_worker()
# setup lithops logging
if not self.is_lithops_worker:
# if is lithops worker, logging has been set up in entry_point.py
if log_level:
setup_lithops_logger(log_level)
elif log_level is False and logger.getEffectiveLevel(
) == logging.WARNING:
# Set default logging from config
setup_lithops_logger(*get_log_info(config))
# load mode of execution
mode = mode or get_mode(backend, config)
config_ow = {'lithops': {'mode': mode}, mode: {}}
# overwrite user-provided parameters
if runtime is not None:
config_ow[mode]['runtime'] = runtime
if backend is not None:
config_ow[mode]['backend'] = backend
if runtime_memory is not None:
config_ow[mode]['runtime_memory'] = int(runtime_memory)
if remote_invoker is not None:
config_ow[mode]['remote_invoker'] = remote_invoker
if storage is not None:
config_ow['lithops']['storage'] = storage
if workers is not None:
config_ow['lithops']['workers'] = workers
if monitoring is not None:
config_ow['lithops']['monitoring'] = monitoring
self.config = default_config(copy.deepcopy(config), config_ow)
self.executor_id = create_executor_id()
self.data_cleaner = self.config['lithops'].get('data_cleaner', True)
if self.data_cleaner and not self.is_lithops_worker:
spawn_cleaner = int(self.executor_id.split('-')[1]) == 0
atexit.register(self.clean,
spawn_cleaner=spawn_cleaner,
clean_cloudobjects=False)
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.storage = self.internal_storage.storage
self.futures = []
self.cleaned_jobs = set()
self.total_jobs = 0
self.last_call = None
if mode == LOCALHOST:
localhost_config = extract_localhost_config(self.config)
self.compute_handler = LocalhostHandler(localhost_config)
elif mode == SERVERLESS:
serverless_config = extract_serverless_config(self.config)
self.compute_handler = ServerlessHandler(serverless_config,
self.internal_storage)
elif mode == STANDALONE:
standalone_config = extract_standalone_config(self.config)
self.compute_handler = StandaloneHandler(standalone_config)
# Create the monitoring system
monitoring_backend = self.config['lithops']['monitoring'].lower()
monitoring_config = self.config.get(monitoring_backend)
self.job_monitor = JobMonitor(monitoring_backend, monitoring_config)
# Create the invokder
self.invoker = create_invoker(self.config, self.executor_id,
self.internal_storage,
self.compute_handler, self.job_monitor)
logger.info('{} Executor created with ID: {}'.format(
mode.capitalize(), self.executor_id))
self.log_path = None
def build_runtime(name, file, config=None):
config = default_config(config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
compute_handler.build_runtime(name, file)
