def __init__(self, config=None):
if isinstance(config, str):
config = load_yaml_config(config)
self._config = extract_storage_config(config)
elif isinstance(config, dict):
if 'lithops' in config:
self._config = extract_storage_config(config)
else:
self._config = config
else:
self._config = extract_storage_config(default_storage_config())
super().__init__(storage_config=self._config)
def create(name, mode, memory, timeout, config):
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
runtime_key = compute_handler.get_runtime_key(name)
runtime_meta = compute_handler.create_runtime(name)
elif mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
runtime_key = compute_handler.get_runtime_key(name)
runtime_meta = compute_handler.create_runtime(name)
else:
raise Exception('Unknown execution mode {}'.format(mode))
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def update(name, config, backend, storage):
""" Update a serverless runtime """
if config:
config = load_yaml_config(config)
verify_runtime_name(name)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def __init__(self, config=None, backend=None, storage_config=None):
""" Creates an Storage instance
:param config: lithops configuration dict
:param backend: storage backend name
:return: Storage instance.
"""
if storage_config:
self.storage_config = storage_config
else:
storage_config = default_storage_config(config_data=config,
backend=backend)
self.storage_config = extract_storage_config(storage_config)
self.backend = self.storage_config['backend']
self.bucket = self.storage_config['bucket']
try:
module_location = 'lithops.storage.backends.{}'.format(self.backend)
sb_module = importlib.import_module(module_location)
StorageBackend = getattr(sb_module, 'StorageBackend')
self.storage_handler = StorageBackend(self.storage_config[self.backend])
except Exception as e:
logger.error("An exception was produced trying to create the "
"'{}' storage backend".format(self.backend))
raise e
self._created_cobjects_n = itertools.count()
def __init__(self, config, executor_id, internal_storage, compute_handler):
self.log_active = logger.getEffectiveLevel() != logging.WARNING
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_handler = compute_handler
self.is_lithops_worker = is_lithops_worker()
self.invokers = []
self.remote_invoker = self.config['serverless'].get(
'remote_invoker', False)
self.workers = self.config['lithops'].get('workers')
logger.debug('ExecutorID {} - Total available workers: {}'.format(
self.executor_id, self.workers))
if not is_lithops_worker() and is_unix_system():
self.token_bucket_q = multiprocessing.Queue()
self.pending_calls_q = multiprocessing.Queue()
self.running_flag = multiprocessing.Value('i', 0)
else:
self.token_bucket_q = queue.Queue()
self.pending_calls_q = queue.Queue()
self.running_flag = SimpleNamespace(value=0)
self.ongoing_activations = 0
self.job_monitor = JobMonitor(self.config, self.internal_storage,
self.token_bucket_q)
logger.debug('ExecutorID {} - Serverless invoker created'.format(
self.executor_id))
def clean(mode, config, debug):
log_level = 'INFO' if not debug else 'DEBUG'
setup_logger(log_level)
logger.info('Cleaning all Lithops information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def __init__(self, config, executor_id, internal_storage, compute_handler,
job_monitor):
log_level = logger.getEffectiveLevel()
self.log_active = log_level != logging.WARNING
self.log_level = LOGGER_LEVEL if not self.log_active else log_level
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_handler = compute_handler
self.is_lithops_worker = is_lithops_worker()
self.job_monitor = job_monitor
self.workers = self.config['lithops'].get('workers')
if self.workers:
logger.debug('ExecutorID {} - Total workers: {}'.format(
self.executor_id, self.workers))
prom_enabled = self.config['lithops'].get('telemetry', False)
prom_config = self.config.get('prometheus', {})
self.prometheus = PrometheusExporter(prom_enabled, prom_config)
self.mode = self.config['lithops']['mode']
self.backend = self.config['lithops']['backend']
self.runtime_name = self.config[self.backend]['runtime']
self.customized_runtime = self.config[self.mode].get(
'customized_runtime', False)
def clean_all(config=None):
logger.info('Cleaning all Lithops information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
default_executor = config['lithops']['executor']
if default_executor == 'localhost':
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif default_executor == 'serverless':
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif default_executor == 'standalone':
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(STORAGE_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def __init__(self, config, num_invokers, log_level):
self.config = config
self.num_invokers = num_invokers
self.log_level = log_level
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.remote_invoker = self.config['lithops'].get(
'remote_invoker', False)
self.rabbitmq_monitor = self.config['lithops'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
self.num_workers = self.config['lithops'].get('workers')
logger.info('Total workers: {}'.format(self.num_workers))
serverless_config = extract_serverless_config(self.config)
self.serverless_handler = ServerlessHandler(serverless_config,
storage_config)
self.token_bucket_q = mp.Queue()
self.pending_calls_q = mp.Queue()
self.job_monitor = JobMonitor(self.config, self.internal_storage,
self.token_bucket_q)
def create(name, storage, backend, memory, timeout, config):
""" Create a serverless runtime """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
logger.info('Creating new lithops runtime: {}'.format(name))
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise ("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def delete(name, config, backend, storage, debug):
""" delete a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime delete" command is only valid for serverless backends')
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def delete(name, config, backend, storage):
""" delete a serverless runtime """
if config:
config = load_yaml_config(config)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
if name:
verify_runtime_name(name)
else:
name = config[mode]['runtime']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def update(name, config, backend, storage, debug):
""" Update a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime update" command is only valid for serverless backends')
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
timeout = compute_config['runtime_memory']
logger.info('Updating runtime: {}'.format(name))
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
runtime_meta = compute_handler.deploy_runtime(runtime[0], runtime[1], timeout)
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def deploy(name, storage, backend, memory, timeout, config, debug):
""" deploy a serverless runtime """
setup_lithops_logger(logging.DEBUG)
verify_runtime_name(name)
if config:
config = load_yaml_config(config)
config_ow = set_config_ow(backend, storage, runtime_name=name)
config = default_config(config, config_ow)
if config['lithops']['mode'] != SERVERLESS:
raise Exception('"lithops runtime create" command is only valid for serverless backends')
logger.info('Creating new lithops runtime: {}'.format(name))
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, internal_storage)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.deploy_runtime(name, mem, timeout=to)
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def run_tests(test_to_run, config=None, mode=None, backend=None, storage=None):
global CONFIG, STORAGE_CONFIG, STORAGE
mode = mode or get_mode(backend, config)
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
CONFIG = default_config(config, config_ow)
STORAGE_CONFIG = extract_storage_config(CONFIG)
STORAGE = Storage(storage_config=STORAGE_CONFIG)
suite = unittest.TestSuite()
if test_to_run == 'all':
suite.addTest(unittest.makeSuite(TestLithops))
else:
try:
suite.addTest(TestLithops(test_to_run))
except ValueError:
print("unknown test, use: --help")
sys.exit()
runner = unittest.TextTestRunner()
runner.run(suite)
def run_tests(tests, config=None, group=None, backend=None, storage=None, fail_fast=False,
keep_datasets=False):
global CONFIG, STORAGE_CONFIG, STORAGE
config_ow = {'lithops': {}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow['lithops']['backend'] = backend
CONFIG = default_config(config, config_ow)
STORAGE_CONFIG = extract_storage_config(CONFIG)
STORAGE = Storage(storage_config=STORAGE_CONFIG)
init_test_variables()
suite = unittest.TestSuite()
config_suite(suite, tests, group)
words_in_data_set = upload_data_sets() # uploads datasets and returns word count
main_util.init_config(CONFIG, STORAGE, STORAGE_CONFIG, words_in_data_set, TEST_FILES_URLS)
runner = unittest.TextTestRunner(verbosity=2, failfast=fail_fast)
tests_results = runner.run(suite)
# removes previously uploaded datasets from storage.
if not keep_datasets:
clean_tests(STORAGE, STORAGE_CONFIG, PREFIX)
if not tests_results.wasSuccessful(): # Fails github workflow action to reject merge to repository
sys.tracebacklimit = 0 # avoid displaying redundant stack track-back info
raise Exception("--------Test procedure failed. Merge rejected--------")
def function_invoker(job_payload):
"""
Method used as a remote invoker
"""
config = job_payload['config']
job = SimpleNamespace(**job_payload['job'])
env = {
'LITHOPS_WORKER': 'True',
'PYTHONUNBUFFERED': 'True',
'__LITHOPS_SESSION_ID': job.job_key
}
os.environ.update(env)
# Create the monitoring system
monitoring_backend = config['lithops']['monitoring'].lower()
monitoring_config = config.get(monitoring_backend)
job_monitor = JobMonitor(monitoring_backend, monitoring_config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
serverless_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(serverless_config, storage_config)
# Create the invokder
invoker = FaaSRemoteInvoker(config, job.executor_id, internal_storage,
compute_handler, job_monitor)
invoker.run_job(job)
def build(name, file, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def function_handler(payload):
job = SimpleNamespace(**payload)
setup_lithops_logger(job.log_level)
processes = min(job.worker_processes, len(job.call_ids))
logger.info('Tasks received: {} - Concurrent processes: {}'.format(
len(job.call_ids), processes))
env = job.extra_env
env['LITHOPS_WORKER'] = 'True'
env['PYTHONUNBUFFERED'] = 'True'
os.environ.update(env)
storage_config = extract_storage_config(job.config)
internal_storage = InternalStorage(storage_config)
job.func = get_function_and_modules(job, internal_storage)
job_data = get_function_data(job, internal_storage)
if processes == 1:
job_queue = queue.Queue()
for call_id in job.call_ids:
data = job_data.pop(0)
job_queue.put((job, call_id, data))
job_queue.put(ShutdownSentinel())
process_runner(job_queue)
else:
manager = SyncManager()
manager.start()
job_queue = manager.Queue()
job_runners = []
for call_id in job.call_ids:
data = job_data.pop(0)
job_queue.put((job, call_id, data))
for i in range(processes):
job_queue.put(ShutdownSentinel())
for runner_id in range(processes):
p = mp.Process(target=process_runner, args=(job_queue, ))
job_runners.append(p)
p.start()
logger.info('Worker process {} started'.format(runner_id))
for runner in job_runners:
runner.join()
manager.shutdown()
# Delete modules path from syspath
module_path = os.path.join(MODULES_DIR, job.job_key)
if module_path in sys.path:
sys.path.remove(module_path)
# Unset specific job env vars
for key in job.extra_env:
os.environ.pop(key, None)
os.environ.pop('__LITHOPS_TOTAL_EXECUTORS', None)
def __init__(self, config, executor_id, internal_storage, compute_handler):
self.log_active = logger.getEffectiveLevel() != logging.WARNING
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_handler = compute_handler
self.runtime_name = self.compute_handler.runtime
def load_config(config_data=None):
if config_data is None:
config_data = {}
if 'runtime_memory' not in config_data['serverless']:
config_data['serverless']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['serverless']:
config_data['serverless']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['serverless']:
config_data['serverless']['runtime'] = 'python' + \
version_str(sys.version_info)
if 'workers' not in config_data['lithops']:
config_data['lithops']['workers'] = MAX_CONCURRENT_WORKERS
if config_data['serverless'][
'runtime_memory'] not in RUNTIME_MEMORY_OPTIONS:
raise Exception('{} MB runtime is not available (Only {} MB)'.format(
config_data['serverless']['runtime_memory'],
RUNTIME_MEMORY_OPTIONS))
if config_data['serverless']['runtime_memory'] > RUNTIME_MEMORY_MAX:
config_data['serverless']['runtime_memory'] = RUNTIME_MEMORY_MAX
if config_data['serverless']['runtime_timeout'] > RUNTIME_TIMEOUT_DEFAULT:
config_data['serverless']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'gcp' not in config_data:
raise Exception("'gcp' section is mandatory in the configuration")
config_data['gcp']['retries'] = RETRIES
config_data['gcp']['retry_sleeps'] = RETRY_SLEEPS
# Put storage data into compute backend config dict entry
storage_config = dict()
storage_config['lithops'] = config_data['lithops'].copy()
storage_config['gcp_storage'] = config_data['gcp'].copy()
config_data['gcp']['storage'] = lithops_config.extract_storage_config(
storage_config)
required_parameters_0 = ('project_name', 'service_account',
'credentials_path')
if not set(required_parameters_0) <= set(config_data['gcp']):
raise Exception(
"'project_name', 'service_account' and 'credentials_path' \
are mandatory under 'gcp' section")
if not exists(config_data['gcp']['credentials_path']) or not isfile(
config_data['gcp']['credentials_path']):
raise Exception("Path {} must be credentials JSON file.".format(
config_data['gcp']['credentials_path']))
config_data['gcp_functions'] = config_data['gcp'].copy()
if 'region' not in config_data['gcp_functions']:
config_data['gcp_functions']['region'] = config_data['pywren'][
'compute_backend_region']
def delete_runtime(name, config=None):
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(config)
compute_handler = Compute(compute_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def __init__(self, config, num_invokers, log_level):
self.config = config
self.num_invokers = num_invokers
self.log_level = log_level
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
compute_config = extract_compute_config(self.config)
self.remote_invoker = self.config['lithops'].get(
'remote_invoker', False)
self.rabbitmq_monitor = self.config['lithops'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
self.num_workers = self.config['lithops'].get('workers')
logger.debug('Total workers: {}'.format(self.num_workers))
self.compute_handlers = []
cb = compute_config['backend']
regions = compute_config[cb].get('region')
if regions and type(regions) == list:
for region in regions:
new_compute_config = compute_config.copy()
new_compute_config[cb]['region'] = region
compute_handler = Compute(new_compute_config)
self.compute_handlers.append(compute_handler)
else:
if cb == 'localhost':
global CBH
if cb in CBH and CBH[
cb].compute_handler.num_workers != self.num_workers:
del CBH[cb]
if cb in CBH:
logger.info(
'{} compute handler already started'.format(cb))
compute_handler = CBH[cb]
self.compute_handlers.append(compute_handler)
else:
logger.info('Starting {} compute handler'.format(cb))
compute_handler = Compute(compute_config)
CBH[cb] = compute_handler
self.compute_handlers.append(compute_handler)
else:
compute_handler = Compute(compute_config)
self.compute_handlers.append(compute_handler)
self.token_bucket_q = Queue()
self.pending_calls_q = Queue()
self.job_monitor = JobMonitor(self.config, self.internal_storage,
self.token_bucket_q)
def _init_keeper(config):
global keeper
compute_config = extract_compute_config(config)
client = get_remote_client(compute_config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
keeper = threading.Thread(target=budget_keeper,
args=(
client,
internal_storage,
))
keeper.start()
def delete(name, config):
verify_runtime_name(name)
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
runtimes = compute_handler.list_runtimes(name)
for runtime in runtimes:
compute_handler.delete_runtime(runtime[0], runtime[1])
runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1])
internal_storage.delete_runtime_meta(runtime_key)
def __init__(self, config, executor_id, internal_storage, compute_handler):
self.log_active = logger.getEffectiveLevel() != logging.WARNING
self.config = config
self.executor_id = executor_id
self.storage_config = extract_storage_config(self.config)
self.internal_storage = internal_storage
self.compute_handler = compute_handler
self.is_lithops_worker = is_lithops_worker()
self.workers = self.config['lithops'].get('workers')
logger.debug('ExecutorID {} - Total available workers: {}'.format(
self.executor_id, self.workers))
executor = self.config['lithops']['executor']
self.runtime_name = self.config[executor]['runtime']
def build(name, file, config, backend):
""" build a serverless runtime. """
verify_runtime_name(name)
setup_lithops_logger(logging.DEBUG)
mode = SERVERLESS
config_ow = {'lithops': {'mode': mode}}
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
compute_handler.build_runtime(name, file)
def function_handler(payload):
job = SimpleNamespace(**payload)
processes = min(job.worker_processes, len(job.call_ids))
logger.info('Tasks received: {} - Concurrent workers: {}'.format(
len(job.call_ids), processes))
storage_config = extract_storage_config(job.config)
internal_storage = InternalStorage(storage_config)
job.func = get_function_and_modules(job, internal_storage)
job_data = get_function_data(job, internal_storage)
if processes == 1:
job_queue = queue.Queue()
for task_id in job.call_ids:
data = job_data.pop(0)
job_queue.put((job, task_id, data))
job_queue.put(ShutdownSentinel())
process_runner(job_queue, internal_storage)
else:
manager = SyncManager()
manager.start()
job_queue = manager.Queue()
job_runners = []
for runner_id in range(processes):
p = mp.Process(target=process_runner,
args=(job_queue, internal_storage))
job_runners.append(p)
p.start()
logger.info('Worker process {} started'.format(runner_id))
for call_id in job.call_ids:
data = job_data.pop(0)
job_queue.put((job, call_id, data))
for i in range(processes):
job_queue.put(ShutdownSentinel())
for runner in job_runners:
runner.join()
manager.shutdown()
# Delete modules path from syspath
module_path = os.path.join(MODULES_DIR, job.job_key)
if module_path in sys.path:
sys.path.remove(module_path)
def __init__(self,
storage_config=None,
lithops_config=None,
storage_backend=None,
bucket=None):
if lithops_config is None:
self.lithops_config = config.default_config()
if storage_config is None:
self.storage_config = config.extract_storage_config(
self.lithops_config)
if storage_backend is None:
self.storage_backend = self.storage_config['backend']
if bucket is not None:
self.storage_config['bucket'] = bucket
super().__init__(storage_config=self.storage_config,
lithops_config=self.lithops_config,
storage_backend=self.storage_backend)
def run_tests(test_to_run, config=None):
global CONFIG, STORAGE_CONFIG, STORAGE
CONFIG = json.load(args.config) if config else default_config()
STORAGE_CONFIG = extract_storage_config(CONFIG)
STORAGE = InternalStorage(STORAGE_CONFIG).storage
suite = unittest.TestSuite()
if test_to_run == 'all':
suite.addTest(unittest.makeSuite(TestLithops))
else:
try:
suite.addTest(TestLithops(test_to_run))
except ValueError:
print("unknown test, use: --help")
sys.exit()
runner = unittest.TextTestRunner()
runner.run(suite)