def run_worker(master_ip, job_key):
"""
Run a job
"""
global BUDGET_KEEPER
while True:
url = 'http://{}:{}/get-task/{}'.format(master_ip, STANDALONE_SERVICE_PORT, job_key)
logger.info('Getting task from {}'.format(url))
resp = requests.get(url)
if resp.status_code != 200:
logger.info('All tasks completed'.format(url))
return
job_payload = resp.json()
logger.info(job_payload)
logger.info('Got tasks {}'.format(', '.join(job_payload['call_ids'])))
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception:
return
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running'
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime})
localhost_handler.invoke(job_payload)
wait_job_completed(job_key)
def run_job_local(work_queue):
"""
Localhost jobs manager process for consume mode
"""
global localhos_handler
global last_job_key
pull_runtime = standalone_config.get('pull_runtime', False)
def wait_job_completed(job_key):
done = os.path.join(JOBS_DIR, job_key+'.done')
while True:
if os.path.isfile(done):
break
time.sleep(1)
try:
localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime})
while True:
job_payload = work_queue.get()
job_key = job_payload['job_key']
last_job_key = job_key
job_payload['config']['lithops']['backend'] = 'localhost'
localhos_handler.invoke(job_payload)
wait_job_completed(job_key)
except Exception as e:
logger.error(e)
def run():
"""
Run a job
"""
global last_usage_time
global jobs
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
last_usage_time = time.time()
act_id = str(uuid.uuid4()).replace('-', '')[:12]
runtime = message['job_description']['runtime_name']
logger.info("Running job in {}".format(runtime))
executor_id = message['job_description']['executor_id']
job_id = message['job_description']['job_id']
jobs['{}_{}'.format(executor_id.replace('/', '-'), job_id)] = 'running'
localhost_handler = LocalhostHandler({'runtime': runtime})
localhost_handler.run_job(message)
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def run():
"""
Run a job
"""
global BUDGET_KEEPER
job_payload = flask.request.get_json(force=True, silent=True)
if job_payload and not isinstance(job_payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running'
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
localhost_handler = LocalhostHandler({
'runtime': runtime,
'pull_runtime': pull_runtime
})
localhost_handler.run_job(job_payload)
act_id = str(uuid.uuid4()).replace('-', '')[:12]
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def run():
"""
Run a job
"""
global last_usage_time
global backend_handler
global jobs
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
last_usage_time = time.time()
standalone_config = message['config']['standalone']
backend_handler.auto_dismantle = standalone_config['auto_dismantle']
backend_handler.soft_dismantle_timeout = standalone_config[
'soft_dismantle_timeout']
backend_handler.hard_dismantle_timeout = standalone_config[
'hard_dismantle_timeout']
act_id = str(uuid.uuid4()).replace('-', '')[:12]
runtime = message['job_description']['runtime_name']
executor_id = message['executor_id']
job_id = message['job_id']
jobs['{}_{}'.format(executor_id.replace('/', '-'), job_id)] = 'running'
localhost_handler = LocalhostHandler({'runtime': runtime})
localhost_handler.run_job(message)
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def run():
"""
Run a job locally, in consume mode
"""
global BUDGET_KEEPER
global WORK_QUEUES
global JOB_PROCESSES
job_payload = flask.request.get_json(force=True, silent=True)
if job_payload and not isinstance(job_payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
job_key = job_payload['job_key']
logger.info('Received job {}'.format(job_key))
BUDGET_KEEPER.last_usage_time = time.time()
BUDGET_KEEPER.update_config(job_payload['config']['standalone'])
BUDGET_KEEPER.jobs[job_key] = 'running'
exec_mode = job_payload['config']['standalone'].get('exec_mode', 'consume')
if exec_mode == 'consume':
# Consume mode runs the job locally
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
try:
localhost_handler = LocalhostHandler({
'runtime': runtime,
'pull_runtime': pull_runtime
})
localhost_handler.invoke(job_payload)
except Exception as e:
logger.error(e)
elif exec_mode == 'create':
# Create mode runs the job in worker VMs
work_queue = MP_MANAGER.Queue()
WORK_QUEUES[job_key] = work_queue
jp = mp.Process(target=run_job_process, args=(job_payload, work_queue))
jp.daemon = True
jp.start()
JOB_PROCESSES[job_key] = jp
act_id = str(uuid.uuid4()).replace('-', '')[:12]
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def run_worker(master_ip, work_queue):
"""
Run a job
"""
global budget_keeper
global localhos_handler
global last_job_key
pull_runtime = stanbdalone_config.get('pull_runtime', False)
localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime})
while True:
url = f'http://{master_ip}:{SA_SERVICE_PORT}/get-task/{work_queue}'
logger.debug(f'Getting task from {url}')
try:
resp = requests.get(url)
except Exception:
time.sleep(1)
continue
if resp.status_code != 200:
if stanbdalone_config.get('exec_mode') == 'reuse':
time.sleep(1)
continue
else:
logger.debug('All tasks completed'.format(url))
return
job_payload = resp.json()
logger.debug(job_payload)
logger.debug('Got tasks {}'.format(', '.join(job_payload['call_ids'])))
try:
runtime = job_payload['runtime_name']
verify_runtime_name(runtime)
except Exception:
return
job_key = job_payload['job_key']
last_job_key = job_key
budget_keeper.last_usage_time = time.time()
budget_keeper.update_config(job_payload['config']['standalone'])
budget_keeper.jobs[job_key] = 'running'
try:
localhos_handler.invoke(job_payload)
except Exception as e:
logger.error(e)
wait_job_completed(job_key)
def preinstalls():
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
runtime = message['runtime']
localhost_handler = LocalhostHandler(message)
runtime_meta = localhost_handler.create_runtime(runtime)
response = flask.jsonify(runtime_meta)
response.status_code = 200
return response
def preinstalls():
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
runtime = message['runtime']
logger.info("Extracting preinstalled Python modules from {}".format(runtime))
localhost_handler = LocalhostHandler(message)
runtime_meta = localhost_handler.create_runtime(runtime)
response = flask.jsonify(runtime_meta)
response.status_code = 200
return response
def create(name, mode, memory, timeout, config):
setup_logger(logging.DEBUG)
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
mem = memory if memory else compute_config['runtime_memory']
to = timeout if timeout else compute_config['runtime_timeout']
runtime_key = compute_handler.get_runtime_key(name, mem)
runtime_meta = compute_handler.create_runtime(name, mem, timeout=to)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
runtime_key = compute_handler.get_runtime_key(name)
runtime_meta = compute_handler.create_runtime(name)
elif mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
runtime_key = compute_handler.get_runtime_key(name)
runtime_meta = compute_handler.create_runtime(name)
else:
raise Exception('Unknown execution mode {}'.format(mode))
try:
internal_storage.put_runtime_meta(runtime_key, runtime_meta)
except Exception:
raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def preinstalls():
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = message['runtime']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
localhost_handler = LocalhostHandler(message)
runtime_meta = localhost_handler.create_runtime(runtime)
response = flask.jsonify(runtime_meta)
response.status_code = 200
return response
def preinstalls():
payload = flask.request.get_json(force=True, silent=True)
if payload and not isinstance(payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = payload['runtime']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False)
localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime})
runtime_meta = localhost_handler.create_runtime(runtime)
response = flask.jsonify(runtime_meta)
response.status_code = 200
return response
def run():
"""
Run a job
"""
global last_usage_time
global backend_handler
global jobs
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = message['job_description']['runtime_name']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
last_usage_time = time.time()
standalone_config = message['config']['standalone']
backend_handler.auto_dismantle = standalone_config['auto_dismantle']
backend_handler.soft_dismantle_timeout = standalone_config[
'soft_dismantle_timeout']
backend_handler.hard_dismantle_timeout = standalone_config[
'hard_dismantle_timeout']
act_id = str(uuid.uuid4()).replace('-', '')[:12]
executor_id = message['executor_id']
job_id = message['job_id']
job_key = create_job_key(executor_id, job_id)
jobs[job_key] = 'running'
localhost_handler = LocalhostHandler({'runtime': runtime})
localhost_handler.run_job(message)
response = flask.jsonify({'activationId': act_id})
response.status_code = 202
return response
def clean_all(config=None):
logger.info('Cleaning all Lithops information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
default_executor = config['lithops']['executor']
if default_executor == 'localhost':
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif default_executor == 'serverless':
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif default_executor == 'standalone':
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(STORAGE_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def clean(mode, config, debug):
log_level = 'INFO' if not debug else 'DEBUG'
setup_logger(log_level)
logger.info('Cleaning all Lithops information')
config = default_config(config)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def preinstalls():
payload = flask.request.get_json(force=True, silent=True)
if payload and not isinstance(payload, dict):
return error('The action did not receive a dictionary as an argument.')
try:
runtime = payload['runtime']
verify_runtime_name(runtime)
except Exception as e:
return error(str(e))
pull_runtime = standalone_config.get('pull_runtime', False)
lh = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime})
runtime_meta = lh.create_runtime(runtime)
if 'lithops_version' in runtime_meta:
logger.debug("Runtime metdata extracted correctly: Lithops "
f"{runtime_meta['lithops_version']}")
response = flask.jsonify(runtime_meta)
response.status_code = 200
return response
def clean(config, mode, backend, storage, debug):
if config:
config = load_yaml_config(config)
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
logger.info('Cleaning all Lithops information')
mode = mode or get_mode(backend, config)
config_ow = {'lithops': {'mode': mode}}
if storage:
config_ow['lithops']['storage'] = storage
if backend:
config_ow[mode] = {'backend': backend}
config = default_config(config, config_ow)
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
mode = config['lithops']['mode'] if not mode else mode
if mode == LOCALHOST:
compute_config = extract_localhost_config(config)
compute_handler = LocalhostHandler(compute_config)
elif mode == SERVERLESS:
compute_config = extract_serverless_config(config)
compute_handler = ServerlessHandler(compute_config, storage_config)
elif mode == STANDALONE:
compute_config = extract_standalone_config(config)
compute_handler = StandaloneHandler(compute_config)
compute_handler.clean()
# Clean object storage temp dirs
storage = internal_storage.storage
clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1)
clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1)
# Clean localhost executor temp dirs
shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True)
# Clean local lithops cache
shutil.rmtree(CACHE_DIR, ignore_errors=True)
def __init__(self,
mode=None,
config=None,
backend=None,
storage=None,
runtime=None,
runtime_memory=None,
monitoring=None,
workers=None,
remote_invoker=None,
log_level=False):
""" Create a FunctionExecutor Class """
if mode and mode not in [LOCALHOST, SERVERLESS, STANDALONE]:
raise Exception("Function executor mode must be one of '{}', '{}' "
"or '{}'".format(LOCALHOST, SERVERLESS,
STANDALONE))
self.is_lithops_worker = is_lithops_worker()
# setup lithops logging
if not self.is_lithops_worker:
# if is lithops worker, logging has been set up in entry_point.py
if log_level:
setup_lithops_logger(log_level)
elif log_level is False and logger.getEffectiveLevel(
) == logging.WARNING:
# Set default logging from config
setup_lithops_logger(*get_log_info(config))
# load mode of execution
mode = mode or get_mode(backend, config)
config_ow = {'lithops': {'mode': mode}, mode: {}}
# overwrite user-provided parameters
if runtime is not None:
config_ow[mode]['runtime'] = runtime
if backend is not None:
config_ow[mode]['backend'] = backend
if runtime_memory is not None:
config_ow[mode]['runtime_memory'] = int(runtime_memory)
if remote_invoker is not None:
config_ow[mode]['remote_invoker'] = remote_invoker
if storage is not None:
config_ow['lithops']['storage'] = storage
if workers is not None:
config_ow['lithops']['workers'] = workers
if monitoring is not None:
config_ow['lithops']['monitoring'] = monitoring
self.config = default_config(copy.deepcopy(config), config_ow)
self.executor_id = create_executor_id()
self.data_cleaner = self.config['lithops'].get('data_cleaner', True)
if self.data_cleaner and not self.is_lithops_worker:
spawn_cleaner = int(self.executor_id.split('-')[1]) == 0
atexit.register(self.clean,
spawn_cleaner=spawn_cleaner,
clean_cloudobjects=False)
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.storage = self.internal_storage.storage
self.futures = []
self.cleaned_jobs = set()
self.total_jobs = 0
self.last_call = None
if mode == LOCALHOST:
localhost_config = extract_localhost_config(self.config)
self.compute_handler = LocalhostHandler(localhost_config)
elif mode == SERVERLESS:
serverless_config = extract_serverless_config(self.config)
self.compute_handler = ServerlessHandler(serverless_config,
self.internal_storage)
elif mode == STANDALONE:
standalone_config = extract_standalone_config(self.config)
self.compute_handler = StandaloneHandler(standalone_config)
# Create the monitoring system
monitoring_backend = self.config['lithops']['monitoring'].lower()
monitoring_config = self.config.get(monitoring_backend)
self.job_monitor = JobMonitor(monitoring_backend, monitoring_config)
# Create the invokder
self.invoker = create_invoker(self.config, self.executor_id,
self.internal_storage,
self.compute_handler, self.job_monitor)
logger.info('{} Executor created with ID: {}'.format(
mode.capitalize(), self.executor_id))
self.log_path = None
def __init__(self,
type=None,
config=None,
backend=None,
storage=None,
runtime=None,
runtime_memory=None,
rabbitmq_monitor=None,
workers=None,
remote_invoker=None,
log_level=None):
if type is None:
config = default_config(copy.deepcopy(config))
type = config['lithops']['executor']
if log_level:
default_logging_config(log_level)
config_ow = {'lithops': {'executor': type}, type: {}}
if runtime is not None:
config_ow[type]['runtime'] = runtime
if backend is not None:
config_ow[type]['backend'] = backend
if runtime_memory is not None:
config_ow[type]['runtime_memory'] = int(runtime_memory)
if remote_invoker is not None:
config_ow[type]['remote_invoker'] = remote_invoker
if storage is not None:
config_ow['lithops']['storage'] = storage
if workers is not None:
config_ow['lithops']['workers'] = workers
if rabbitmq_monitor is not None:
config_ow['lithops']['rabbitmq_monitor'] = rabbitmq_monitor
self.config = default_config(copy.deepcopy(config), config_ow)
self.log_active = logger.getEffectiveLevel() != logging.WARNING
self.is_lithops_worker = is_lithops_worker()
self.executor_id = create_executor_id()
self.data_cleaner = self.config['lithops'].get('data_cleaner', True)
self.rabbitmq_monitor = self.config['lithops'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config[
'rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception("You cannot use rabbitmq_mnonitor since "
"'amqp_url' is not present in configuration")
self.storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(self.storage_config)
self.storage = self.internal_storage.storage
self.futures = []
self.total_jobs = 0
self.cleaned_jobs = set()
self.last_call = None
if type == 'localhost':
localhost_config = extract_localhost_config(self.config)
self.compute_handler = LocalhostHandler(localhost_config)
self.invoker = StandaloneInvoker(self.config, self.executor_id,
self.internal_storage,
self.compute_handler)
elif type == 'serverless':
serverless_config = extract_serverless_config(self.config)
self.compute_handler = ServerlessHandler(serverless_config,
self.storage_config)
self.invoker = ServerlessInvoker(self.config, self.executor_id,
self.internal_storage,
self.compute_handler)
elif type == 'standalone':
standalone_config = extract_standalone_config(self.config)
self.compute_handler = StandaloneHandler(standalone_config)
self.invoker = StandaloneInvoker(self.config, self.executor_id,
self.internal_storage,
self.compute_handler)
else:
raise Exception("Function executor type must be one of "
"'localhost', 'serverless' or 'standalone'")
logger.info('{} Executor created with ID: {}'.format(
type.capitalize(), self.executor_id))
def __init__(self, type=None, mode=None, config=None, backend=None, storage=None,
runtime=None, runtime_memory=None, rabbitmq_monitor=None,
workers=None, remote_invoker=None, log_level=None):
mode = mode or type
if mode is None:
config = default_config(copy.deepcopy(config))
mode = config['lithops']['mode']
if mode not in [LOCALHOST, SERVERLESS, STANDALONE]:
raise Exception("Function executor mode must be one of '{}', '{}' "
"or '{}'".format(LOCALHOST, SERVERLESS, STANDALONE))
if log_level:
setup_logger(log_level)
if type is not None:
logger.warning("'type' parameter is deprecated and it will be removed"
"in future releases. Use 'mode' parameter instead")
config_ow = {'lithops': {'mode': mode}, mode: {}}
if runtime is not None:
config_ow[mode]['runtime'] = runtime
if backend is not None:
config_ow[mode]['backend'] = backend
if runtime_memory is not None:
config_ow[mode]['runtime_memory'] = int(runtime_memory)
if remote_invoker is not None:
config_ow[mode]['remote_invoker'] = remote_invoker
if storage is not None:
config_ow['lithops']['storage'] = storage
if workers is not None:
config_ow['lithops']['workers'] = workers
if rabbitmq_monitor is not None:
config_ow['lithops']['rabbitmq_monitor'] = rabbitmq_monitor
self.config = default_config(copy.deepcopy(config), config_ow)
self.log_active = logger.getEffectiveLevel() != logging.WARNING
self.is_lithops_worker = is_lithops_worker()
self.executor_id = create_executor_id()
self.data_cleaner = self.config['lithops'].get('data_cleaner', True)
if self.data_cleaner and not self.is_lithops_worker:
spawn_cleaner = int(self.executor_id.split('-')[1]) == 0
atexit.register(self.clean, spawn_cleaner=spawn_cleaner,
clean_cloudobjects=False)
self.rabbitmq_monitor = self.config['lithops'].get('rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception("You cannot use rabbitmq_mnonitor since "
"'amqp_url' is not present in configuration")
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.storage = self.internal_storage.storage
self.futures = []
self.cleaned_jobs = set()
self.total_jobs = 0
self.last_call = None
if mode == LOCALHOST:
localhost_config = extract_localhost_config(self.config)
self.compute_handler = LocalhostHandler(localhost_config)
self.invoker = StandaloneInvoker(self.config,
self.executor_id,
self.internal_storage,
self.compute_handler)
elif mode == SERVERLESS:
serverless_config = extract_serverless_config(self.config)
self.compute_handler = ServerlessHandler(serverless_config,
storage_config)
self.invoker = ServerlessInvoker(self.config,
self.executor_id,
self.internal_storage,
self.compute_handler)
elif mode == STANDALONE:
standalone_config = extract_standalone_config(self.config)
self.compute_handler = StandaloneHandler(standalone_config)
self.invoker = StandaloneInvoker(self.config,
self.executor_id,
self.internal_storage,
self.compute_handler)
logger.info('{} Executor created with ID: {}'
.format(mode.capitalize(), self.executor_id))
def __init__(self,
mode: Optional[str] = None,
config: Optional[Dict[str, Any]] = None,
backend: Optional[str] = None,
storage: Optional[str] = None,
runtime: Optional[str] = None,
runtime_memory: Optional[int] = None,
monitoring: Optional[str] = None,
max_workers: Optional[int] = None,
worker_processes: Optional[int] = None,
remote_invoker: Optional[bool] = None,
log_level: Optional[str] = False):
self.is_lithops_worker = is_lithops_worker()
self.executor_id = create_executor_id()
self.futures = []
self.cleaned_jobs = set()
self.total_jobs = 0
self.last_call = None
# setup lithops logging
if not self.is_lithops_worker:
# if is lithops worker, logging has been set up in entry_point.py
if log_level:
setup_lithops_logger(log_level)
elif log_level is False and logger.getEffectiveLevel(
) == logging.WARNING:
# Set default logging from config
setup_lithops_logger(*get_log_info(config))
# overwrite user-provided parameters
config_ow = {'lithops': {}, 'backend': {}}
if runtime is not None:
config_ow['backend']['runtime'] = runtime
if runtime_memory is not None:
config_ow['backend']['runtime_memory'] = int(runtime_memory)
if remote_invoker is not None:
config_ow['backend']['remote_invoker'] = remote_invoker
if worker_processes is not None:
config_ow['backend']['worker_processes'] = worker_processes
if max_workers is not None:
config_ow['backend']['max_workers'] = max_workers
if mode is not None:
config_ow['lithops']['mode'] = mode
if backend is not None:
config_ow['lithops']['backend'] = backend
if storage is not None:
config_ow['lithops']['storage'] = storage
if monitoring is not None:
config_ow['lithops']['monitoring'] = monitoring
# Load configuration
self.config = default_config(copy.deepcopy(config), config_ow)
self.data_cleaner = self.config['lithops'].get('data_cleaner', True)
if self.data_cleaner and not self.is_lithops_worker:
atexit.register(self.clean,
clean_cloudobjects=False,
clean_fn=True)
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.storage = self.internal_storage.storage
self.backend = self.config['lithops']['backend']
self.mode = self.config['lithops']['mode']
if self.mode == LOCALHOST:
localhost_config = extract_localhost_config(self.config)
self.compute_handler = LocalhostHandler(localhost_config)
elif self.mode == SERVERLESS:
serverless_config = extract_serverless_config(self.config)
self.compute_handler = ServerlessHandler(serverless_config,
self.internal_storage)
elif self.mode == STANDALONE:
standalone_config = extract_standalone_config(self.config)
self.compute_handler = StandaloneHandler(standalone_config)
# Create the monitoring system
self.job_monitor = JobMonitor(executor_id=self.executor_id,
internal_storage=self.internal_storage,
config=self.config)
# Create the invoker
self.invoker = create_invoker(config=self.config,
executor_id=self.executor_id,
internal_storage=self.internal_storage,
compute_handler=self.compute_handler,
job_monitor=self.job_monitor)
logger.debug(
f'Function executor for {self.backend} created with ID: {self.executor_id}'
)
self.log_path = None
