def run_worker(master_ip, job_key): """ Run a job """ global BUDGET_KEEPER while True: url = 'http://{}:{}/get-task/{}'.format(master_ip, STANDALONE_SERVICE_PORT, job_key) logger.info('Getting task from {}'.format(url)) resp = requests.get(url) if resp.status_code != 200: logger.info('All tasks completed'.format(url)) return job_payload = resp.json() logger.info(job_payload) logger.info('Got tasks {}'.format(', '.join(job_payload['call_ids']))) try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception: return BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running' pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime}) localhost_handler.invoke(job_payload) wait_job_completed(job_key)
def run_job_local(work_queue): """ Localhost jobs manager process for consume mode """ global localhos_handler global last_job_key pull_runtime = standalone_config.get('pull_runtime', False) def wait_job_completed(job_key): done = os.path.join(JOBS_DIR, job_key+'.done') while True: if os.path.isfile(done): break time.sleep(1) try: localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime}) while True: job_payload = work_queue.get() job_key = job_payload['job_key'] last_job_key = job_key job_payload['config']['lithops']['backend'] = 'localhost' localhos_handler.invoke(job_payload) wait_job_completed(job_key) except Exception as e: logger.error(e)
def run(): """ Run a job """ global last_usage_time global jobs message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() last_usage_time = time.time() act_id = str(uuid.uuid4()).replace('-', '')[:12] runtime = message['job_description']['runtime_name'] logger.info("Running job in {}".format(runtime)) executor_id = message['job_description']['executor_id'] job_id = message['job_description']['job_id'] jobs['{}_{}'.format(executor_id.replace('/', '-'), job_id)] = 'running' localhost_handler = LocalhostHandler({'runtime': runtime}) localhost_handler.run_job(message) response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def run(): """ Run a job """ global BUDGET_KEEPER job_payload = flask.request.get_json(force=True, silent=True) if job_payload and not isinstance(job_payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_payload['job_key']] = 'running' pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) localhost_handler = LocalhostHandler({ 'runtime': runtime, 'pull_runtime': pull_runtime }) localhost_handler.run_job(job_payload) act_id = str(uuid.uuid4()).replace('-', '')[:12] response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def run(): """ Run a job """ global last_usage_time global backend_handler global jobs message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() last_usage_time = time.time() standalone_config = message['config']['standalone'] backend_handler.auto_dismantle = standalone_config['auto_dismantle'] backend_handler.soft_dismantle_timeout = standalone_config[ 'soft_dismantle_timeout'] backend_handler.hard_dismantle_timeout = standalone_config[ 'hard_dismantle_timeout'] act_id = str(uuid.uuid4()).replace('-', '')[:12] runtime = message['job_description']['runtime_name'] executor_id = message['executor_id'] job_id = message['job_id'] jobs['{}_{}'.format(executor_id.replace('/', '-'), job_id)] = 'running' localhost_handler = LocalhostHandler({'runtime': runtime}) localhost_handler.run_job(message) response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def run(): """ Run a job locally, in consume mode """ global BUDGET_KEEPER global WORK_QUEUES global JOB_PROCESSES job_payload = flask.request.get_json(force=True, silent=True) if job_payload and not isinstance(job_payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) job_key = job_payload['job_key'] logger.info('Received job {}'.format(job_key)) BUDGET_KEEPER.last_usage_time = time.time() BUDGET_KEEPER.update_config(job_payload['config']['standalone']) BUDGET_KEEPER.jobs[job_key] = 'running' exec_mode = job_payload['config']['standalone'].get('exec_mode', 'consume') if exec_mode == 'consume': # Consume mode runs the job locally pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) try: localhost_handler = LocalhostHandler({ 'runtime': runtime, 'pull_runtime': pull_runtime }) localhost_handler.invoke(job_payload) except Exception as e: logger.error(e) elif exec_mode == 'create': # Create mode runs the job in worker VMs work_queue = MP_MANAGER.Queue() WORK_QUEUES[job_key] = work_queue jp = mp.Process(target=run_job_process, args=(job_payload, work_queue)) jp.daemon = True jp.start() JOB_PROCESSES[job_key] = jp act_id = str(uuid.uuid4()).replace('-', '')[:12] response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def run_worker(master_ip, work_queue): """ Run a job """ global budget_keeper global localhos_handler global last_job_key pull_runtime = stanbdalone_config.get('pull_runtime', False) localhos_handler = LocalhostHandler({'pull_runtime': pull_runtime}) while True: url = f'http://{master_ip}:{SA_SERVICE_PORT}/get-task/{work_queue}' logger.debug(f'Getting task from {url}') try: resp = requests.get(url) except Exception: time.sleep(1) continue if resp.status_code != 200: if stanbdalone_config.get('exec_mode') == 'reuse': time.sleep(1) continue else: logger.debug('All tasks completed'.format(url)) return job_payload = resp.json() logger.debug(job_payload) logger.debug('Got tasks {}'.format(', '.join(job_payload['call_ids']))) try: runtime = job_payload['runtime_name'] verify_runtime_name(runtime) except Exception: return job_key = job_payload['job_key'] last_job_key = job_key budget_keeper.last_usage_time = time.time() budget_keeper.update_config(job_payload['config']['standalone']) budget_keeper.jobs[job_key] = 'running' try: localhos_handler.invoke(job_payload) except Exception as e: logger.error(e) wait_job_completed(job_key)
def preinstalls(): message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() runtime = message['runtime'] localhost_handler = LocalhostHandler(message) runtime_meta = localhost_handler.create_runtime(runtime) response = flask.jsonify(runtime_meta) response.status_code = 200 return response
def preinstalls(): message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() runtime = message['runtime'] logger.info("Extracting preinstalled Python modules from {}".format(runtime)) localhost_handler = LocalhostHandler(message) runtime_meta = localhost_handler.create_runtime(runtime) response = flask.jsonify(runtime_meta) response.status_code = 200 return response
def create(name, mode, memory, timeout, config): setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) runtime_key = compute_handler.get_runtime_key(name) runtime_meta = compute_handler.create_runtime(name) elif mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) runtime_key = compute_handler.get_runtime_key(name) runtime_meta = compute_handler.create_runtime(name) else: raise Exception('Unknown execution mode {}'.format(mode)) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def preinstalls(): message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = message['runtime'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) localhost_handler = LocalhostHandler(message) runtime_meta = localhost_handler.create_runtime(runtime) response = flask.jsonify(runtime_meta) response.status_code = 200 return response
def preinstalls(): payload = flask.request.get_json(force=True, silent=True) if payload and not isinstance(payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = payload['runtime'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) pull_runtime = STANDALONE_CONFIG.get('pull_runtime', False) localhost_handler = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime}) runtime_meta = localhost_handler.create_runtime(runtime) response = flask.jsonify(runtime_meta) response.status_code = 200 return response
def run(): """ Run a job """ global last_usage_time global backend_handler global jobs message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = message['job_description']['runtime_name'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) last_usage_time = time.time() standalone_config = message['config']['standalone'] backend_handler.auto_dismantle = standalone_config['auto_dismantle'] backend_handler.soft_dismantle_timeout = standalone_config[ 'soft_dismantle_timeout'] backend_handler.hard_dismantle_timeout = standalone_config[ 'hard_dismantle_timeout'] act_id = str(uuid.uuid4()).replace('-', '')[:12] executor_id = message['executor_id'] job_id = message['job_id'] job_key = create_job_key(executor_id, job_id) jobs[job_key] = 'running' localhost_handler = LocalhostHandler({'runtime': runtime}) localhost_handler.run_job(message) response = flask.jsonify({'activationId': act_id}) response.status_code = 202 return response
def clean_all(config=None): logger.info('Cleaning all Lithops information') config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) default_executor = config['lithops']['executor'] if default_executor == 'localhost': compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif default_executor == 'serverless': compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif default_executor == 'standalone': compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(STORAGE_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def clean(mode, config, debug): log_level = 'INFO' if not debug else 'DEBUG' setup_logger(log_level) logger.info('Cleaning all Lithops information') config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def preinstalls(): payload = flask.request.get_json(force=True, silent=True) if payload and not isinstance(payload, dict): return error('The action did not receive a dictionary as an argument.') try: runtime = payload['runtime'] verify_runtime_name(runtime) except Exception as e: return error(str(e)) pull_runtime = standalone_config.get('pull_runtime', False) lh = LocalhostHandler({'runtime': runtime, 'pull_runtime': pull_runtime}) runtime_meta = lh.create_runtime(runtime) if 'lithops_version' in runtime_meta: logger.debug("Runtime metdata extracted correctly: Lithops " f"{runtime_meta['lithops_version']}") response = flask.jsonify(runtime_meta) response.status_code = 200 return response
def clean(config, mode, backend, storage, debug): if config: config = load_yaml_config(config) log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) logger.info('Cleaning all Lithops information') mode = mode or get_mode(backend, config) config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def __init__(self, mode=None, config=None, backend=None, storage=None, runtime=None, runtime_memory=None, monitoring=None, workers=None, remote_invoker=None, log_level=False): """ Create a FunctionExecutor Class """ if mode and mode not in [LOCALHOST, SERVERLESS, STANDALONE]: raise Exception("Function executor mode must be one of '{}', '{}' " "or '{}'".format(LOCALHOST, SERVERLESS, STANDALONE)) self.is_lithops_worker = is_lithops_worker() # setup lithops logging if not self.is_lithops_worker: # if is lithops worker, logging has been set up in entry_point.py if log_level: setup_lithops_logger(log_level) elif log_level is False and logger.getEffectiveLevel( ) == logging.WARNING: # Set default logging from config setup_lithops_logger(*get_log_info(config)) # load mode of execution mode = mode or get_mode(backend, config) config_ow = {'lithops': {'mode': mode}, mode: {}} # overwrite user-provided parameters if runtime is not None: config_ow[mode]['runtime'] = runtime if backend is not None: config_ow[mode]['backend'] = backend if runtime_memory is not None: config_ow[mode]['runtime_memory'] = int(runtime_memory) if remote_invoker is not None: config_ow[mode]['remote_invoker'] = remote_invoker if storage is not None: config_ow['lithops']['storage'] = storage if workers is not None: config_ow['lithops']['workers'] = workers if monitoring is not None: config_ow['lithops']['monitoring'] = monitoring self.config = default_config(copy.deepcopy(config), config_ow) self.executor_id = create_executor_id() self.data_cleaner = self.config['lithops'].get('data_cleaner', True) if self.data_cleaner and not self.is_lithops_worker: spawn_cleaner = int(self.executor_id.split('-')[1]) == 0 atexit.register(self.clean, spawn_cleaner=spawn_cleaner, clean_cloudobjects=False) storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.storage = self.internal_storage.storage self.futures = [] self.cleaned_jobs = set() self.total_jobs = 0 self.last_call = None if mode == LOCALHOST: localhost_config = extract_localhost_config(self.config) self.compute_handler = LocalhostHandler(localhost_config) elif mode == SERVERLESS: serverless_config = extract_serverless_config(self.config) self.compute_handler = ServerlessHandler(serverless_config, self.internal_storage) elif mode == STANDALONE: standalone_config = extract_standalone_config(self.config) self.compute_handler = StandaloneHandler(standalone_config) # Create the monitoring system monitoring_backend = self.config['lithops']['monitoring'].lower() monitoring_config = self.config.get(monitoring_backend) self.job_monitor = JobMonitor(monitoring_backend, monitoring_config) # Create the invokder self.invoker = create_invoker(self.config, self.executor_id, self.internal_storage, self.compute_handler, self.job_monitor) logger.info('{} Executor created with ID: {}'.format( mode.capitalize(), self.executor_id)) self.log_path = None
def __init__(self, type=None, config=None, backend=None, storage=None, runtime=None, runtime_memory=None, rabbitmq_monitor=None, workers=None, remote_invoker=None, log_level=None): if type is None: config = default_config(copy.deepcopy(config)) type = config['lithops']['executor'] if log_level: default_logging_config(log_level) config_ow = {'lithops': {'executor': type}, type: {}} if runtime is not None: config_ow[type]['runtime'] = runtime if backend is not None: config_ow[type]['backend'] = backend if runtime_memory is not None: config_ow[type]['runtime_memory'] = int(runtime_memory) if remote_invoker is not None: config_ow[type]['remote_invoker'] = remote_invoker if storage is not None: config_ow['lithops']['storage'] = storage if workers is not None: config_ow['lithops']['workers'] = workers if rabbitmq_monitor is not None: config_ow['lithops']['rabbitmq_monitor'] = rabbitmq_monitor self.config = default_config(copy.deepcopy(config), config_ow) self.log_active = logger.getEffectiveLevel() != logging.WARNING self.is_lithops_worker = is_lithops_worker() self.executor_id = create_executor_id() self.data_cleaner = self.config['lithops'].get('data_cleaner', True) self.rabbitmq_monitor = self.config['lithops'].get( 'rabbitmq_monitor', False) if self.rabbitmq_monitor: if 'rabbitmq' in self.config and 'amqp_url' in self.config[ 'rabbitmq']: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') else: raise Exception("You cannot use rabbitmq_mnonitor since " "'amqp_url' is not present in configuration") self.storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(self.storage_config) self.storage = self.internal_storage.storage self.futures = [] self.total_jobs = 0 self.cleaned_jobs = set() self.last_call = None if type == 'localhost': localhost_config = extract_localhost_config(self.config) self.compute_handler = LocalhostHandler(localhost_config) self.invoker = StandaloneInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) elif type == 'serverless': serverless_config = extract_serverless_config(self.config) self.compute_handler = ServerlessHandler(serverless_config, self.storage_config) self.invoker = ServerlessInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) elif type == 'standalone': standalone_config = extract_standalone_config(self.config) self.compute_handler = StandaloneHandler(standalone_config) self.invoker = StandaloneInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) else: raise Exception("Function executor type must be one of " "'localhost', 'serverless' or 'standalone'") logger.info('{} Executor created with ID: {}'.format( type.capitalize(), self.executor_id))
def __init__(self, type=None, mode=None, config=None, backend=None, storage=None, runtime=None, runtime_memory=None, rabbitmq_monitor=None, workers=None, remote_invoker=None, log_level=None): mode = mode or type if mode is None: config = default_config(copy.deepcopy(config)) mode = config['lithops']['mode'] if mode not in [LOCALHOST, SERVERLESS, STANDALONE]: raise Exception("Function executor mode must be one of '{}', '{}' " "or '{}'".format(LOCALHOST, SERVERLESS, STANDALONE)) if log_level: setup_logger(log_level) if type is not None: logger.warning("'type' parameter is deprecated and it will be removed" "in future releases. Use 'mode' parameter instead") config_ow = {'lithops': {'mode': mode}, mode: {}} if runtime is not None: config_ow[mode]['runtime'] = runtime if backend is not None: config_ow[mode]['backend'] = backend if runtime_memory is not None: config_ow[mode]['runtime_memory'] = int(runtime_memory) if remote_invoker is not None: config_ow[mode]['remote_invoker'] = remote_invoker if storage is not None: config_ow['lithops']['storage'] = storage if workers is not None: config_ow['lithops']['workers'] = workers if rabbitmq_monitor is not None: config_ow['lithops']['rabbitmq_monitor'] = rabbitmq_monitor self.config = default_config(copy.deepcopy(config), config_ow) self.log_active = logger.getEffectiveLevel() != logging.WARNING self.is_lithops_worker = is_lithops_worker() self.executor_id = create_executor_id() self.data_cleaner = self.config['lithops'].get('data_cleaner', True) if self.data_cleaner and not self.is_lithops_worker: spawn_cleaner = int(self.executor_id.split('-')[1]) == 0 atexit.register(self.clean, spawn_cleaner=spawn_cleaner, clean_cloudobjects=False) self.rabbitmq_monitor = self.config['lithops'].get('rabbitmq_monitor', False) if self.rabbitmq_monitor: if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') else: raise Exception("You cannot use rabbitmq_mnonitor since " "'amqp_url' is not present in configuration") storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.storage = self.internal_storage.storage self.futures = [] self.cleaned_jobs = set() self.total_jobs = 0 self.last_call = None if mode == LOCALHOST: localhost_config = extract_localhost_config(self.config) self.compute_handler = LocalhostHandler(localhost_config) self.invoker = StandaloneInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) elif mode == SERVERLESS: serverless_config = extract_serverless_config(self.config) self.compute_handler = ServerlessHandler(serverless_config, storage_config) self.invoker = ServerlessInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) elif mode == STANDALONE: standalone_config = extract_standalone_config(self.config) self.compute_handler = StandaloneHandler(standalone_config) self.invoker = StandaloneInvoker(self.config, self.executor_id, self.internal_storage, self.compute_handler) logger.info('{} Executor created with ID: {}' .format(mode.capitalize(), self.executor_id))
def __init__(self, mode: Optional[str] = None, config: Optional[Dict[str, Any]] = None, backend: Optional[str] = None, storage: Optional[str] = None, runtime: Optional[str] = None, runtime_memory: Optional[int] = None, monitoring: Optional[str] = None, max_workers: Optional[int] = None, worker_processes: Optional[int] = None, remote_invoker: Optional[bool] = None, log_level: Optional[str] = False): self.is_lithops_worker = is_lithops_worker() self.executor_id = create_executor_id() self.futures = [] self.cleaned_jobs = set() self.total_jobs = 0 self.last_call = None # setup lithops logging if not self.is_lithops_worker: # if is lithops worker, logging has been set up in entry_point.py if log_level: setup_lithops_logger(log_level) elif log_level is False and logger.getEffectiveLevel( ) == logging.WARNING: # Set default logging from config setup_lithops_logger(*get_log_info(config)) # overwrite user-provided parameters config_ow = {'lithops': {}, 'backend': {}} if runtime is not None: config_ow['backend']['runtime'] = runtime if runtime_memory is not None: config_ow['backend']['runtime_memory'] = int(runtime_memory) if remote_invoker is not None: config_ow['backend']['remote_invoker'] = remote_invoker if worker_processes is not None: config_ow['backend']['worker_processes'] = worker_processes if max_workers is not None: config_ow['backend']['max_workers'] = max_workers if mode is not None: config_ow['lithops']['mode'] = mode if backend is not None: config_ow['lithops']['backend'] = backend if storage is not None: config_ow['lithops']['storage'] = storage if monitoring is not None: config_ow['lithops']['monitoring'] = monitoring # Load configuration self.config = default_config(copy.deepcopy(config), config_ow) self.data_cleaner = self.config['lithops'].get('data_cleaner', True) if self.data_cleaner and not self.is_lithops_worker: atexit.register(self.clean, clean_cloudobjects=False, clean_fn=True) storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.storage = self.internal_storage.storage self.backend = self.config['lithops']['backend'] self.mode = self.config['lithops']['mode'] if self.mode == LOCALHOST: localhost_config = extract_localhost_config(self.config) self.compute_handler = LocalhostHandler(localhost_config) elif self.mode == SERVERLESS: serverless_config = extract_serverless_config(self.config) self.compute_handler = ServerlessHandler(serverless_config, self.internal_storage) elif self.mode == STANDALONE: standalone_config = extract_standalone_config(self.config) self.compute_handler = StandaloneHandler(standalone_config) # Create the monitoring system self.job_monitor = JobMonitor(executor_id=self.executor_id, internal_storage=self.internal_storage, config=self.config) # Create the invoker self.invoker = create_invoker(config=self.config, executor_id=self.executor_id, internal_storage=self.internal_storage, compute_handler=self.compute_handler, job_monitor=self.job_monitor) logger.debug( f'Function executor for {self.backend} created with ID: {self.executor_id}' ) self.log_path = None