def run_tests(test_to_run, config=None, mode=None, backend=None, storage=None): global CONFIG, STORAGE_CONFIG, STORAGE mode = mode or get_mode(backend, config) config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} CONFIG = default_config(config, config_ow) STORAGE_CONFIG = extract_storage_config(CONFIG) STORAGE = Storage(storage_config=STORAGE_CONFIG) suite = unittest.TestSuite() if test_to_run == 'all': suite.addTest(unittest.makeSuite(TestLithops)) else: try: suite.addTest(TestLithops(test_to_run)) except ValueError: print("unknown test, use: --help") sys.exit() runner = unittest.TextTestRunner() runner.run(suite)
def clean(config, mode, backend, storage, debug): if config: config = load_yaml_config(config) log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) logger.info('Cleaning all Lithops information') mode = mode or get_mode(backend, config) config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def __init__(self, mode=None, config=None, backend=None, storage=None, runtime=None, runtime_memory=None, monitoring=None, workers=None, remote_invoker=None, log_level=False): """ Create a FunctionExecutor Class """ if mode and mode not in [LOCALHOST, SERVERLESS, STANDALONE]: raise Exception("Function executor mode must be one of '{}', '{}' " "or '{}'".format(LOCALHOST, SERVERLESS, STANDALONE)) self.is_lithops_worker = is_lithops_worker() # setup lithops logging if not self.is_lithops_worker: # if is lithops worker, logging has been set up in entry_point.py if log_level: setup_lithops_logger(log_level) elif log_level is False and logger.getEffectiveLevel( ) == logging.WARNING: # Set default logging from config setup_lithops_logger(*get_log_info(config)) # load mode of execution mode = mode or get_mode(backend, config) config_ow = {'lithops': {'mode': mode}, mode: {}} # overwrite user-provided parameters if runtime is not None: config_ow[mode]['runtime'] = runtime if backend is not None: config_ow[mode]['backend'] = backend if runtime_memory is not None: config_ow[mode]['runtime_memory'] = int(runtime_memory) if remote_invoker is not None: config_ow[mode]['remote_invoker'] = remote_invoker if storage is not None: config_ow['lithops']['storage'] = storage if workers is not None: config_ow['lithops']['workers'] = workers if monitoring is not None: config_ow['lithops']['monitoring'] = monitoring self.config = default_config(copy.deepcopy(config), config_ow) self.executor_id = create_executor_id() self.data_cleaner = self.config['lithops'].get('data_cleaner', True) if self.data_cleaner and not self.is_lithops_worker: spawn_cleaner = int(self.executor_id.split('-')[1]) == 0 atexit.register(self.clean, spawn_cleaner=spawn_cleaner, clean_cloudobjects=False) storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.storage = self.internal_storage.storage self.futures = [] self.cleaned_jobs = set() self.total_jobs = 0 self.last_call = None if mode == LOCALHOST: localhost_config = extract_localhost_config(self.config) self.compute_handler = LocalhostHandler(localhost_config) elif mode == SERVERLESS: serverless_config = extract_serverless_config(self.config) self.compute_handler = ServerlessHandler(serverless_config, self.internal_storage) elif mode == STANDALONE: standalone_config = extract_standalone_config(self.config) self.compute_handler = StandaloneHandler(standalone_config) # Create the monitoring system monitoring_backend = self.config['lithops']['monitoring'].lower() monitoring_config = self.config.get(monitoring_backend) self.job_monitor = JobMonitor(monitoring_backend, monitoring_config) # Create the invokder self.invoker = create_invoker(self.config, self.executor_id, self.internal_storage, self.compute_handler, self.job_monitor) logger.info('{} Executor created with ID: {}'.format( mode.capitalize(), self.executor_id)) self.log_path = None