def __init__(self, config=None): if isinstance(config, str): config = load_yaml_config(config) self._config = extract_storage_config(config) elif isinstance(config, dict): if 'lithops' in config: self._config = extract_storage_config(config) else: self._config = config else: self._config = extract_storage_config(default_storage_config()) super().__init__(storage_config=self._config)
def create(name, mode, memory, timeout, config): setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) runtime_key = compute_handler.get_runtime_key(name) runtime_meta = compute_handler.create_runtime(name) elif mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) runtime_key = compute_handler.get_runtime_key(name) runtime_meta = compute_handler.create_runtime(name) else: raise Exception('Unknown execution mode {}'.format(mode)) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def update(name, config, backend, storage): """ Update a serverless runtime """ if config: config = load_yaml_config(config) verify_runtime_name(name) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def __init__(self, config=None, backend=None, storage_config=None): """ Creates an Storage instance :param config: lithops configuration dict :param backend: storage backend name :return: Storage instance. """ if storage_config: self.storage_config = storage_config else: storage_config = default_storage_config(config_data=config, backend=backend) self.storage_config = extract_storage_config(storage_config) self.backend = self.storage_config['backend'] self.bucket = self.storage_config['bucket'] try: module_location = 'lithops.storage.backends.{}'.format(self.backend) sb_module = importlib.import_module(module_location) StorageBackend = getattr(sb_module, 'StorageBackend') self.storage_handler = StorageBackend(self.storage_config[self.backend]) except Exception as e: logger.error("An exception was produced trying to create the " "'{}' storage backend".format(self.backend)) raise e self._created_cobjects_n = itertools.count()
def __init__(self, config, executor_id, internal_storage, compute_handler): self.log_active = logger.getEffectiveLevel() != logging.WARNING self.config = config self.executor_id = executor_id self.storage_config = extract_storage_config(self.config) self.internal_storage = internal_storage self.compute_handler = compute_handler self.is_lithops_worker = is_lithops_worker() self.invokers = [] self.remote_invoker = self.config['serverless'].get( 'remote_invoker', False) self.workers = self.config['lithops'].get('workers') logger.debug('ExecutorID {} - Total available workers: {}'.format( self.executor_id, self.workers)) if not is_lithops_worker() and is_unix_system(): self.token_bucket_q = multiprocessing.Queue() self.pending_calls_q = multiprocessing.Queue() self.running_flag = multiprocessing.Value('i', 0) else: self.token_bucket_q = queue.Queue() self.pending_calls_q = queue.Queue() self.running_flag = SimpleNamespace(value=0) self.ongoing_activations = 0 self.job_monitor = JobMonitor(self.config, self.internal_storage, self.token_bucket_q) logger.debug('ExecutorID {} - Serverless invoker created'.format( self.executor_id))
def clean(mode, config, debug): log_level = 'INFO' if not debug else 'DEBUG' setup_logger(log_level) logger.info('Cleaning all Lithops information') config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def __init__(self, config, executor_id, internal_storage, compute_handler, job_monitor): log_level = logger.getEffectiveLevel() self.log_active = log_level != logging.WARNING self.log_level = LOGGER_LEVEL if not self.log_active else log_level self.config = config self.executor_id = executor_id self.storage_config = extract_storage_config(self.config) self.internal_storage = internal_storage self.compute_handler = compute_handler self.is_lithops_worker = is_lithops_worker() self.job_monitor = job_monitor self.workers = self.config['lithops'].get('workers') if self.workers: logger.debug('ExecutorID {} - Total workers: {}'.format( self.executor_id, self.workers)) prom_enabled = self.config['lithops'].get('telemetry', False) prom_config = self.config.get('prometheus', {}) self.prometheus = PrometheusExporter(prom_enabled, prom_config) self.mode = self.config['lithops']['mode'] self.backend = self.config['lithops']['backend'] self.runtime_name = self.config[self.backend]['runtime'] self.customized_runtime = self.config[self.mode].get( 'customized_runtime', False)
def clean_all(config=None): logger.info('Cleaning all Lithops information') config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) default_executor = config['lithops']['executor'] if default_executor == 'localhost': compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif default_executor == 'serverless': compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif default_executor == 'standalone': compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(STORAGE_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def __init__(self, config, num_invokers, log_level): self.config = config self.num_invokers = num_invokers self.log_level = log_level storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.remote_invoker = self.config['lithops'].get( 'remote_invoker', False) self.rabbitmq_monitor = self.config['lithops'].get( 'rabbitmq_monitor', False) if self.rabbitmq_monitor: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') self.num_workers = self.config['lithops'].get('workers') logger.info('Total workers: {}'.format(self.num_workers)) serverless_config = extract_serverless_config(self.config) self.serverless_handler = ServerlessHandler(serverless_config, storage_config) self.token_bucket_q = mp.Queue() self.pending_calls_q = mp.Queue() self.job_monitor = JobMonitor(self.config, self.internal_storage, self.token_bucket_q)
def create(name, storage, backend, memory, timeout, config): """ Create a serverless runtime """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] logger.info('Creating new lithops runtime: {}'.format(name)) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise ("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def delete(name, config, backend, storage, debug): """ delete a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime delete" command is only valid for serverless backends') storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def delete(name, config, backend, storage): """ delete a serverless runtime """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def update(name, config, backend, storage, debug): """ Update a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime update" command is only valid for serverless backends') storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.deploy_runtime(runtime[0], runtime[1], timeout) internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def deploy(name, storage, backend, memory, timeout, config, debug): """ deploy a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime create" command is only valid for serverless backends') logger.info('Creating new lithops runtime: {}'.format(name)) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.deploy_runtime(name, mem, timeout=to) internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def run_tests(test_to_run, config=None, mode=None, backend=None, storage=None): global CONFIG, STORAGE_CONFIG, STORAGE mode = mode or get_mode(backend, config) config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} CONFIG = default_config(config, config_ow) STORAGE_CONFIG = extract_storage_config(CONFIG) STORAGE = Storage(storage_config=STORAGE_CONFIG) suite = unittest.TestSuite() if test_to_run == 'all': suite.addTest(unittest.makeSuite(TestLithops)) else: try: suite.addTest(TestLithops(test_to_run)) except ValueError: print("unknown test, use: --help") sys.exit() runner = unittest.TextTestRunner() runner.run(suite)
def run_tests(tests, config=None, group=None, backend=None, storage=None, fail_fast=False, keep_datasets=False): global CONFIG, STORAGE_CONFIG, STORAGE config_ow = {'lithops': {}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow['lithops']['backend'] = backend CONFIG = default_config(config, config_ow) STORAGE_CONFIG = extract_storage_config(CONFIG) STORAGE = Storage(storage_config=STORAGE_CONFIG) init_test_variables() suite = unittest.TestSuite() config_suite(suite, tests, group) words_in_data_set = upload_data_sets() # uploads datasets and returns word count main_util.init_config(CONFIG, STORAGE, STORAGE_CONFIG, words_in_data_set, TEST_FILES_URLS) runner = unittest.TextTestRunner(verbosity=2, failfast=fail_fast) tests_results = runner.run(suite) # removes previously uploaded datasets from storage. if not keep_datasets: clean_tests(STORAGE, STORAGE_CONFIG, PREFIX) if not tests_results.wasSuccessful(): # Fails github workflow action to reject merge to repository sys.tracebacklimit = 0 # avoid displaying redundant stack track-back info raise Exception("--------Test procedure failed. Merge rejected--------")
def function_invoker(job_payload): """ Method used as a remote invoker """ config = job_payload['config'] job = SimpleNamespace(**job_payload['job']) env = { 'LITHOPS_WORKER': 'True', 'PYTHONUNBUFFERED': 'True', '__LITHOPS_SESSION_ID': job.job_key } os.environ.update(env) # Create the monitoring system monitoring_backend = config['lithops']['monitoring'].lower() monitoring_config = config.get(monitoring_backend) job_monitor = JobMonitor(monitoring_backend, monitoring_config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) serverless_config = extract_serverless_config(config) compute_handler = ServerlessHandler(serverless_config, storage_config) # Create the invokder invoker = FaaSRemoteInvoker(config, job.executor_id, internal_storage, compute_handler, job_monitor) invoker.run_job(job)
def build(name, file, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) compute_handler.build_runtime(name, file)
def function_handler(payload): job = SimpleNamespace(**payload) setup_lithops_logger(job.log_level) processes = min(job.worker_processes, len(job.call_ids)) logger.info('Tasks received: {} - Concurrent processes: {}'.format( len(job.call_ids), processes)) env = job.extra_env env['LITHOPS_WORKER'] = 'True' env['PYTHONUNBUFFERED'] = 'True' os.environ.update(env) storage_config = extract_storage_config(job.config) internal_storage = InternalStorage(storage_config) job.func = get_function_and_modules(job, internal_storage) job_data = get_function_data(job, internal_storage) if processes == 1: job_queue = queue.Queue() for call_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, call_id, data)) job_queue.put(ShutdownSentinel()) process_runner(job_queue) else: manager = SyncManager() manager.start() job_queue = manager.Queue() job_runners = [] for call_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, call_id, data)) for i in range(processes): job_queue.put(ShutdownSentinel()) for runner_id in range(processes): p = mp.Process(target=process_runner, args=(job_queue, )) job_runners.append(p) p.start() logger.info('Worker process {} started'.format(runner_id)) for runner in job_runners: runner.join() manager.shutdown() # Delete modules path from syspath module_path = os.path.join(MODULES_DIR, job.job_key) if module_path in sys.path: sys.path.remove(module_path) # Unset specific job env vars for key in job.extra_env: os.environ.pop(key, None) os.environ.pop('__LITHOPS_TOTAL_EXECUTORS', None)
def __init__(self, config, executor_id, internal_storage, compute_handler): self.log_active = logger.getEffectiveLevel() != logging.WARNING self.config = config self.executor_id = executor_id self.storage_config = extract_storage_config(self.config) self.internal_storage = internal_storage self.compute_handler = compute_handler self.runtime_name = self.compute_handler.runtime
def load_config(config_data=None): if config_data is None: config_data = {} if 'runtime_memory' not in config_data['serverless']: config_data['serverless']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT if 'runtime_timeout' not in config_data['serverless']: config_data['serverless']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT if 'runtime' not in config_data['serverless']: config_data['serverless']['runtime'] = 'python' + \ version_str(sys.version_info) if 'workers' not in config_data['lithops']: config_data['lithops']['workers'] = MAX_CONCURRENT_WORKERS if config_data['serverless'][ 'runtime_memory'] not in RUNTIME_MEMORY_OPTIONS: raise Exception('{} MB runtime is not available (Only {} MB)'.format( config_data['serverless']['runtime_memory'], RUNTIME_MEMORY_OPTIONS)) if config_data['serverless']['runtime_memory'] > RUNTIME_MEMORY_MAX: config_data['serverless']['runtime_memory'] = RUNTIME_MEMORY_MAX if config_data['serverless']['runtime_timeout'] > RUNTIME_TIMEOUT_DEFAULT: config_data['serverless']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT if 'gcp' not in config_data: raise Exception("'gcp' section is mandatory in the configuration") config_data['gcp']['retries'] = RETRIES config_data['gcp']['retry_sleeps'] = RETRY_SLEEPS # Put storage data into compute backend config dict entry storage_config = dict() storage_config['lithops'] = config_data['lithops'].copy() storage_config['gcp_storage'] = config_data['gcp'].copy() config_data['gcp']['storage'] = lithops_config.extract_storage_config( storage_config) required_parameters_0 = ('project_name', 'service_account', 'credentials_path') if not set(required_parameters_0) <= set(config_data['gcp']): raise Exception( "'project_name', 'service_account' and 'credentials_path' \ are mandatory under 'gcp' section") if not exists(config_data['gcp']['credentials_path']) or not isfile( config_data['gcp']['credentials_path']): raise Exception("Path {} must be credentials JSON file.".format( config_data['gcp']['credentials_path'])) config_data['gcp_functions'] = config_data['gcp'].copy() if 'region' not in config_data['gcp_functions']: config_data['gcp_functions']['region'] = config_data['pywren'][ 'compute_backend_region']
def delete_runtime(name, config=None): config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_compute_config(config) compute_handler = Compute(compute_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def __init__(self, config, num_invokers, log_level): self.config = config self.num_invokers = num_invokers self.log_level = log_level storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) compute_config = extract_compute_config(self.config) self.remote_invoker = self.config['lithops'].get( 'remote_invoker', False) self.rabbitmq_monitor = self.config['lithops'].get( 'rabbitmq_monitor', False) if self.rabbitmq_monitor: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') self.num_workers = self.config['lithops'].get('workers') logger.debug('Total workers: {}'.format(self.num_workers)) self.compute_handlers = [] cb = compute_config['backend'] regions = compute_config[cb].get('region') if regions and type(regions) == list: for region in regions: new_compute_config = compute_config.copy() new_compute_config[cb]['region'] = region compute_handler = Compute(new_compute_config) self.compute_handlers.append(compute_handler) else: if cb == 'localhost': global CBH if cb in CBH and CBH[ cb].compute_handler.num_workers != self.num_workers: del CBH[cb] if cb in CBH: logger.info( '{} compute handler already started'.format(cb)) compute_handler = CBH[cb] self.compute_handlers.append(compute_handler) else: logger.info('Starting {} compute handler'.format(cb)) compute_handler = Compute(compute_config) CBH[cb] = compute_handler self.compute_handlers.append(compute_handler) else: compute_handler = Compute(compute_config) self.compute_handlers.append(compute_handler) self.token_bucket_q = Queue() self.pending_calls_q = Queue() self.job_monitor = JobMonitor(self.config, self.internal_storage, self.token_bucket_q)
def _init_keeper(config): global keeper compute_config = extract_compute_config(config) client = get_remote_client(compute_config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) keeper = threading.Thread(target=budget_keeper, args=( client, internal_storage, )) keeper.start()
def delete(name, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def __init__(self, config, executor_id, internal_storage, compute_handler): self.log_active = logger.getEffectiveLevel() != logging.WARNING self.config = config self.executor_id = executor_id self.storage_config = extract_storage_config(self.config) self.internal_storage = internal_storage self.compute_handler = compute_handler self.is_lithops_worker = is_lithops_worker() self.workers = self.config['lithops'].get('workers') logger.debug('ExecutorID {} - Total available workers: {}'.format( self.executor_id, self.workers)) executor = self.config['lithops']['executor'] self.runtime_name = self.config[executor]['runtime']
def build(name, file, config, backend): """ build a serverless runtime. """ verify_runtime_name(name) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) compute_handler.build_runtime(name, file)
def function_handler(payload): job = SimpleNamespace(**payload) processes = min(job.worker_processes, len(job.call_ids)) logger.info('Tasks received: {} - Concurrent workers: {}'.format( len(job.call_ids), processes)) storage_config = extract_storage_config(job.config) internal_storage = InternalStorage(storage_config) job.func = get_function_and_modules(job, internal_storage) job_data = get_function_data(job, internal_storage) if processes == 1: job_queue = queue.Queue() for task_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, task_id, data)) job_queue.put(ShutdownSentinel()) process_runner(job_queue, internal_storage) else: manager = SyncManager() manager.start() job_queue = manager.Queue() job_runners = [] for runner_id in range(processes): p = mp.Process(target=process_runner, args=(job_queue, internal_storage)) job_runners.append(p) p.start() logger.info('Worker process {} started'.format(runner_id)) for call_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, call_id, data)) for i in range(processes): job_queue.put(ShutdownSentinel()) for runner in job_runners: runner.join() manager.shutdown() # Delete modules path from syspath module_path = os.path.join(MODULES_DIR, job.job_key) if module_path in sys.path: sys.path.remove(module_path)
def __init__(self, storage_config=None, lithops_config=None, storage_backend=None, bucket=None): if lithops_config is None: self.lithops_config = config.default_config() if storage_config is None: self.storage_config = config.extract_storage_config( self.lithops_config) if storage_backend is None: self.storage_backend = self.storage_config['backend'] if bucket is not None: self.storage_config['bucket'] = bucket super().__init__(storage_config=self.storage_config, lithops_config=self.lithops_config, storage_backend=self.storage_backend)
def run_tests(test_to_run, config=None): global CONFIG, STORAGE_CONFIG, STORAGE CONFIG = json.load(args.config) if config else default_config() STORAGE_CONFIG = extract_storage_config(CONFIG) STORAGE = InternalStorage(STORAGE_CONFIG).storage suite = unittest.TestSuite() if test_to_run == 'all': suite.addTest(unittest.makeSuite(TestLithops)) else: try: suite.addTest(TestLithops(test_to_run)) except ValueError: print("unknown test, use: --help") sys.exit() runner = unittest.TextTestRunner() runner.run(suite)