def __init__(self, aws_lambda_config, storage_config): """ Initialize AWS Lambda Backend """ logger.debug('Creating AWS Lambda client') self.name = 'aws_lambda' self.aws_lambda_config = aws_lambda_config self.user_key = aws_lambda_config['access_key_id'][-4:] self.package = 'lithops_v{}_{}'.format(lithops.__version__, self.user_key) self.region_name = aws_lambda_config['region_name'] self.role_arn = aws_lambda_config['execution_role'] logger.debug('Creating Boto3 AWS Session and Lambda Client') self.aws_session = boto3.Session( aws_access_key_id=aws_lambda_config['access_key_id'], aws_secret_access_key=aws_lambda_config['secret_access_key'], region_name=self.region_name) self.lambda_client = self.aws_session.client( 'lambda', region_name=self.region_name) self.internal_storage = InternalStorage(storage_config) msg = COMPUTE_CLI_MSG.format('AWS Lambda') logger.info("{} - Region: {}".format(msg, self.region_name))
def create(name, mode, memory, timeout, config): setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) runtime_key = compute_handler.get_runtime_key(name) runtime_meta = compute_handler.create_runtime(name) elif mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) runtime_key = compute_handler.get_runtime_key(name) runtime_meta = compute_handler.create_runtime(name) else: raise Exception('Unknown execution mode {}'.format(mode)) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def create(name, storage, backend, memory, timeout, config): """ Create a serverless runtime """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] logger.info('Creating new lithops runtime: {}'.format(name)) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.create_runtime(name, mem, timeout=to) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise ("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def update(name, config, backend, storage): """ Update a serverless runtime """ if config: config = load_yaml_config(config) verify_runtime_name(name) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def update(name, config, backend, storage, debug): """ Update a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime update" command is only valid for serverless backends') storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) timeout = compute_config['runtime_memory'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.deploy_runtime(runtime[0], runtime[1], timeout) internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def __init__(self, gcp_functions_config, storage_config): self.log_active = logger.getEffectiveLevel() != logging.WARNING self.name = 'gcp_functions' self.gcp_functions_config = gcp_functions_config self.package = 'lithops_v' + __version__ self.region = gcp_functions_config['region'] self.service_account = gcp_functions_config['service_account'] self.project = gcp_functions_config['project_name'] self.credentials_path = gcp_functions_config['credentials_path'] self.num_retries = gcp_functions_config['retries'] self.retry_sleeps = gcp_functions_config['retry_sleeps'] # Instantiate storage client (used to upload function bin) self.internal_storage = InternalStorage( gcp_functions_config['storage']) # Setup Pub/Sub client try: # Get credentials from JSON file service_account_info = json.load(open(self.credentials_path)) credentials = jwt.Credentials.from_service_account_info( service_account_info, audience=AUDIENCE) credentials_pub = credentials.with_claims(audience=AUDIENCE) except: # Get credentials from gcp function environment credentials_pub = None self.publisher_client = pubsub_v1.PublisherClient( credentials=credentials_pub) log_msg = 'lithops v{} init for GCP Functions - Project: {} - Region: {}'.format( __version__, self.project, self.region) logger.info(log_msg) if not self.log_active: print(log_msg)
def delete(name, config, backend, storage, debug): """ delete a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime delete" command is only valid for serverless backends') storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def delete(name, config, backend, storage): """ delete a serverless runtime """ if config: config = load_yaml_config(config) setup_lithops_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if storage: config_ow['lithops']['storage'] = storage if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) if name: verify_runtime_name(name) else: name = config[mode]['runtime'] storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def deploy(name, storage, backend, memory, timeout, config, debug): """ deploy a serverless runtime """ setup_lithops_logger(logging.DEBUG) verify_runtime_name(name) if config: config = load_yaml_config(config) config_ow = set_config_ow(backend, storage, runtime_name=name) config = default_config(config, config_ow) if config['lithops']['mode'] != SERVERLESS: raise Exception('"lithops runtime create" command is only valid for serverless backends') logger.info('Creating new lithops runtime: {}'.format(name)) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, internal_storage) mem = memory if memory else compute_config['runtime_memory'] to = timeout if timeout else compute_config['runtime_timeout'] runtime_key = compute_handler.get_runtime_key(name, mem) runtime_meta = compute_handler.deploy_runtime(name, mem, timeout=to) internal_storage.put_runtime_meta(runtime_key, runtime_meta)
def __init__(self, gcp_functions_config, storage_config): self.name = 'gcp_functions' self.gcp_functions_config = gcp_functions_config self.package = 'lithops_v' + __version__ self.region = gcp_functions_config['region'] self.service_account = gcp_functions_config['service_account'] self.project = gcp_functions_config['project_name'] self.credentials_path = gcp_functions_config['credentials_path'] self.num_retries = gcp_functions_config['retries'] self.retry_sleep = gcp_functions_config['retry_sleep'] # Instantiate storage client (used to upload function bin) self.internal_storage = InternalStorage(storage_config) # Setup Pub/Sub client try: # Get credentials from JSON file service_account_info = json.load(open(self.credentials_path)) credentials = jwt.Credentials.from_service_account_info( service_account_info, audience=AUDIENCE) credentials_pub = credentials.with_claims(audience=AUDIENCE) except: # Get credentials from gcp function environment credentials_pub = None self.publisher_client = pubsub_v1.PublisherClient( credentials=credentials_pub) msg = COMPUTE_CLI_MSG.format('GCP Functions') logger.info("{} - Region: {} - Project: {}".format( msg, self.region, self.project))
def result(self, throw_except=True, internal_storage=None): """ Return the value returned by the call. If the call raised an exception, this method will raise the same exception If the future is cancelled before completing then CancelledError will be raised. :param throw_except: Reraise exception if call raised. Default true. :param internal_storage: Storage handler to poll cloud storage. Default None. :return: Result of the call. :raises CancelledError: If the job is cancelled before completed. :raises TimeoutError: If job is not complete after `timeout` seconds. """ if not self._produce_output: self._set_state(ResponseFuture.State.Done) if self._state == ResponseFuture.State.New: raise ValueError("task not yet invoked") if self._state == ResponseFuture.State.Done: return self._return_val if self._state == ResponseFuture.State.Futures: return self._new_futures if internal_storage is None: internal_storage = InternalStorage(storage_config=self._storage_config) self.status(throw_except=throw_except, internal_storage=internal_storage) if self._state == ResponseFuture.State.Done: return self._return_val if not self._call_output: call_output = internal_storage.get_call_output(self.executor_id, self.job_id, self.call_id) self._output_query_count += 1 while call_output is None and self._output_query_count < self.GET_RESULT_MAX_RETRIES: time.sleep(self.GET_RESULT_SLEEP_SECS) call_output = internal_storage.get_call_output(self.executor_id, self.job_id, self.call_id) self._output_query_count += 1 if call_output is None: if throw_except: raise Exception('Unable to get the result from call {} - ' 'Activation ID: {}'.format(self.call_id, self.activation_id)) else: self._set_state(ResponseFuture.State.Error) return None self._call_output = pickle.loads(call_output) self.stats['host_result_done_tstamp'] = time.time() self.stats['host_result_query_count'] = self._output_query_count logger.debug('ExecutorID {} | JobID {} - Got output from call {} - Activation ' 'ID: {}'.format(self.executor_id, self.job_id, self.call_id, self.activation_id)) self._return_val = self._call_output['result'] if not self._return_val else self._return_val self._set_state(ResponseFuture.State.Done) return self._return_val
def delete_runtime(name, config=None): config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_compute_config(config) compute_handler = Compute(compute_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def delete(name, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def __init__(self, config, num_invokers, log_level): self.config = config self.num_invokers = num_invokers self.log_level = log_level storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.remote_invoker = self.config['lithops'].get( 'remote_invoker', False) self.rabbitmq_monitor = self.config['lithops'].get( 'rabbitmq_monitor', False) if self.rabbitmq_monitor: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') self.num_workers = self.config['lithops'].get('workers') logger.info('Total workers: {}'.format(self.num_workers)) serverless_config = extract_serverless_config(self.config) self.serverless_handler = ServerlessHandler(serverless_config, storage_config) self.token_bucket_q = mp.Queue() self.pending_calls_q = mp.Queue() self.job_monitor = JobMonitor(self.config, self.internal_storage, self.token_bucket_q)
def clean_all(config=None): logger.info('Cleaning all Lithops information') config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) default_executor = config['lithops']['executor'] if default_executor == 'localhost': compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif default_executor == 'serverless': compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif default_executor == 'standalone': compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(STORAGE_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def function_invoker(job_payload): """ Method used as a remote invoker """ config = job_payload['config'] job = SimpleNamespace(**job_payload['job']) env = { 'LITHOPS_WORKER': 'True', 'PYTHONUNBUFFERED': 'True', '__LITHOPS_SESSION_ID': job.job_key } os.environ.update(env) # Create the monitoring system monitoring_backend = config['lithops']['monitoring'].lower() monitoring_config = config.get(monitoring_backend) job_monitor = JobMonitor(monitoring_backend, monitoring_config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) serverless_config = extract_serverless_config(config) compute_handler = ServerlessHandler(serverless_config, storage_config) # Create the invokder invoker = FaaSRemoteInvoker(config, job.executor_id, internal_storage, compute_handler, job_monitor) invoker.run_job(job)
def _create_jobs_from_futures(fs, internal_storage): """ Creates a dummy job necessary for the job monitor """ jobs = [] present_jobs = {f.job_key for f in fs} for job_key in present_jobs: job_data = {} job = SimpleNamespace() job.futures = [f for f in fs if f.job_key == job_key] job.total_calls = len(job.futures) f = job.futures[0] job.executor_id = f.executor_id job.job_id = f.job_id job.job_key = f.job_key job_data['job'] = job if internal_storage and internal_storage.backend == f._storage_config[ 'backend']: job_data['internal_storage'] = internal_storage else: job_data['internal_storage'] = InternalStorage(f._storage_config) jobs.append(job_data) return jobs
def __init__(self, code_engine_config, storage_config): logger.debug("Creating Code Engine client") self.log_active = logger.getEffectiveLevel() != logging.WARNING self.name = 'code_engine' self.code_engine_config = code_engine_config self.is_lithops_worker = is_lithops_worker() self.storage_config = storage_config self.internal_storage = InternalStorage(storage_config) config.load_kube_config( config_file=code_engine_config.get('kubectl_config')) self.capi = client.CustomObjectsApi() self.user_agent = code_engine_config['user_agent'] contexts = config.list_kube_config_contexts( config_file=code_engine_config.get('kubectl_config')) current_context = contexts[1].get('context') self.user = current_context.get('user') self.user_key = self.user self.package = 'lithops_v{}_{}'.format(__version__, self.user_key) self.namespace = current_context.get('namespace', 'default') self.cluster = current_context.get('cluster') log_msg = ('Lithops v{} init for Code Engine - Namespace: {} - ' 'Cluster: {} - User {}'.format(__version__, self.namespace, self.cluster, self.user)) if not self.log_active: print(log_msg) self.job_def_ids = set() logger.info("Code Engine client created successfully")
def clean(mode, config, debug): log_level = 'INFO' if not debug else 'DEBUG' setup_logger(log_level) logger.info('Cleaning all Lithops information') config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) mode = config['lithops']['mode'] if not mode else mode if mode == LOCALHOST: compute_config = extract_localhost_config(config) compute_handler = LocalhostHandler(compute_config) elif mode == SERVERLESS: compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) elif mode == STANDALONE: compute_config = extract_standalone_config(config) compute_handler = StandaloneHandler(compute_config) compute_handler.clean() # Clean object storage temp dirs storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean localhost executor temp dirs shutil.rmtree(LITHOPS_TEMP_DIR, ignore_errors=True) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)
def function_handler(payload): job = SimpleNamespace(**payload) setup_lithops_logger(job.log_level) processes = min(job.worker_processes, len(job.call_ids)) logger.info('Tasks received: {} - Concurrent processes: {}'.format( len(job.call_ids), processes)) env = job.extra_env env['LITHOPS_WORKER'] = 'True' env['PYTHONUNBUFFERED'] = 'True' os.environ.update(env) storage_config = extract_storage_config(job.config) internal_storage = InternalStorage(storage_config) job.func = get_function_and_modules(job, internal_storage) job_data = get_function_data(job, internal_storage) if processes == 1: job_queue = queue.Queue() for call_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, call_id, data)) job_queue.put(ShutdownSentinel()) process_runner(job_queue) else: manager = SyncManager() manager.start() job_queue = manager.Queue() job_runners = [] for call_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, call_id, data)) for i in range(processes): job_queue.put(ShutdownSentinel()) for runner_id in range(processes): p = mp.Process(target=process_runner, args=(job_queue, )) job_runners.append(p) p.start() logger.info('Worker process {} started'.format(runner_id)) for runner in job_runners: runner.join() manager.shutdown() # Delete modules path from syspath module_path = os.path.join(MODULES_DIR, job.job_key) if module_path in sys.path: sys.path.remove(module_path) # Unset specific job env vars for key in job.extra_env: os.environ.pop(key, None) os.environ.pop('__LITHOPS_TOTAL_EXECUTORS', None)
def runtime_packages(storage_config): logger.info("Extracting preinstalled Python modules...") internal_storage = InternalStorage(storage_config) runtime_meta = dict() mods = list(pkgutil.iter_modules()) runtime_meta['preinstalls'] = [entry for entry in sorted([[mod, is_pkg] for _, mod, is_pkg in mods])] python_version = sys.version_info runtime_meta['python_ver'] = str(python_version[0])+"."+str(python_version[1]) activation_id = storage_config['activation_id'] status_key = create_runtime_meta_key(JOBS_PREFIX, activation_id) logger.debug("Runtime metadata key {}".format(status_key)) dmpd_response_status = json.dumps(runtime_meta) drs = sizeof_fmt(len(dmpd_response_status)) logger.info("Storing execution stats - Size: {}".format(drs)) internal_storage.put_data(status_key, dmpd_response_status)
def delete(name, config, backend): """ delete a serverless runtime """ verify_runtime_name(name) setup_logger(logging.DEBUG) mode = SERVERLESS config_ow = {'lithops': {'mode': mode}} if backend: config_ow[mode] = {'backend': backend} config = default_config(config, config_ow) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: compute_handler.delete_runtime(runtime[0], runtime[1]) runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) internal_storage.delete_runtime_meta(runtime_key)
def __init__(self, config, num_invokers, log_level): self.config = config self.num_invokers = num_invokers self.log_level = log_level storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) compute_config = extract_compute_config(self.config) self.remote_invoker = self.config['lithops'].get( 'remote_invoker', False) self.rabbitmq_monitor = self.config['lithops'].get( 'rabbitmq_monitor', False) if self.rabbitmq_monitor: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') self.num_workers = self.config['lithops'].get('workers') logger.debug('Total workers: {}'.format(self.num_workers)) self.compute_handlers = [] cb = compute_config['backend'] regions = compute_config[cb].get('region') if regions and type(regions) == list: for region in regions: new_compute_config = compute_config.copy() new_compute_config[cb]['region'] = region compute_handler = Compute(new_compute_config) self.compute_handlers.append(compute_handler) else: if cb == 'localhost': global CBH if cb in CBH and CBH[ cb].compute_handler.num_workers != self.num_workers: del CBH[cb] if cb in CBH: logger.info( '{} compute handler already started'.format(cb)) compute_handler = CBH[cb] self.compute_handlers.append(compute_handler) else: logger.info('Starting {} compute handler'.format(cb)) compute_handler = Compute(compute_config) CBH[cb] = compute_handler self.compute_handlers.append(compute_handler) else: compute_handler = Compute(compute_config) self.compute_handlers.append(compute_handler) self.token_bucket_q = Queue() self.pending_calls_q = Queue() self.job_monitor = JobMonitor(self.config, self.internal_storage, self.token_bucket_q)
def update(name, config): verify_runtime_name(name) setup_logger(logging.DEBUG) config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_serverless_config(config) compute_handler = ServerlessHandler(compute_config, storage_config) timeout = config['lithops']['runtime_timeout'] logger.info('Updating runtime: {}'.format(name)) runtimes = compute_handler.list_runtimes(name) for runtime in runtimes: runtime_key = compute_handler.get_runtime_key(runtime[0], runtime[1]) runtime_meta = compute_handler.create_runtime(runtime[0], runtime[1], timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise("Unable to upload 'preinstalled-modules' file into {}".format(internal_storage.backend))
def _init_keeper(config): global keeper compute_config = extract_compute_config(config) client = get_remote_client(compute_config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) keeper = threading.Thread(target=budget_keeper, args=( client, internal_storage, )) keeper.start()
def create_runtime(name, memory=None, config=None): config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_compute_config(config) compute_handler = Compute(compute_config) memory = config['lithops']['runtime_memory'] if not memory else memory timeout = config['lithops']['runtime_timeout'] logger.info('Creating runtime: {}, memory: {}'.format(name, memory)) runtime_key = compute_handler.get_runtime_key(name, memory) runtime_meta = compute_handler.create_runtime(name, memory, timeout=timeout) try: internal_storage.put_runtime_meta(runtime_key, runtime_meta) except Exception: raise ("Unable to upload 'preinstalled-modules' file into {}".format( internal_storage.backend))
def function_handler(payload): job = SimpleNamespace(**payload) processes = min(job.worker_processes, len(job.call_ids)) logger.info('Tasks received: {} - Concurrent workers: {}'.format( len(job.call_ids), processes)) storage_config = extract_storage_config(job.config) internal_storage = InternalStorage(storage_config) job.func = get_function_and_modules(job, internal_storage) job_data = get_function_data(job, internal_storage) if processes == 1: job_queue = queue.Queue() for task_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, task_id, data)) job_queue.put(ShutdownSentinel()) process_runner(job_queue, internal_storage) else: manager = SyncManager() manager.start() job_queue = manager.Queue() job_runners = [] for runner_id in range(processes): p = mp.Process(target=process_runner, args=(job_queue, internal_storage)) job_runners.append(p) p.start() logger.info('Worker process {} started'.format(runner_id)) for call_id in job.call_ids: data = job_data.pop(0) job_queue.put((job, call_id, data)) for i in range(processes): job_queue.put(ShutdownSentinel()) for runner in job_runners: runner.join() manager.shutdown() # Delete modules path from syspath module_path = os.path.join(MODULES_DIR, job.job_key) if module_path in sys.path: sys.path.remove(module_path)
def get_result(fs, throw_except=True, timeout=None, THREADPOOL_SIZE=128, WAIT_DUR_SEC=1, internal_storage=None): """ For getting the results from all function activations :param fs: Futures list. Default None :param throw_except: Reraise exception if call raised. Default True. :param verbose: Shows some information prints. Default False :param timeout: Timeout for waiting for results. :param THREADPOOL_SIZE: Number of threads to use. Default 128 :param WAIT_DUR_SEC: Time interval between each check. :return: The result of the future/s """ if type(fs) != list: fs = [fs] if not internal_storage: internal_storage = InternalStorage(fs[0]._storage_config) fs_done, _ = wait(fs=fs, throw_except=throw_except, timeout=timeout, download_results=True, internal_storage=internal_storage, THREADPOOL_SIZE=THREADPOOL_SIZE, WAIT_DUR_SEC=WAIT_DUR_SEC) result = [] fs_done = [f for f in fs_done if not f.futures and f._produce_output] for f in fs_done: result.append( f.result(throw_except=throw_except, internal_storage=internal_storage)) logger.debug("Finished getting results") if len(result) == 1: return result[0] return result
def run_tests(test_to_run, config=None): global CONFIG, STORAGE_CONFIG, STORAGE CONFIG = json.load(args.config) if config else default_config() STORAGE_CONFIG = extract_storage_config(CONFIG) STORAGE = InternalStorage(STORAGE_CONFIG).storage suite = unittest.TestSuite() if test_to_run == 'all': suite.addTest(unittest.makeSuite(TestLithops)) else: try: suite.addTest(TestLithops(test_to_run)) except ValueError: print("unknown test, use: --help") sys.exit() runner = unittest.TextTestRunner() runner.run(suite)
def clean_all(config=None): logger.info('Cleaning all Lithops information') config = default_config(config) storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) compute_config = extract_compute_config(config) compute_handler = Compute(compute_config) # Clean localhost executor temp dirs shutil.rmtree(STORAGE_FOLDER, ignore_errors=True) shutil.rmtree(DOCKER_FOLDER, ignore_errors=True) # Clean object storage temp dirs compute_handler.delete_all_runtimes() storage = internal_storage.storage clean_bucket(storage, storage_config['bucket'], RUNTIMES_PREFIX, sleep=1) clean_bucket(storage, storage_config['bucket'], JOBS_PREFIX, sleep=1) # Clean local lithops cache shutil.rmtree(CACHE_DIR, ignore_errors=True)