class IBMCloudFunctionsBackend: """ A wrap-up around IBM Cloud Functions backend. """ def __init__(self, ibm_cf_config): logger.debug("Creating IBM Cloud Functions client") self.log_active = logger.getEffectiveLevel() != logging.WARNING self.name = 'ibm_cf' self.ibm_cf_config = ibm_cf_config self.is_lithops_function = is_lithops_function() self.user_agent = ibm_cf_config['user_agent'] self.region = ibm_cf_config['region'] self.endpoint = ibm_cf_config['regions'][self.region]['endpoint'] self.namespace = ibm_cf_config['regions'][self.region]['namespace'] self.namespace_id = ibm_cf_config['regions'][self.region].get( 'namespace_id', None) self.api_key = ibm_cf_config['regions'][self.region].get( 'api_key', None) self.iam_api_key = ibm_cf_config.get('iam_api_key', None) logger.info("Set IBM CF Namespace to {}".format(self.namespace)) logger.info("Set IBM CF Endpoint to {}".format(self.endpoint)) self.user_key = self.api_key[: 5] if self.api_key else self.iam_api_key[: 5] self.package = 'lithops_v{}_{}'.format(__version__, self.user_key) if self.api_key: enc_api_key = str.encode(self.api_key) auth_token = base64.encodebytes(enc_api_key).replace(b'\n', b'') auth = 'Basic %s' % auth_token.decode('UTF-8') self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace, auth=auth, user_agent=self.user_agent) elif self.iam_api_key: token_manager = DefaultTokenManager(api_key_id=self.iam_api_key) token_filename = os.path.join(CACHE_DIR, 'ibm_cf', 'iam_token') if 'token' in self.ibm_cf_config: logger.debug( "Using IBM IAM API Key - Reusing Token from config") token_manager._token = self.ibm_cf_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cf_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int( (token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format( token_manager._expiry_time, token_minutes_diff)) elif os.path.exists(token_filename): logger.debug( "Using IBM IAM API Key - Reusing Token from local cache") token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int( (token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format( token_manager._expiry_time, token_minutes_diff)) if (token_manager._is_expired() or token_minutes_diff < 11) and not is_lithops_function(): logger.debug( "Using IBM IAM API Key - Token expired. Requesting new token" ) token_manager._token = None token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) ibm_cf_config['token'] = token_manager._token ibm_cf_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') auth_token = token_manager._token auth = 'Bearer ' + auth_token self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace_id, auth=auth, user_agent=self.user_agent) log_msg = ( 'Lithops v{} init for IBM Cloud Functions - Namespace: {} - ' 'Region: {}'.format(__version__, self.namespace, self.region)) if not self.log_active: print(log_msg) logger.info("IBM CF client created successfully") def _format_action_name(self, runtime_name, runtime_memory): runtime_name = runtime_name.replace('/', '_').replace(':', '_') return '{}_{}MB'.format(runtime_name, runtime_memory) def _unformat_action_name(self, action_name): runtime_name, memory = action_name.rsplit('_', 1) image_name = runtime_name.replace('_', '/', 1) image_name = image_name.replace('_', ':', -1) return image_name, int(memory.replace('MB', '')) def _get_default_runtime_image_name(self): python_version = version_str(sys.version_info) return ibmcf_config.RUNTIME_DEFAULT[python_version] def _delete_function_handler_zip(self): os.remove(ibmcf_config.FH_ZIP_LOCATION) def build_runtime(self, docker_image_name, dockerfile): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.info('Building a new docker image from Dockerfile') logger.info('Docker image name: {}'.format(docker_image_name)) if dockerfile: cmd = 'docker build -t {} -f {} .'.format(docker_image_name, dockerfile) else: cmd = 'docker build -t {} .'.format(docker_image_name) res = os.system(cmd) if res != 0: raise Exception('There was an error building the runtime') cmd = 'docker push {}'.format(docker_image_name) res = os.system(cmd) if res != 0: raise Exception( 'There was an error pushing the runtime to the container registry' ) def create_runtime(self, docker_image_name, memory, timeout): """ Creates a new runtime into IBM CF namespace from an already built Docker image """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() runtime_meta = self._generate_runtime_meta(docker_image_name) logger.info( 'Creating new Lithops runtime based on Docker image {}'.format( docker_image_name)) self.cf_client.create_package(self.package) action_name = self._format_action_name(docker_image_name, memory) create_function_handler_zip(ibmcf_config.FH_ZIP_LOCATION, '__main__.py', __file__) with open(ibmcf_config.FH_ZIP_LOCATION, "rb") as action_zip: action_bin = action_zip.read() self.cf_client.create_action(self.package, action_name, docker_image_name, code=action_bin, memory=memory, is_binary=True, timeout=timeout * 1000) self._delete_function_handler_zip() return runtime_meta def delete_runtime(self, docker_image_name, memory): """ Deletes a runtime """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() action_name = self._format_action_name(docker_image_name, memory) self.cf_client.delete_action(self.package, action_name) def delete_all_runtimes(self): """ Deletes all runtimes from all packages """ packages = self.cf_client.list_packages() for pkg in packages: if (pkg['name'].startswith('lithops') and pkg['name'].endswith(self.user_key)) or \ (pkg['name'].startswith('lithops') and pkg['name'].count('_') == 1): actions = self.cf_client.list_actions(pkg['name']) while actions: for action in actions: self.cf_client.delete_action(pkg['name'], action['name']) actions = self.cf_client.list_actions(pkg['name']) self.cf_client.delete_package(pkg['name']) def list_runtimes(self, docker_image_name='all'): """ List all the runtimes deployed in the IBM CF service return: list of tuples (docker_image_name, memory) """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() runtimes = [] actions = self.cf_client.list_actions(self.package) for action in actions: action_image_name, memory = self._unformat_action_name( action['name']) if docker_image_name == action_image_name or docker_image_name == 'all': runtimes.append((action_image_name, memory)) return runtimes def invoke(self, docker_image_name, runtime_memory, payload): """ Invoke -- return information about this invocation """ action_name = self._format_action_name(docker_image_name, runtime_memory) activation_id = self.cf_client.invoke( package=self.package, action_name=action_name, payload=payload, is_ow_action=self.is_lithops_function) return activation_id def get_runtime_key(self, docker_image_name, runtime_memory): """ Method that creates and returns the runtime key. Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ action_name = self._format_action_name(docker_image_name, runtime_memory) runtime_key = os.path.join(self.name, self.region, self.namespace, action_name) return runtime_key def _generate_runtime_meta(self, docker_image_name): """ Extract installed Python modules from docker image """ action_code = """ import sys import pkgutil def main(args): print("Extracting preinstalled Python modules...") runtime_meta = dict() mods = list(pkgutil.iter_modules()) runtime_meta["preinstalls"] = [entry for entry in sorted([[mod, is_pkg] for _, mod, is_pkg in mods])] python_version = sys.version_info runtime_meta["python_ver"] = str(python_version[0])+"."+str(python_version[1]) print("Done!") return runtime_meta """ runtime_memory = 128 # old_stdout = sys.stdout # sys.stdout = open(os.devnull, 'w') action_name = self._format_action_name(docker_image_name, runtime_memory) self.cf_client.create_package(self.package) self.cf_client.create_action(self.package, action_name, docker_image_name, is_binary=False, code=textwrap.dedent(action_code), memory=runtime_memory, timeout=30000) # sys.stdout = old_stdout logger.debug("Extracting Python modules list from: {}".format( docker_image_name)) try: retry_invoke = True while retry_invoke: retry_invoke = False runtime_meta = self.cf_client.invoke_with_result( self.package, action_name) if 'activationId' in runtime_meta: retry_invoke = True except Exception: raise ("Unable to invoke 'modules' action") try: self.delete_runtime(docker_image_name, runtime_memory) except Exception: raise Exception("Unable to delete 'modules' action") if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) return runtime_meta
class OpenWhiskBackend: """ A wrap-up around OpenWhisk Functions backend. """ def __init__(self, ow_config, internal_storage): logger.debug("Creating OpenWhisk client") self.name = 'openwhisk' self.type = 'faas' self.ow_config = ow_config self.is_lithops_worker = is_lithops_worker() self.user_agent = ow_config['user_agent'] self.endpoint = ow_config['endpoint'] self.namespace = ow_config['namespace'] self.api_key = ow_config['api_key'] self.insecure = ow_config.get('insecure', False) logger.debug("Set OpenWhisk Endpoint to {}".format(self.endpoint)) logger.debug("Set OpenWhisk Namespace to {}".format(self.namespace)) logger.debug("Set OpenWhisk Insecure to {}".format(self.insecure)) self.user_key = self.api_key[:5] self.package = 'lithops_v{}_{}'.format(__version__, self.user_key) self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace, api_key=self.api_key, insecure=self.insecure, user_agent=self.user_agent) msg = COMPUTE_CLI_MSG.format('OpenWhisk') logger.info("{} - Namespace: {}".format(msg, self.namespace)) def _format_function_name(self, runtime_name, runtime_memory): runtime_name = runtime_name.replace('/', '_').replace(':', '_') return '{}_{}MB'.format(runtime_name, runtime_memory) def _unformat_function_name(self, action_name): runtime_name, memory = action_name.rsplit('_', 1) image_name = runtime_name.replace('_', '/', 1) image_name = image_name.replace('_', ':', -1) return image_name, int(memory.replace('MB', '')) def _get_default_runtime_image_name(self): python_version = version_str(sys.version_info) return ow_config.RUNTIME_DEFAULT[python_version] def _delete_function_handler_zip(self): os.remove(ow_config.FH_ZIP_LOCATION) def build_runtime(self, docker_image_name, dockerfile, extra_args=[]): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.info('Building a new docker image from Dockerfile') logger.info('Docker image name: {}'.format(docker_image_name)) if dockerfile: cmd = '{} build -t {} -f {} . '.format(ow_config.DOCKER_PATH, docker_image_name, dockerfile) else: cmd = '{} build -t {} . '.format(ow_config.DOCKER_PATH, docker_image_name) cmd = cmd + ' '.join(extra_args) if logger.getEffectiveLevel() != logging.DEBUG: cmd = cmd + " >{} 2>&1".format(os.devnull) res = os.system(cmd) if res != 0: raise Exception('There was an error building the runtime') cmd = 'docker push {}'.format(docker_image_name) res = os.system(cmd) if res != 0: raise Exception( 'There was an error pushing the runtime to the container registry' ) logger.info('Building done!') def deploy_runtime(self, docker_image_name, memory, timeout): """ Deploys a new runtime into IBM CF namespace from an already built Docker image """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() logger.debug( f"Deploying runtime: {docker_image_name} - Memory: {memory} Timeout: {timeout}" ) self.cf_client.create_package(self.package) action_name = self._format_function_name(docker_image_name, memory) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(ow_config.FH_ZIP_LOCATION, entry_point, '__main__.py') with open(ow_config.FH_ZIP_LOCATION, "rb") as action_zip: action_bin = action_zip.read() self.cf_client.create_action(self.package, action_name, docker_image_name, code=action_bin, memory=memory, is_binary=True, timeout=timeout * 1000) self._delete_function_handler_zip() return self._generate_runtime_meta(docker_image_name, memory) def delete_runtime(self, docker_image_name, memory): """ Deletes a runtime """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() action_name = self._format_function_name(docker_image_name, memory) self.cf_client.delete_action(self.package, action_name) def clean(self): """ Deletes all runtimes from all packages """ packages = self.cf_client.list_packages() for pkg in packages: if pkg['name'].startswith('lithops') and pkg['name'].endswith( self.user_key): actions = self.cf_client.list_actions(pkg['name']) while actions: for action in actions: self.cf_client.delete_action(pkg['name'], action['name']) actions = self.cf_client.list_actions(pkg['name']) self.cf_client.delete_package(pkg['name']) def list_runtimes(self, docker_image_name='all'): """ List all the runtimes deployed in the IBM CF service return: list of tuples (docker_image_name, memory) """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() runtimes = [] actions = self.cf_client.list_actions(self.package) for action in actions: action_image_name, memory = self._unformat_function_name( action['name']) if docker_image_name == action_image_name or docker_image_name == 'all': runtimes.append((action_image_name, memory)) return runtimes def invoke(self, docker_image_name, runtime_memory, payload): """ Invoke -- return information about this invocation """ action_name = self._format_function_name(docker_image_name, runtime_memory) activation_id = self.cf_client.invoke(self.package, action_name, payload, self.is_lithops_worker) return activation_id def get_runtime_key(self, docker_image_name, runtime_memory): """ Method that creates and returns the runtime key. Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ action_name = self._format_function_name(docker_image_name, runtime_memory) runtime_key = os.path.join(self.name, self.namespace, action_name) return runtime_key def _generate_runtime_meta(self, docker_image_name, memory): """ Extract installed Python modules from the docker image """ logger.debug("Extracting Python modules list from: {}".format( docker_image_name)) action_name = self._format_function_name(docker_image_name, memory) payload = { 'log_level': logger.getEffectiveLevel(), 'get_preinstalls': True } try: retry_invoke = True while retry_invoke: retry_invoke = False runtime_meta = self.cf_client.invoke_with_result( self.package, action_name, payload) if 'activationId' in runtime_meta: retry_invoke = True except Exception as e: raise ("Unable to extract runtime preinstalls: {}".format(e)) if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) return runtime_meta
class IBMCloudFunctionsBackend: """ A wrap-up around IBM Cloud Functions backend. """ def __init__(self, ibm_cf_config, internal_storage): logger.debug("Creating IBM Cloud Functions client") self.name = 'ibm_cf' self.type = 'faas' self.config = ibm_cf_config self.is_lithops_worker = is_lithops_worker() self.user_agent = ibm_cf_config['user_agent'] self.region = ibm_cf_config['region'] self.endpoint = ibm_cf_config['regions'][self.region]['endpoint'] self.namespace = ibm_cf_config['regions'][self.region]['namespace'] self.namespace_id = ibm_cf_config['regions'][self.region].get('namespace_id', None) self.api_key = ibm_cf_config['regions'][self.region].get('api_key', None) self.iam_api_key = ibm_cf_config.get('iam_api_key', None) logger.debug("Set IBM CF Namespace to {}".format(self.namespace)) logger.debug("Set IBM CF Endpoint to {}".format(self.endpoint)) self.user_key = self.api_key.split(':')[1][:4] if self.api_key else self.iam_api_key[:4] self.package = 'lithops_v{}_{}'.format(__version__, self.user_key) if self.api_key: enc_api_key = str.encode(self.api_key) auth_token = base64.encodebytes(enc_api_key).replace(b'\n', b'') auth = 'Basic %s' % auth_token.decode('UTF-8') self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace, auth=auth, user_agent=self.user_agent) elif self.iam_api_key: api_key_type = 'IAM' token = self.config.get('token', None) token_expiry_time = self.config.get('token_expiry_time', None) self.ibm_token_manager = IBMTokenManager(self.iam_api_key, api_key_type, token, token_expiry_time) token, token_expiry_time = self.ibm_token_manager.get_token() self.config['token'] = token self.config['token_expiry_time'] = token_expiry_time auth = 'Bearer ' + token self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace_id, auth=auth, user_agent=self.user_agent) msg = COMPUTE_CLI_MSG.format('IBM CF') logger.info("{} - Region: {} - Namespace: {}".format(msg, self.region, self.namespace)) def _format_action_name(self, runtime_name, runtime_memory): runtime_name = runtime_name.replace('/', '_').replace(':', '_') return '{}_{}MB'.format(runtime_name, runtime_memory) def _unformat_action_name(self, action_name): runtime_name, memory = action_name.rsplit('_', 1) image_name = runtime_name.replace('_', '/', 1) image_name = image_name.replace('_', ':', -1) return image_name, int(memory.replace('MB', '')) def _get_default_runtime_image_name(self): python_version = version_str(sys.version_info) return ibmcf_config.RUNTIME_DEFAULT[python_version] def _delete_function_handler_zip(self): os.remove(ibmcf_config.FH_ZIP_LOCATION) def build_runtime(self, docker_image_name, dockerfile): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.info('Building a new docker image from Dockerfile') logger.info('Docker image name: {}'.format(docker_image_name)) if dockerfile: cmd = 'docker build -t {} -f {} .'.format(docker_image_name, dockerfile) else: cmd = 'docker build -t {} .'.format(docker_image_name) res = os.system(cmd) if res != 0: raise Exception('There was an error building the runtime') cmd = 'docker push {}'.format(docker_image_name) res = os.system(cmd) if res != 0: raise Exception('There was an error pushing the runtime to the container registry') def create_runtime(self, docker_image_name, memory, timeout): """ Creates a new runtime into IBM CF namespace from an already built Docker image """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() logger.info('Creating new Lithops runtime based on Docker image {}'.format(docker_image_name)) self.cf_client.create_package(self.package) action_name = self._format_action_name(docker_image_name, memory) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_handler_zip(ibmcf_config.FH_ZIP_LOCATION, entry_point, '__main__.py') with open(ibmcf_config.FH_ZIP_LOCATION, "rb") as action_zip: action_bin = action_zip.read() self.cf_client.create_action(self.package, action_name, docker_image_name, code=action_bin, memory=memory, is_binary=True, timeout=timeout * 1000) self._delete_function_handler_zip() runtime_meta = self._generate_runtime_meta(docker_image_name, memory) return runtime_meta def delete_runtime(self, docker_image_name, memory): """ Deletes a runtime """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() action_name = self._format_action_name(docker_image_name, memory) self.cf_client.delete_action(self.package, action_name) def clean(self): """ Deletes all runtimes from all packages """ packages = self.cf_client.list_packages() for pkg in packages: if (pkg['name'].startswith('lithops') and pkg['name'].endswith(self.user_key)) or \ (pkg['name'].startswith('lithops') and pkg['name'].count('_') == 1): actions = self.cf_client.list_actions(pkg['name']) while actions: for action in actions: self.cf_client.delete_action(pkg['name'], action['name']) actions = self.cf_client.list_actions(pkg['name']) self.cf_client.delete_package(pkg['name']) def list_runtimes(self, docker_image_name='all'): """ List all the runtimes deployed in the IBM CF service return: list of tuples (docker_image_name, memory) """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() runtimes = [] actions = self.cf_client.list_actions(self.package) for action in actions: action_image_name, memory = self._unformat_action_name(action['name']) if docker_image_name == action_image_name or docker_image_name == 'all': runtimes.append((action_image_name, memory)) return runtimes def invoke(self, docker_image_name, runtime_memory, payload): """ Invoke -- return information about this invocation """ action_name = self._format_action_name(docker_image_name, runtime_memory) activation_id = self.cf_client.invoke(package=self.package, action_name=action_name, payload=payload, is_ow_action=self.is_lithops_worker) if activation_id == 401: # unauthorized. Probably token expired if using IAM auth if self.iam_api_key and not self.is_lithops_worker: self._refresh_cf_client() return self.invoke(docker_image_name, runtime_memory, payload) else: raise Exception('Unauthorized. Review your API key') return activation_id def _refresh_cf_client(self): """ Recreates the OpenWhisk client with a new token. This is only called by the invoke method when it receives a 401 error. """ token_mutex.acquire() token, token_expiry_time = self.ibm_token_manager.get_token() if token != self.config['token']: self.config['token'] = token self.config['token_expiry_time'] = token_expiry_time auth = 'Bearer ' + token self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace_id, auth=auth, user_agent=self.user_agent) token_mutex.release() def get_runtime_key(self, docker_image_name, runtime_memory): """ Method that creates and returns the runtime key. Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ action_name = self._format_action_name(docker_image_name, runtime_memory) runtime_key = os.path.join(self.name, self.region, self.namespace, action_name) return runtime_key def _generate_runtime_meta(self, docker_image_name, memory): """ Extract installed Python modules from the docker image """ logger.debug("Extracting Python modules list from: {}".format(docker_image_name)) action_name = self._format_action_name(docker_image_name, memory) payload = {'log_level': logger.getEffectiveLevel(), 'get_preinstalls': True} try: retry_invoke = True while retry_invoke: retry_invoke = False runtime_meta = self.cf_client.invoke_with_result(self.package, action_name, payload) if 'activationId' in runtime_meta: retry_invoke = True except Exception as e: raise ("Unable to extract runtime preinstalls: {}".format(e)) if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) return runtime_meta def calc_cost(self, runtimes, memory, *argv, **arg): """ returns total cost associated with executing the calling function-executor's job. :params *argv and **arg: made to support compatibility with similarly named functions in alternative computational backends. """ return ibmcf_config.UNIT_PRICE * sum(runtimes[i] * memory[i] / 1024 for i in range(len(runtimes)))
class OpenWhiskBackend: """ A wrap-up around OpenWhisk Functions backend. """ def __init__(self, ow_config, storage_config): logger.debug("Creating OpenWhisk client") self.log_active = logger.getEffectiveLevel() != logging.WARNING self.name = 'openwhisk' self.ow_config = ow_config self.is_lithops_worker = is_lithops_worker() self.user_agent = ow_config['user_agent'] self.endpoint = ow_config['endpoint'] self.namespace = ow_config['namespace'] self.api_key = ow_config['api_key'] self.insecure = ow_config.get('insecure', False) logger.info("Set OpenWhisk Endpoint to {}".format(self.endpoint)) logger.info("Set OpenWhisk Namespace to {}".format(self.namespace)) logger.info("Set OpenWhisk Insecure to {}".format(self.insecure)) self.user_key = self.api_key[:5] self.package = 'lithops_v{}_{}'.format(__version__, self.user_key) self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace, api_key=self.api_key, insecure=self.insecure, user_agent=self.user_agent) log_msg = ('Lithops v{} init for OpenWhisk - Namespace: {}' .format(__version__, self.namespace)) if not self.log_active: print(log_msg) logger.info("OpenWhisk client created successfully") def _format_action_name(self, runtime_name, runtime_memory): runtime_name = runtime_name.replace('/', '_').replace(':', '_') return '{}_{}MB'.format(runtime_name, runtime_memory) def _unformat_action_name(self, action_name): runtime_name, memory = action_name.rsplit('_', 1) image_name = runtime_name.replace('_', '/', 1) image_name = image_name.replace('_', ':', -1) return image_name, int(memory.replace('MB', '')) def _get_default_runtime_image_name(self): python_version = version_str(sys.version_info) return openwhisk_config.RUNTIME_DEFAULT[python_version] def _delete_function_handler_zip(self): os.remove(openwhisk_config.FH_ZIP_LOCATION) def build_runtime(self, docker_image_name, dockerfile): """ Builds a new runtime from a Docker file and pushes it to the Docker hub """ logger.info('Building a new docker image from Dockerfile') logger.info('Docker image name: {}'.format(docker_image_name)) if dockerfile: cmd = 'docker build -t {} -f {} .'.format(docker_image_name, dockerfile) else: cmd = 'docker build -t {} .'.format(docker_image_name) res = os.system(cmd) if res != 0: exit() cmd = 'docker push {}'.format(docker_image_name) res = os.system(cmd) if res != 0: exit() def create_runtime(self, docker_image_name, memory, timeout): """ Creates a new runtime into IBM CF namespace from an already built Docker image """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() runtime_meta = self._generate_runtime_meta(docker_image_name) logger.info('Creating new Lithops runtime based on Docker image {}'.format(docker_image_name)) self.cf_client.create_package(self.package) action_name = self._format_action_name(docker_image_name, memory) entry_point = os.path.join(os.path.dirname(__file__), 'entry_point.py') create_function_handler_zip(openwhisk_config.FH_ZIP_LOCATION, entry_point, '__main__.py') with open(openwhisk_config.FH_ZIP_LOCATION, "rb") as action_zip: action_bin = action_zip.read() self.cf_client.create_action(self.package, action_name, docker_image_name, code=action_bin, memory=memory, is_binary=True, timeout=timeout*1000) self._delete_function_handler_zip() return runtime_meta def delete_runtime(self, docker_image_name, memory): """ Deletes a runtime """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() action_name = self._format_action_name(docker_image_name, memory) self.cf_client.delete_action(self.package, action_name) def clean(self): """ Deletes all runtimes from all packages """ packages = self.cf_client.list_packages() for pkg in packages: if pkg['name'].startswith('lithops') and pkg['name'].endswith(self.user_key): actions = self.cf_client.list_actions(pkg['name']) while actions: for action in actions: self.cf_client.delete_action(pkg['name'], action['name']) actions = self.cf_client.list_actions(pkg['name']) self.cf_client.delete_package(pkg['name']) def list_runtimes(self, docker_image_name='all'): """ List all the runtimes deployed in the IBM CF service return: list of tuples (docker_image_name, memory) """ if docker_image_name == 'default': docker_image_name = self._get_default_runtime_image_name() runtimes = [] actions = self.cf_client.list_actions(self.package) for action in actions: action_image_name, memory = self._unformat_action_name(action['name']) if docker_image_name == action_image_name or docker_image_name == 'all': runtimes.append((action_image_name, memory)) return runtimes def invoke(self, docker_image_name, runtime_memory, payload): """ Invoke -- return information about this invocation """ action_name = self._format_action_name(docker_image_name, runtime_memory) activation_id = self.cf_client.invoke(self.package, action_name, payload, self.is_lithops_worker) return activation_id def get_runtime_key(self, docker_image_name, runtime_memory): """ Method that creates and returns the runtime key. Runtime keys are used to uniquely identify runtimes within the storage, in order to know which runtimes are installed and which not. """ action_name = self._format_action_name(docker_image_name, runtime_memory) runtime_key = os.path.join(self.name, self.namespace, action_name) return runtime_key def _generate_runtime_meta(self, docker_image_name): """ Extract installed Python modules from docker image """ action_code = """ import sys import pkgutil def main(args): print("Extracting preinstalled Python modules...") runtime_meta = dict() mods = list(pkgutil.iter_modules()) runtime_meta["preinstalls"] = [entry for entry in sorted([[mod, is_pkg] for _, mod, is_pkg in mods])] python_version = sys.version_info runtime_meta["python_ver"] = str(python_version[0])+"."+str(python_version[1]) print("Done!") return runtime_meta """ runtime_memory = 128 # old_stdout = sys.stdout # sys.stdout = open(os.devnull, 'w') action_name = self._format_action_name(docker_image_name, runtime_memory) self.cf_client.create_package(self.package) self.cf_client.create_action(self.package, action_name, docker_image_name, is_binary=False, code=textwrap.dedent(action_code), memory=runtime_memory, timeout=30000) # sys.stdout = old_stdout logger.debug("Extracting Python modules list from: {}".format(docker_image_name)) try: retry_invoke = True while retry_invoke: retry_invoke = False runtime_meta = self.cf_client.invoke_with_result(self.package, action_name) if 'activationId' in runtime_meta: retry_invoke = True except Exception: raise("Unable to invoke 'modules' action") try: self.delete_runtime(docker_image_name, runtime_memory) except Exception: raise Exception("Unable to delete 'modules' action") if not runtime_meta or 'preinstalls' not in runtime_meta: raise Exception(runtime_meta) return runtime_meta