def get_runtime_meta(self, key): """ Get the metadata given a runtime name. :param runtime: name of the runtime :return: runtime metadata """ path = ['runtimes', __version__, key + ".meta.json"] filename_local_path = os.path.join(CACHE_DIR, *path) if os.path.exists(filename_local_path) and not is_remote_cluster(): logger.debug("Runtime metadata found in local cache") with open(filename_local_path, "r") as f: runtime_meta = json.loads(f.read()) return runtime_meta else: logger.debug( "Runtime metadata not found in local cache. Retrieving it from storage" ) try: obj_key = '/'.join(path).replace('\\', '/') json_str = self.storage_handler.get_object( self.bucket, obj_key) runtime_meta = json.loads(json_str.decode("ascii")) # Save runtime meta to cache if not os.path.exists(os.path.dirname(filename_local_path)): os.makedirs(os.path.dirname(filename_local_path)) with open(filename_local_path, "w") as f: f.write(json.dumps(runtime_meta)) return runtime_meta except StorageNoSuchKeyError: raise Exception( 'The runtime {} is not installed.'.format(obj_key))
def put_runtime_meta(self, key, runtime_meta): """ Puit the metadata given a runtime config. :param runtime: name of the runtime :param runtime_meta metadata """ path = ['runtimes', __version__, key + ".meta.json"] obj_key = '/'.join(path).replace('\\', '/') logger.debug("Uploading runtime metadata to: /{}/{}".format( self.bucket, obj_key)) self.storage_handler.put_object(self.bucket, obj_key, json.dumps(runtime_meta)) if not is_remote_cluster(): filename_local_path = os.path.join(CACHE_DIR, *path) logger.debug("Saving runtime metadata into local cache: {}".format( filename_local_path)) if not os.path.exists(os.path.dirname(filename_local_path)): os.makedirs(os.path.dirname(filename_local_path)) with open(filename_local_path, "w") as f: f.write(json.dumps(runtime_meta))
def __init__(self, config=None, runtime=None, runtime_memory=None, compute_backend=None, compute_backend_region=None, storage_backend=None, storage_backend_region=None, rabbitmq_monitor=None, log_level=None): """ Initialize a FunctionExecutor class. :param config: Settings passed in here will override those in config file. Default None. :param runtime: Runtime name to use. Default None. :param runtime_memory: memory to use in the runtime. Default None. :param compute_backend: Name of the compute backend to use. Default None. :param compute_backend_region: Name of the compute backend region to use. Default None. :param storage_backend: Name of the storage backend to use. Default None. :param storage_backend_region: Name of the storage backend region to use. Default None. :param log_level: log level to use during the execution. Default None. :param rabbitmq_monitor: use rabbitmq as the monitoring system. Default None. :return `FunctionExecutor` object. """ self.start_time = time.time() self._state = FunctionExecutor.State.New self.is_remote_cluster = is_remote_cluster() # Log level Configuration self.log_level = log_level if not self.log_level: if (logger.getEffectiveLevel() != logging.WARNING): self.log_level = logging.getLevelName( logger.getEffectiveLevel()) if self.log_level: os.environ["PYWREN_LOGLEVEL"] = self.log_level if not self.is_remote_cluster: default_logging_config(self.log_level) # Overwrite pywren config parameters config_ow = {'pywren': {}} if runtime is not None: config_ow['pywren']['runtime'] = runtime if runtime_memory is not None: config_ow['pywren']['runtime_memory'] = int(runtime_memory) if compute_backend is not None: config_ow['pywren']['compute_backend'] = compute_backend if compute_backend_region is not None: config_ow['pywren'][ 'compute_backend_region'] = compute_backend_region if storage_backend is not None: config_ow['pywren']['storage_backend'] = storage_backend if storage_backend_region is not None: config_ow['pywren'][ 'storage_backend_region'] = storage_backend_region if rabbitmq_monitor is not None: config_ow['pywren']['rabbitmq_monitor'] = rabbitmq_monitor self.config = default_config(config, config_ow) self.executor_id = create_executor_id() logger.debug('FunctionExecutor created with ID: {}'.format( self.executor_id)) # RabbitMQ monitor configuration self.rabbitmq_monitor = self.config['pywren'].get( 'rabbitmq_monitor', False) if self.rabbitmq_monitor: if 'rabbitmq' in self.config and 'amqp_url' in self.config[ 'rabbitmq']: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') else: raise Exception( "You cannot use rabbitmq_mnonitor since 'amqp_url'" " is not present in configuration") self.data_cleaner = self.config['pywren']['data_cleaner'] storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.invoker = FunctionInvoker(self.config, self.executor_id, self.internal_storage) self.jobs = {}
def __init__(self, ibm_cf_config): logger.debug("Creating IBM Cloud Functions client") self.log_level = os.getenv('PYWREN_LOGLEVEL') self.name = 'ibm_cf' self.ibm_cf_config = ibm_cf_config self.package = 'pywren_v' + __version__ self.is_remote_cluster = is_remote_cluster() self.user_agent = ibm_cf_config['user_agent'] self.region = ibm_cf_config['region'] self.endpoint = ibm_cf_config['regions'][self.region]['endpoint'] self.namespace = ibm_cf_config['regions'][self.region]['namespace'] self.namespace_id = ibm_cf_config['regions'][self.region].get( 'namespace_id', None) self.api_key = ibm_cf_config['regions'][self.region].get( 'api_key', None) self.iam_api_key = ibm_cf_config.get('iam_api_key', None) logger.debug("Set IBM CF Namespace to {}".format(self.namespace)) logger.debug("Set IBM CF Endpoint to {}".format(self.endpoint)) if self.api_key: self.cf_client = CloudFunctionsClient(region=self.region, endpoint=self.endpoint, namespace=self.namespace, api_key=self.api_key, user_agent=self.user_agent) elif self.iam_api_key: token_manager = DefaultTokenManager(api_key_id=self.iam_api_key) token_filename = os.path.join(CACHE_DIR, 'IAM_TOKEN') if 'token' in self.ibm_cf_config: logger.debug("Using IBM IAM API Key - Reusing Token") token_manager._token = self.ibm_cf_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cf_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') elif os.path.exists(token_filename): logger.debug( "Using IBM IAM API Key - Reusing Token from local cache") token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') if token_manager._is_expired() and not is_remote_cluster(): logger.debug( "Using IBM IAM API Key - Token expired. Requesting new token" ) token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) ibm_cf_config['token'] = token_manager._token ibm_cf_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') self.cf_client = CloudFunctionsClient( region=self.region, endpoint=self.endpoint, namespace=self.namespace, namespace_id=self.namespace_id, token_manager=token_manager, user_agent=self.user_agent) log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - ' 'Region: {}'.format(__version__, self.namespace, self.region)) if not self.log_level: print(log_msg) logger.debug("IBM CF client created successfully")
def __init__(self, ibm_cos_config): logger.debug("Creating IBM COS client") self.ibm_cos_config = ibm_cos_config self.is_remote_cluster = is_remote_cluster() service_endpoint = ibm_cos_config.get('endpoint').replace( 'http:', 'https:') if self.is_remote_cluster and 'private_endpoint' in ibm_cos_config: service_endpoint = ibm_cos_config.get('private_endpoint') if 'api_key' in ibm_cos_config: service_endpoint = service_endpoint.replace('http:', 'https:') logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint)) api_key = None if 'api_key' in ibm_cos_config: api_key = ibm_cos_config.get('api_key') api_key_type = 'COS' elif 'iam_api_key' in ibm_cos_config: api_key = ibm_cos_config.get('iam_api_key') api_key_type = 'IAM' if {'secret_key', 'access_key'} <= set(ibm_cos_config): logger.debug("Using access_key and secret_key") access_key = ibm_cos_config.get('access_key') secret_key = ibm_cos_config.get('secret_key') client_config = ibm_botocore.client.Config( max_pool_connections=128, user_agent_extra='pywren-ibm-cloud', connect_timeout=1) self.cos_client = ibm_boto3.client( 's3', aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=client_config, endpoint_url=service_endpoint) elif api_key is not None: client_config = ibm_botocore.client.Config( signature_version='oauth', max_pool_connections=128, user_agent_extra='pywren-ibm-cloud', connect_timeout=1) token_manager = DefaultTokenManager(api_key_id=api_key) token_filename = os.path.join(CACHE_DIR, api_key_type + '_TOKEN') if 'token' in self.ibm_cos_config: logger.debug("Using IBM {} API Key - Reusing Token".format( api_key_type)) token_manager._token = self.ibm_cos_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cos_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') elif os.path.exists(token_filename): logger.debug( "Using IBM {} API Key - Reusing Token from local cache". format(api_key_type)) token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') if token_manager._is_expired() and not is_remote_cluster(): logger.debug( "Using IBM {} API Key - Token expired. Requesting new token" .format(api_key_type)) token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) self.ibm_cos_config['token'] = token_manager._token self.ibm_cos_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') self.cos_client = ibm_boto3.client('s3', token_manager=token_manager, config=client_config, endpoint_url=service_endpoint) logger.debug("IBM COS client created successfully")