def __init__(self, ibm_cos_config): logger.debug("Creating IBM COS client") self.ibm_cos_config = ibm_cos_config self.is_pywren_function = is_pywren_function() service_endpoint = ibm_cos_config.get('endpoint').replace( 'http:', 'https:') if self.is_pywren_function and 'private_endpoint' in ibm_cos_config: service_endpoint = ibm_cos_config.get('private_endpoint') if 'api_key' in ibm_cos_config: service_endpoint = service_endpoint.replace('http:', 'https:') logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint)) api_key = None if 'api_key' in ibm_cos_config: api_key = ibm_cos_config.get('api_key') api_key_type = 'COS' elif 'iam_api_key' in ibm_cos_config: api_key = ibm_cos_config.get('iam_api_key') api_key_type = 'IAM' if {'secret_key', 'access_key'} <= set(ibm_cos_config): logger.debug("Using access_key and secret_key") access_key = ibm_cos_config.get('access_key') secret_key = ibm_cos_config.get('secret_key') client_config = ibm_botocore.client.Config( max_pool_connections=128, user_agent_extra='pywren-ibm-cloud', connect_timeout=1) self.cos_client = ibm_boto3.client( 's3', aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=client_config, endpoint_url=service_endpoint) elif api_key is not None: client_config = ibm_botocore.client.Config( signature_version='oauth', max_pool_connections=128, user_agent_extra='pywren-ibm-cloud', connect_timeout=1) token_manager = DefaultTokenManager(api_key_id=api_key) token_filename = os.path.join(CACHE_DIR, api_key_type + '_TOKEN') if 'token' in self.ibm_cos_config: logger.debug("Using IBM {} API Key - Reusing Token".format( api_key_type)) token_manager._token = self.ibm_cos_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cos_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') elif os.path.exists(token_filename): logger.debug( "Using IBM {} API Key - Reusing Token from local cache". format(api_key_type)) token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') if token_manager._is_expired() and not is_pywren_function(): logger.debug( "Using IBM {} API Key - Token expired. Requesting new token" .format(api_key_type)) token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) self.ibm_cos_config['token'] = token_manager._token self.ibm_cos_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') self.cos_client = ibm_boto3.client('s3', token_manager=token_manager, config=client_config, endpoint_url=service_endpoint) logger.debug("IBM COS client created successfully")
def __init__(self, ibm_cf_config): logger.debug("Creating IBM Cloud Functions client") self.log_level = os.getenv('CLOUDBUTTON_LOGLEVEL') self.name = 'ibm_cf' self.ibm_cf_config = ibm_cf_config self.is_cloudbutton_function = is_cloudbutton_function() self.user_agent = ibm_cf_config['user_agent'] self.region = ibm_cf_config['region'] self.endpoint = ibm_cf_config['regions'][self.region]['endpoint'] self.namespace = ibm_cf_config['regions'][self.region]['namespace'] self.namespace_id = ibm_cf_config['regions'][self.region].get( 'namespace_id', None) self.api_key = ibm_cf_config['regions'][self.region].get( 'api_key', None) self.iam_api_key = ibm_cf_config.get('iam_api_key', None) logger.info("Set IBM CF Namespace to {}".format(self.namespace)) logger.info("Set IBM CF Endpoint to {}".format(self.endpoint)) self.user_key = self.api_key[: 5] if self.api_key else self.iam_api_key[: 5] self.package = 'cloudbutton_v{}_{}'.format(__version__, self.user_key) if self.api_key: enc_api_key = str.encode(self.api_key) auth_token = base64.encodebytes(enc_api_key).replace(b'\n', b'') auth = 'Basic %s' % auth_token.decode('UTF-8') self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace, auth=auth, user_agent=self.user_agent) elif self.iam_api_key: token_manager = DefaultTokenManager(api_key_id=self.iam_api_key) token_filename = os.path.join(CACHE_DIR, 'ibm_cf', 'iam_token') if 'token' in self.ibm_cf_config: logger.debug( "Using IBM IAM API Key - Reusing Token from config") token_manager._token = self.ibm_cf_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cf_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int( (token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format( token_manager._expiry_time, token_minutes_diff)) elif os.path.exists(token_filename): logger.debug( "Using IBM IAM API Key - Reusing Token from local cache") token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int( (token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format( token_manager._expiry_time, token_minutes_diff)) if (token_manager._is_expired() or token_minutes_diff < 11) and not is_cloudbutton_function(): logger.debug( "Using IBM IAM API Key - Token expired. Requesting new token" ) token_manager._token = None token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) ibm_cf_config['token'] = token_manager._token ibm_cf_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') auth_token = token_manager._token auth = 'Bearer ' + auth_token self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace_id, auth=auth, user_agent=self.user_agent) log_msg = ( 'Cloudbutton v{} init for IBM Cloud Functions - Namespace: {} - ' 'Region: {}'.format(__version__, self.namespace, self.region)) if not self.log_level: print(log_msg) logger.info("IBM CF client created successfully")
def __init__(self, ibm_cos_config, **kwargs): logger.debug("Creating IBM COS client") self.ibm_cos_config = ibm_cos_config self.is_lithops_function = is_lithops_function() user_agent = ibm_cos_config['user_agent'] service_endpoint = ibm_cos_config.get('endpoint').replace('http:', 'https:') if self.is_lithops_function and 'private_endpoint' in ibm_cos_config: service_endpoint = ibm_cos_config.get('private_endpoint') if 'api_key' in ibm_cos_config: service_endpoint = service_endpoint.replace('http:', 'https:') logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint)) api_key = None if 'api_key' in ibm_cos_config: api_key = ibm_cos_config.get('api_key') api_key_type = 'COS' elif 'iam_api_key' in ibm_cos_config: api_key = ibm_cos_config.get('iam_api_key') api_key_type = 'IAM' if {'secret_key', 'access_key'} <= set(ibm_cos_config): logger.debug("Using access_key and secret_key") access_key = ibm_cos_config.get('access_key') secret_key = ibm_cos_config.get('secret_key') client_config = ibm_botocore.client.Config(max_pool_connections=128, user_agent_extra=user_agent, connect_timeout=CONN_READ_TIMEOUT, read_timeout=CONN_READ_TIMEOUT, retries={'max_attempts': OBJ_REQ_RETRIES}) self.cos_client = ibm_boto3.client('s3', aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=client_config, endpoint_url=service_endpoint) elif api_key is not None: client_config = ibm_botocore.client.Config(signature_version='oauth', max_pool_connections=128, user_agent_extra=user_agent, connect_timeout=CONN_READ_TIMEOUT, read_timeout=CONN_READ_TIMEOUT, retries={'max_attempts': OBJ_REQ_RETRIES}) token_manager = DefaultTokenManager(api_key_id=api_key) token_filename = os.path.join(CACHE_DIR, 'ibm_cos', api_key_type.lower()+'_token') token_minutes_diff = 0 if 'token' in self.ibm_cos_config: logger.debug("Using IBM {} API Key - Reusing Token from config".format(api_key_type)) token_manager._token = self.ibm_cos_config['token'] token_manager._expiry_time = datetime.strptime(self.ibm_cos_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int((token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format(token_manager._expiry_time, token_minutes_diff)) elif os.path.exists(token_filename): token_data = load_yaml_config(token_filename) logger.debug("Using IBM {} API Key - Reusing Token from local cache".format(api_key_type)) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime(token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int((token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format(token_manager._expiry_time, token_minutes_diff)) if (token_manager._is_expired() or token_minutes_diff < 11) and not is_lithops_function(): logger.debug("Using IBM {} API Key - Token expired. Requesting new token".format(api_key_type)) token_manager._token = None token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) if token_manager._token: self.ibm_cos_config['token'] = token_manager._token if token_manager._expiry_time: self.ibm_cos_config['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z') self.cos_client = ibm_boto3.client('s3', token_manager=token_manager, config=client_config, endpoint_url=service_endpoint) logger.debug("IBM COS client created successfully")
class IBMTokenManager: def __init__(self, api_key, api_key_type='IAM', token=None, token_expiry_time=None): self.api_key = api_key self.api_key_type = api_key_type self._token_manager = DefaultTokenManager(api_key_id=self.api_key) self._token_filename = os.path.join(CACHE_DIR, 'ibm_{}'.format(api_key_type.lower()), 'token') if token: logger.debug("Using IBM {} API Key - Reusing Token from config".format(self.api_key_type)) self._token_manager._token = token self._token_manager._expiry_time = datetime.strptime(token_expiry_time, '%Y-%m-%d %H:%M:%S.%f%z') logger.debug("Token expiry time: {} - Minutes left: {}" .format(self._token_manager._expiry_time, self._get_token_minutes_diff())) elif os.path.exists(self._token_filename): logger.debug("Using IBM {} API Key - Reusing Token from local cache".format(self.api_key_type)) token_data = load_yaml_config(self._token_filename) self._token_manager._token = token_data['token'] self._token_manager._expiry_time = datetime.strptime(token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') logger.debug("Token expiry time: {} - Minutes left: {}". format(self._token_manager._expiry_time, self._get_token_minutes_diff())) def _is_token_expired(self): """ Checks if a token already expired """ return self._get_token_minutes_diff() < 1 def _get_token_minutes_diff(self): """ Gets the remaining minutes in which the current token is valid """ expiry_time = self._token_manager._expiry_time return max(0, int((expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0)) def _generate_new_token(self): self._token_manager._token = None self._token_manager.get_token() token_data = {} token_data['token'] = self._token_manager._token token_data['token_expiry_time'] = self._token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(self._token_filename, token_data) logger.debug("Token expiry time: {} - Minutes left: {}". format(self._token_manager._expiry_time, self._get_token_minutes_diff())) def get_token(self): """ Gets a new token within a mutex block to prevent multiple threads requesting new tokens at the same time. """ if (self._token_manager._is_expired() or self._is_token_expired()) \ and not is_lithops_worker(): logger.debug("Token expired. Requesting new token".format(self.api_key_type)) self._generate_new_token() token = self._token_manager._token token_expiry_time = self._token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z') return token, token_expiry_time
def __init__(self, cos_config): self.is_cf_cluster = is_cf_cluster() iam_config = cos_config['ibm_iam'] service_endpoint = cos_config.get('endpoint').replace( 'http:', 'https:') if self.is_cf_cluster and 'private_endpoint' in cos_config: service_endpoint = cos_config.get('private_endpoint') if 'api_key' in cos_config: service_endpoint = service_endpoint.replace('http:', 'https:') ibm_auth_endpoint = iam_config['ibm_auth_endpoint'] logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint)) logger.debug( "Set IBM IAM Auth Endpoint to {}".format(ibm_auth_endpoint)) api_key = None if 'api_key' in iam_config: api_key = iam_config.get('api_key') elif 'api_key' in cos_config: api_key = cos_config.get('api_key') if api_key is not None: client_config = ibm_botocore.client.Config( signature_version='oauth', max_pool_connections=128, user_agent_extra='pywren-ibm-cloud', connect_timeout=1) token_manager = DefaultTokenManager( api_key_id=api_key, auth_endpoint=ibm_auth_endpoint) if 'token' not in cos_config: logger.debug("IBM COS: Using api_key - Requesting new token") cos_config['token'] = token_manager.get_token() cos_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') else: logger.debug("IBM COS: Using api_key - Using token") token_manager._token = cos_config['token'] expiry_time = cos_config['token_expiry_time'] token_manager._expiry_time = datetime.strptime( expiry_time, '%Y-%m-%d %H:%M:%S.%f%z') if token_manager._is_expired() and not self.is_cf_cluster: # Only request new token on client machine logger.debug( "IBM COS: Using api_key - Token expired, requesting new token" ) cos_config['token'] = token_manager.get_token() cos_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') self.cos_client = ibm_boto3.client('s3', token_manager=token_manager, config=client_config, endpoint_url=service_endpoint) elif {'secret_key', 'access_key'} <= set(cos_config): logger.debug("IBM COS using access_key and secret_key") access_key = cos_config.get('access_key') secret_key = cos_config.get('secret_key') client_config = ibm_botocore.client.Config( max_pool_connections=128, user_agent_extra='pywren-ibm-cloud', connect_timeout=1) self.cos_client = ibm_boto3.client( 's3', aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=client_config, endpoint_url=service_endpoint)