def __init__(self, cos_config):
service_endpoint = cos_config.get('endpoint').replace('http:', 'https:')
if 'api_key' in cos_config:
client_config = ibm_botocore.client.Config(signature_version='oauth',
max_pool_connections=200,
user_agent_extra='pywren-ibm-cloud')
api_key = cos_config.get('api_key')
token_manager = DefaultTokenManager(api_key_id=api_key)
if 'token' in cos_config:
token_manager._token = cos_config['token']
expiry_time = cos_config['token_expiry_time']
token_manager._expiry_time = datetime.strptime(expiry_time, '%Y-%m-%d %H:%M:%S.%f%z')
self.cos_client = ibm_boto3.client('s3',
token_manager=token_manager,
config=client_config,
endpoint_url=service_endpoint)
if 'token' not in cos_config:
cos_config['token'] = token_manager.get_token()
cos_config['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z')
elif {'secret_key', 'access_key'} <= set(cos_config):
secret_key = cos_config.get('secret_key')
access_key = cos_config.get('access_key')
client_config = ibm_botocore.client.Config(max_pool_connections=200,
user_agent_extra='pywren-ibm-cloud')
self.cos_client = ibm_boto3.client('s3',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=client_config,
endpoint_url=service_endpoint)
def __init__(self, cos_config):
if 'cos_endpoint' in cos_config:
service_endpoint = cos_config.get('cos_endpoint').replace(
'http:', 'https:')
elif {'cos_endpoints', 'cos_region'} <= set(cos_config):
endpoints = requests.get(cos_config.get('cos_endpoints')).json()
region = cos_config.get('cos_region')
if region in endpoints['service-endpoints']['cross-region']['us'][
'public']:
cos_host = endpoints['service-endpoints']['cross-region'][
'us']['public'][region]
elif region in endpoints['service-endpoints']['cross-region'][
'eu']['public']:
cos_host = endpoints['service-endpoints']['cross-region'][
'eu']['public'][region]
elif region in endpoints['service-endpoints']['regional']:
cos_host = endpoints['service-endpoints']['regional'][region][
'public'][region]
elif region in endpoints['service-endpoints']['regional']:
cos_host = endpoints['service-endpoints']['regional'][region][
'public'][region]
service_endpoint = 'https://' + cos_host
if 'cos_api_key' in cos_config:
client_config = ibm_botocore.client.Config(
signature_version='oauth', max_pool_connections=200)
api_key = cos_config.get('cos_api_key')
token_manager = DefaultTokenManager(api_key_id=api_key)
if 'cos_token' in cos_config:
token_manager._token = cos_config.get('cos_token')
self.cos_client = ibm_boto3.client('s3',
token_manager=token_manager,
config=client_config,
endpoint_url=service_endpoint)
cos_config['cos_token'] = token_manager.get_token()
elif {'cos_secret_key', 'cos_access_key'} <= set(cos_config):
secret_key = cos_config.get('cos_secret_key')
access_key = cos_config.get('cos_access_key')
client_config = ibm_botocore.client.Config(
max_pool_connections=200)
self.cos_client = ibm_boto3.client(
's3',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=client_config,
endpoint_url=service_endpoint)
def __init__(self, ibm_cos_config, **kwargs):
logger.debug("Creating IBM COS client")
self.ibm_cos_config = ibm_cos_config
self.is_cloudbutton_function = is_cloudbutton_function()
user_agent = ibm_cos_config['user_agent']
service_endpoint = ibm_cos_config.get('endpoint').replace(
'http:', 'https:')
if self.is_cloudbutton_function and 'private_endpoint' in ibm_cos_config:
service_endpoint = ibm_cos_config.get('private_endpoint')
if 'api_key' in ibm_cos_config:
service_endpoint = service_endpoint.replace('http:', 'https:')
logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint))
api_key = None
if 'api_key' in ibm_cos_config:
api_key = ibm_cos_config.get('api_key')
api_key_type = 'COS'
elif 'iam_api_key' in ibm_cos_config:
api_key = ibm_cos_config.get('iam_api_key')
api_key_type = 'IAM'
if {'secret_key', 'access_key'} <= set(ibm_cos_config):
logger.debug("Using access_key and secret_key")
access_key = ibm_cos_config.get('access_key')
secret_key = ibm_cos_config.get('secret_key')
client_config = ibm_botocore.client.Config(
max_pool_connections=128,
user_agent_extra=user_agent,
connect_timeout=CONN_READ_TIMEOUT,
read_timeout=CONN_READ_TIMEOUT,
retries={'max_attempts': OBJ_REQ_RETRIES})
self.cos_client = ibm_boto3.client(
's3',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=client_config,
endpoint_url=service_endpoint)
elif api_key is not None:
client_config = ibm_botocore.client.Config(
signature_version='oauth',
max_pool_connections=128,
user_agent_extra=user_agent,
connect_timeout=CONN_READ_TIMEOUT,
read_timeout=CONN_READ_TIMEOUT,
retries={'max_attempts': OBJ_REQ_RETRIES})
token_manager = DefaultTokenManager(api_key_id=api_key)
token_filename = os.path.join(CACHE_DIR, 'ibm_cos',
api_key_type.lower() + '_token')
token_minutes_diff = 0
if 'token' in self.ibm_cos_config:
logger.debug(
"Using IBM {} API Key - Reusing Token from config".format(
api_key_type))
token_manager._token = self.ibm_cos_config['token']
token_manager._expiry_time = datetime.strptime(
self.ibm_cos_config['token_expiry_time'],
'%Y-%m-%d %H:%M:%S.%f%z')
token_minutes_diff = int(
(token_manager._expiry_time -
datetime.now(timezone.utc)).total_seconds() / 60.0)
logger.debug("Token expiry time: {} - Minutes left: {}".format(
token_manager._expiry_time, token_minutes_diff))
elif os.path.exists(token_filename):
token_data = load_yaml_config(token_filename)
logger.debug(
"Using IBM {} API Key - Reusing Token from local cache".
format(api_key_type))
token_manager._token = token_data['token']
token_manager._expiry_time = datetime.strptime(
token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
token_minutes_diff = int(
(token_manager._expiry_time -
datetime.now(timezone.utc)).total_seconds() / 60.0)
logger.debug("Token expiry time: {} - Minutes left: {}".format(
token_manager._expiry_time, token_minutes_diff))
if (token_manager._is_expired() or
token_minutes_diff < 11) and not is_cloudbutton_function():
logger.debug(
"Using IBM {} API Key - Token expired. Requesting new token"
.format(api_key_type))
token_manager._token = None
token_manager.get_token()
token_data = {}
token_data['token'] = token_manager._token
token_data[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
dump_yaml_config(token_filename, token_data)
if token_manager._token:
self.ibm_cos_config['token'] = token_manager._token
if token_manager._expiry_time:
self.ibm_cos_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
self.cos_client = ibm_boto3.client('s3',
token_manager=token_manager,
config=client_config,
endpoint_url=service_endpoint)
logger.debug("IBM COS client created successfully")
def __init__(self, ibm_cf_config):
logger.debug("Creating IBM Cloud Functions client")
self.log_active = logger.getEffectiveLevel() != logging.WARNING
self.name = 'ibm_cf'
self.ibm_cf_config = ibm_cf_config
self.is_lithops_function = is_lithops_function()
self.user_agent = ibm_cf_config['user_agent']
self.region = ibm_cf_config['region']
self.endpoint = ibm_cf_config['regions'][self.region]['endpoint']
self.namespace = ibm_cf_config['regions'][self.region]['namespace']
self.namespace_id = ibm_cf_config['regions'][self.region].get(
'namespace_id', None)
self.api_key = ibm_cf_config['regions'][self.region].get(
'api_key', None)
self.iam_api_key = ibm_cf_config.get('iam_api_key', None)
logger.info("Set IBM CF Namespace to {}".format(self.namespace))
logger.info("Set IBM CF Endpoint to {}".format(self.endpoint))
self.user_key = self.api_key[:
5] if self.api_key else self.iam_api_key[:
5]
self.package = 'lithops_v{}_{}'.format(__version__, self.user_key)
if self.api_key:
enc_api_key = str.encode(self.api_key)
auth_token = base64.encodebytes(enc_api_key).replace(b'\n', b'')
auth = 'Basic %s' % auth_token.decode('UTF-8')
self.cf_client = OpenWhiskClient(endpoint=self.endpoint,
namespace=self.namespace,
auth=auth,
user_agent=self.user_agent)
elif self.iam_api_key:
token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
token_filename = os.path.join(CACHE_DIR, 'ibm_cf', 'iam_token')
if 'token' in self.ibm_cf_config:
logger.debug(
"Using IBM IAM API Key - Reusing Token from config")
token_manager._token = self.ibm_cf_config['token']
token_manager._expiry_time = datetime.strptime(
self.ibm_cf_config['token_expiry_time'],
'%Y-%m-%d %H:%M:%S.%f%z')
token_minutes_diff = int(
(token_manager._expiry_time -
datetime.now(timezone.utc)).total_seconds() / 60.0)
logger.debug("Token expiry time: {} - Minutes left: {}".format(
token_manager._expiry_time, token_minutes_diff))
elif os.path.exists(token_filename):
logger.debug(
"Using IBM IAM API Key - Reusing Token from local cache")
token_data = load_yaml_config(token_filename)
token_manager._token = token_data['token']
token_manager._expiry_time = datetime.strptime(
token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
token_minutes_diff = int(
(token_manager._expiry_time -
datetime.now(timezone.utc)).total_seconds() / 60.0)
logger.debug("Token expiry time: {} - Minutes left: {}".format(
token_manager._expiry_time, token_minutes_diff))
if (token_manager._is_expired()
or token_minutes_diff < 11) and not is_lithops_function():
logger.debug(
"Using IBM IAM API Key - Token expired. Requesting new token"
)
token_manager._token = None
token_manager.get_token()
token_data = {}
token_data['token'] = token_manager._token
token_data[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
dump_yaml_config(token_filename, token_data)
ibm_cf_config['token'] = token_manager._token
ibm_cf_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
auth_token = token_manager._token
auth = 'Bearer ' + auth_token
self.cf_client = OpenWhiskClient(endpoint=self.endpoint,
namespace=self.namespace_id,
auth=auth,
user_agent=self.user_agent)
log_msg = (
'Lithops v{} init for IBM Cloud Functions - Namespace: {} - '
'Region: {}'.format(__version__, self.namespace, self.region))
if not self.log_active:
print(log_msg)
logger.info("IBM CF client created successfully")
def __init__(self, ibm_cf_config):
logger.debug("Creating IBM Cloud Functions client")
self.log_level = os.getenv('PYWREN_LOGLEVEL')
self.name = 'ibm_cf'
self.ibm_cf_config = ibm_cf_config
self.package = 'pywren_v' + __version__
self.is_remote_cluster = is_remote_cluster()
self.user_agent = ibm_cf_config['user_agent']
self.region = ibm_cf_config['region']
self.endpoint = ibm_cf_config['regions'][self.region]['endpoint']
self.namespace = ibm_cf_config['regions'][self.region]['namespace']
self.namespace_id = ibm_cf_config['regions'][self.region].get(
'namespace_id', None)
self.api_key = ibm_cf_config['regions'][self.region].get(
'api_key', None)
self.iam_api_key = ibm_cf_config.get('iam_api_key', None)
logger.debug("Set IBM CF Namespace to {}".format(self.namespace))
logger.debug("Set IBM CF Endpoint to {}".format(self.endpoint))
if self.api_key:
self.cf_client = CloudFunctionsClient(region=self.region,
endpoint=self.endpoint,
namespace=self.namespace,
api_key=self.api_key,
user_agent=self.user_agent)
elif self.iam_api_key:
token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
token_filename = os.path.join(CACHE_DIR, 'IAM_TOKEN')
if 'token' in self.ibm_cf_config:
logger.debug("Using IBM IAM API Key - Reusing Token")
token_manager._token = self.ibm_cf_config['token']
token_manager._expiry_time = datetime.strptime(
self.ibm_cf_config['token_expiry_time'],
'%Y-%m-%d %H:%M:%S.%f%z')
elif os.path.exists(token_filename):
logger.debug(
"Using IBM IAM API Key - Reusing Token from local cache")
token_data = load_yaml_config(token_filename)
token_manager._token = token_data['token']
token_manager._expiry_time = datetime.strptime(
token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
if token_manager._is_expired() and not is_remote_cluster():
logger.debug(
"Using IBM IAM API Key - Token expired. Requesting new token"
)
token_manager.get_token()
token_data = {}
token_data['token'] = token_manager._token
token_data[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
dump_yaml_config(token_filename, token_data)
ibm_cf_config['token'] = token_manager._token
ibm_cf_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
self.cf_client = CloudFunctionsClient(
region=self.region,
endpoint=self.endpoint,
namespace=self.namespace,
namespace_id=self.namespace_id,
token_manager=token_manager,
user_agent=self.user_agent)
log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - '
'Region: {}'.format(__version__, self.namespace,
self.region))
if not self.log_level:
print(log_msg)
logger.debug("IBM CF client created successfully")
def __init__(self, ibm_cos_config):
self.is_cf_cluster = is_cf_cluster()
self.ibm_cos_config = ibm_cos_config
iam_config = ibm_cos_config['ibm_iam']
service_endpoint = ibm_cos_config.get('endpoint').replace(
'http:', 'https:')
if self.is_cf_cluster and 'private_endpoint' in ibm_cos_config:
service_endpoint = ibm_cos_config.get('private_endpoint')
if 'api_key' in ibm_cos_config:
service_endpoint = service_endpoint.replace('http:', 'https:')
ibm_auth_endpoint = iam_config['ibm_auth_endpoint']
logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint))
logger.debug(
"Set IBM IAM Auth Endpoint to {}".format(ibm_auth_endpoint))
api_key = None
if 'api_key' in ibm_cos_config:
api_key = ibm_cos_config.get('api_key')
api_key_type = 'COS'
elif 'api_key' in iam_config:
api_key = iam_config.get('api_key')
api_key_type = 'IAM'
if {'secret_key', 'access_key'} <= set(ibm_cos_config):
logger.debug("IBM COS using access_key and secret_key")
access_key = ibm_cos_config.get('access_key')
secret_key = ibm_cos_config.get('secret_key')
client_config = ibm_botocore.client.Config(
max_pool_connections=128,
user_agent_extra='pywren-ibm-cloud',
connect_timeout=1)
self.cos_client = ibm_boto3.client(
's3',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=client_config,
endpoint_url=service_endpoint)
elif api_key is not None:
client_config = ibm_botocore.client.Config(
signature_version='oauth',
max_pool_connections=128,
user_agent_extra='pywren-ibm-cloud',
connect_timeout=1)
token_manager = DefaultTokenManager(
api_key_id=api_key, auth_endpoint=ibm_auth_endpoint)
if 'token' not in ibm_cos_config:
logger.debug(
"IBM COS: Using {} api_key - Requesting new token".format(
api_key_type))
ibm_cos_config['token'] = token_manager.get_token()
ibm_cos_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
else:
logger.debug(
"IBM COS: Using {} api_key - Reusing token".format(
api_key_type))
token_manager._token = ibm_cos_config['token']
expiry_time = ibm_cos_config['token_expiry_time']
token_manager._expiry_time = datetime.strptime(
expiry_time, '%Y-%m-%d %H:%M:%S.%f%z')
if token_manager._is_expired() and not self.is_cf_cluster:
# Only request new token on client machine
logger.debug(
"IBM COS: Using {} api_key - Token expired, requesting new token"
.format(api_key_type))
ibm_cos_config['token'] = token_manager.get_token()
ibm_cos_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
self.cos_client = ibm_boto3.client('s3',
token_manager=token_manager,
config=client_config,
endpoint_url=service_endpoint)
def __init__(self, ibm_cos_config):
logger.debug("Creating IBM COS client")
self.ibm_cos_config = ibm_cos_config
self.is_pywren_function = is_pywren_function()
service_endpoint = ibm_cos_config.get('endpoint').replace(
'http:', 'https:')
if self.is_pywren_function and 'private_endpoint' in ibm_cos_config:
service_endpoint = ibm_cos_config.get('private_endpoint')
if 'api_key' in ibm_cos_config:
service_endpoint = service_endpoint.replace('http:', 'https:')
logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint))
api_key = None
if 'api_key' in ibm_cos_config:
api_key = ibm_cos_config.get('api_key')
api_key_type = 'COS'
elif 'iam_api_key' in ibm_cos_config:
api_key = ibm_cos_config.get('iam_api_key')
api_key_type = 'IAM'
if {'secret_key', 'access_key'} <= set(ibm_cos_config):
logger.debug("Using access_key and secret_key")
access_key = ibm_cos_config.get('access_key')
secret_key = ibm_cos_config.get('secret_key')
client_config = ibm_botocore.client.Config(
max_pool_connections=128,
user_agent_extra='pywren-ibm-cloud',
connect_timeout=1)
self.cos_client = ibm_boto3.client(
's3',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=client_config,
endpoint_url=service_endpoint)
elif api_key is not None:
client_config = ibm_botocore.client.Config(
signature_version='oauth',
max_pool_connections=128,
user_agent_extra='pywren-ibm-cloud',
connect_timeout=1)
token_manager = DefaultTokenManager(api_key_id=api_key)
token_filename = os.path.join(CACHE_DIR, api_key_type + '_TOKEN')
if 'token' in self.ibm_cos_config:
logger.debug("Using IBM {} API Key - Reusing Token".format(
api_key_type))
token_manager._token = self.ibm_cos_config['token']
token_manager._expiry_time = datetime.strptime(
self.ibm_cos_config['token_expiry_time'],
'%Y-%m-%d %H:%M:%S.%f%z')
elif os.path.exists(token_filename):
logger.debug(
"Using IBM {} API Key - Reusing Token from local cache".
format(api_key_type))
token_data = load_yaml_config(token_filename)
token_manager._token = token_data['token']
token_manager._expiry_time = datetime.strptime(
token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
if token_manager._is_expired() and not is_pywren_function():
logger.debug(
"Using IBM {} API Key - Token expired. Requesting new token"
.format(api_key_type))
token_manager.get_token()
token_data = {}
token_data['token'] = token_manager._token
token_data[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
dump_yaml_config(token_filename, token_data)
self.ibm_cos_config['token'] = token_manager._token
self.ibm_cos_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
self.cos_client = ibm_boto3.client('s3',
token_manager=token_manager,
config=client_config,
endpoint_url=service_endpoint)
logger.debug("IBM COS client created successfully")