def get_runtime_meta(self, key):
"""
Get the metadata given a runtime name.
:param runtime: name of the runtime
:return: runtime metadata
"""
path = ['runtimes', __version__, key + ".meta.json"]
filename_local_path = os.path.join(CACHE_DIR, *path)
if os.path.exists(filename_local_path) and not is_remote_cluster():
logger.debug("Runtime metadata found in local cache")
with open(filename_local_path, "r") as f:
runtime_meta = json.loads(f.read())
return runtime_meta
else:
logger.debug(
"Runtime metadata not found in local cache. Retrieving it from storage"
)
try:
obj_key = '/'.join(path).replace('\\', '/')
json_str = self.storage_handler.get_object(
self.bucket, obj_key)
runtime_meta = json.loads(json_str.decode("ascii"))
# Save runtime meta to cache
if not os.path.exists(os.path.dirname(filename_local_path)):
os.makedirs(os.path.dirname(filename_local_path))
with open(filename_local_path, "w") as f:
f.write(json.dumps(runtime_meta))
return runtime_meta
except StorageNoSuchKeyError:
raise Exception(
'The runtime {} is not installed.'.format(obj_key))
def put_runtime_meta(self, key, runtime_meta):
"""
Puit the metadata given a runtime config.
:param runtime: name of the runtime
:param runtime_meta metadata
"""
path = ['runtimes', __version__, key + ".meta.json"]
obj_key = '/'.join(path).replace('\\', '/')
logger.debug("Uploading runtime metadata to: /{}/{}".format(
self.bucket, obj_key))
self.storage_handler.put_object(self.bucket, obj_key,
json.dumps(runtime_meta))
if not is_remote_cluster():
filename_local_path = os.path.join(CACHE_DIR, *path)
logger.debug("Saving runtime metadata into local cache: {}".format(
filename_local_path))
if not os.path.exists(os.path.dirname(filename_local_path)):
os.makedirs(os.path.dirname(filename_local_path))
with open(filename_local_path, "w") as f:
f.write(json.dumps(runtime_meta))
def __init__(self,
config=None,
runtime=None,
runtime_memory=None,
compute_backend=None,
compute_backend_region=None,
storage_backend=None,
storage_backend_region=None,
rabbitmq_monitor=None,
log_level=None):
"""
Initialize a FunctionExecutor class.
:param config: Settings passed in here will override those in config file. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime. Default None.
:param compute_backend: Name of the compute backend to use. Default None.
:param compute_backend_region: Name of the compute backend region to use. Default None.
:param storage_backend: Name of the storage backend to use. Default None.
:param storage_backend_region: Name of the storage backend region to use. Default None.
:param log_level: log level to use during the execution. Default None.
:param rabbitmq_monitor: use rabbitmq as the monitoring system. Default None.
:return `FunctionExecutor` object.
"""
self.start_time = time.time()
self._state = FunctionExecutor.State.New
self.is_remote_cluster = is_remote_cluster()
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if (logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(
logger.getEffectiveLevel())
if self.log_level:
os.environ["PYWREN_LOGLEVEL"] = self.log_level
if not self.is_remote_cluster:
default_logging_config(self.log_level)
# Overwrite pywren config parameters
config_ow = {'pywren': {}}
if runtime is not None:
config_ow['pywren']['runtime'] = runtime
if runtime_memory is not None:
config_ow['pywren']['runtime_memory'] = int(runtime_memory)
if compute_backend is not None:
config_ow['pywren']['compute_backend'] = compute_backend
if compute_backend_region is not None:
config_ow['pywren'][
'compute_backend_region'] = compute_backend_region
if storage_backend is not None:
config_ow['pywren']['storage_backend'] = storage_backend
if storage_backend_region is not None:
config_ow['pywren'][
'storage_backend_region'] = storage_backend_region
if rabbitmq_monitor is not None:
config_ow['pywren']['rabbitmq_monitor'] = rabbitmq_monitor
self.config = default_config(config, config_ow)
self.executor_id = create_executor_id()
logger.debug('FunctionExecutor created with ID: {}'.format(
self.executor_id))
# RabbitMQ monitor configuration
self.rabbitmq_monitor = self.config['pywren'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config[
'rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception(
"You cannot use rabbitmq_mnonitor since 'amqp_url'"
" is not present in configuration")
self.data_cleaner = self.config['pywren']['data_cleaner']
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.invoker = FunctionInvoker(self.config, self.executor_id,
self.internal_storage)
self.jobs = {}
def __init__(self, ibm_cf_config):
logger.debug("Creating IBM Cloud Functions client")
self.log_level = os.getenv('PYWREN_LOGLEVEL')
self.name = 'ibm_cf'
self.ibm_cf_config = ibm_cf_config
self.package = 'pywren_v' + __version__
self.is_remote_cluster = is_remote_cluster()
self.user_agent = ibm_cf_config['user_agent']
self.region = ibm_cf_config['region']
self.endpoint = ibm_cf_config['regions'][self.region]['endpoint']
self.namespace = ibm_cf_config['regions'][self.region]['namespace']
self.namespace_id = ibm_cf_config['regions'][self.region].get(
'namespace_id', None)
self.api_key = ibm_cf_config['regions'][self.region].get(
'api_key', None)
self.iam_api_key = ibm_cf_config.get('iam_api_key', None)
logger.debug("Set IBM CF Namespace to {}".format(self.namespace))
logger.debug("Set IBM CF Endpoint to {}".format(self.endpoint))
if self.api_key:
self.cf_client = CloudFunctionsClient(region=self.region,
endpoint=self.endpoint,
namespace=self.namespace,
api_key=self.api_key,
user_agent=self.user_agent)
elif self.iam_api_key:
token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
token_filename = os.path.join(CACHE_DIR, 'IAM_TOKEN')
if 'token' in self.ibm_cf_config:
logger.debug("Using IBM IAM API Key - Reusing Token")
token_manager._token = self.ibm_cf_config['token']
token_manager._expiry_time = datetime.strptime(
self.ibm_cf_config['token_expiry_time'],
'%Y-%m-%d %H:%M:%S.%f%z')
elif os.path.exists(token_filename):
logger.debug(
"Using IBM IAM API Key - Reusing Token from local cache")
token_data = load_yaml_config(token_filename)
token_manager._token = token_data['token']
token_manager._expiry_time = datetime.strptime(
token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
if token_manager._is_expired() and not is_remote_cluster():
logger.debug(
"Using IBM IAM API Key - Token expired. Requesting new token"
)
token_manager.get_token()
token_data = {}
token_data['token'] = token_manager._token
token_data[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
dump_yaml_config(token_filename, token_data)
ibm_cf_config['token'] = token_manager._token
ibm_cf_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
self.cf_client = CloudFunctionsClient(
region=self.region,
endpoint=self.endpoint,
namespace=self.namespace,
namespace_id=self.namespace_id,
token_manager=token_manager,
user_agent=self.user_agent)
log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - '
'Region: {}'.format(__version__, self.namespace,
self.region))
if not self.log_level:
print(log_msg)
logger.debug("IBM CF client created successfully")
def __init__(self, ibm_cos_config):
logger.debug("Creating IBM COS client")
self.ibm_cos_config = ibm_cos_config
self.is_remote_cluster = is_remote_cluster()
service_endpoint = ibm_cos_config.get('endpoint').replace(
'http:', 'https:')
if self.is_remote_cluster and 'private_endpoint' in ibm_cos_config:
service_endpoint = ibm_cos_config.get('private_endpoint')
if 'api_key' in ibm_cos_config:
service_endpoint = service_endpoint.replace('http:', 'https:')
logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint))
api_key = None
if 'api_key' in ibm_cos_config:
api_key = ibm_cos_config.get('api_key')
api_key_type = 'COS'
elif 'iam_api_key' in ibm_cos_config:
api_key = ibm_cos_config.get('iam_api_key')
api_key_type = 'IAM'
if {'secret_key', 'access_key'} <= set(ibm_cos_config):
logger.debug("Using access_key and secret_key")
access_key = ibm_cos_config.get('access_key')
secret_key = ibm_cos_config.get('secret_key')
client_config = ibm_botocore.client.Config(
max_pool_connections=128,
user_agent_extra='pywren-ibm-cloud',
connect_timeout=1)
self.cos_client = ibm_boto3.client(
's3',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=client_config,
endpoint_url=service_endpoint)
elif api_key is not None:
client_config = ibm_botocore.client.Config(
signature_version='oauth',
max_pool_connections=128,
user_agent_extra='pywren-ibm-cloud',
connect_timeout=1)
token_manager = DefaultTokenManager(api_key_id=api_key)
token_filename = os.path.join(CACHE_DIR, api_key_type + '_TOKEN')
if 'token' in self.ibm_cos_config:
logger.debug("Using IBM {} API Key - Reusing Token".format(
api_key_type))
token_manager._token = self.ibm_cos_config['token']
token_manager._expiry_time = datetime.strptime(
self.ibm_cos_config['token_expiry_time'],
'%Y-%m-%d %H:%M:%S.%f%z')
elif os.path.exists(token_filename):
logger.debug(
"Using IBM {} API Key - Reusing Token from local cache".
format(api_key_type))
token_data = load_yaml_config(token_filename)
token_manager._token = token_data['token']
token_manager._expiry_time = datetime.strptime(
token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
if token_manager._is_expired() and not is_remote_cluster():
logger.debug(
"Using IBM {} API Key - Token expired. Requesting new token"
.format(api_key_type))
token_manager.get_token()
token_data = {}
token_data['token'] = token_manager._token
token_data[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
dump_yaml_config(token_filename, token_data)
self.ibm_cos_config['token'] = token_manager._token
self.ibm_cos_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
self.cos_client = ibm_boto3.client('s3',
token_manager=token_manager,
config=client_config,
endpoint_url=service_endpoint)
logger.debug("IBM COS client created successfully")
