def __init__(self, ibm_cf_config):
self.log_level = os.getenv('CB_LOG_LEVEL')
self.name = 'ibm_cf'
self.ibm_cf_config = ibm_cf_config
self.package = 'pywren_v' + __version__
self.region = ibm_cf_config['region']
self.cf_client = CloudFunctionsClient(self.ibm_cf_config)
self.is_cf_cluster = is_cf_cluster()
self.namespace = ibm_cf_config[self.region]['namespace']
log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} '
'- Region: {}'.format(__version__, self.namespace,
self.region))
logger.info(log_msg)
if not self.log_level:
print(log_msg)
def __init__(self, ibm_cf_config):
logger.debug("Creating IBM Cloud Functions client")
self.log_level = os.getenv('PYWREN_LOGLEVEL')
self.name = 'ibm_cf'
self.ibm_cf_config = ibm_cf_config
self.package = 'pywren_v' + __version__
self.is_remote_cluster = is_remote_cluster()
self.user_agent = ibm_cf_config['user_agent']
self.region = ibm_cf_config['region']
self.endpoint = ibm_cf_config['regions'][self.region]['endpoint']
self.namespace = ibm_cf_config['regions'][self.region]['namespace']
self.namespace_id = ibm_cf_config['regions'][self.region].get(
'namespace_id', None)
self.api_key = ibm_cf_config['regions'][self.region].get(
'api_key', None)
self.iam_api_key = ibm_cf_config.get('iam_api_key', None)
logger.debug("Set IBM CF Namespace to {}".format(self.namespace))
logger.debug("Set IBM CF Endpoint to {}".format(self.endpoint))
if self.api_key:
self.cf_client = CloudFunctionsClient(region=self.region,
endpoint=self.endpoint,
namespace=self.namespace,
api_key=self.api_key,
user_agent=self.user_agent)
elif self.iam_api_key:
token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
token_filename = os.path.join(CACHE_DIR, 'IAM_TOKEN')
if 'token' in self.ibm_cf_config:
logger.debug("Using IBM IAM API Key - Reusing Token")
token_manager._token = self.ibm_cf_config['token']
token_manager._expiry_time = datetime.strptime(
self.ibm_cf_config['token_expiry_time'],
'%Y-%m-%d %H:%M:%S.%f%z')
elif os.path.exists(token_filename):
logger.debug(
"Using IBM IAM API Key - Reusing Token from local cache")
token_data = load_yaml_config(token_filename)
token_manager._token = token_data['token']
token_manager._expiry_time = datetime.strptime(
token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
if token_manager._is_expired() and not is_remote_cluster():
logger.debug(
"Using IBM IAM API Key - Token expired. Requesting new token"
)
token_manager.get_token()
token_data = {}
token_data['token'] = token_manager._token
token_data[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
dump_yaml_config(token_filename, token_data)
ibm_cf_config['token'] = token_manager._token
ibm_cf_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
self.cf_client = CloudFunctionsClient(
region=self.region,
endpoint=self.endpoint,
namespace=self.namespace,
namespace_id=self.namespace_id,
token_manager=token_manager,
user_agent=self.user_agent)
log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - '
'Region: {}'.format(__version__, self.namespace,
self.region))
if not self.log_level:
print(log_msg)
logger.debug("IBM CF client created successfully")
class IBMCloudFunctionsBackend:
"""
A wrap-up around IBM Cloud Functions backend.
"""
def __init__(self, ibm_cf_config):
logger.debug("Creating IBM Cloud Functions client")
self.log_level = os.getenv('PYWREN_LOGLEVEL')
self.name = 'ibm_cf'
self.ibm_cf_config = ibm_cf_config
self.package = 'pywren_v' + __version__
self.is_remote_cluster = is_remote_cluster()
self.user_agent = ibm_cf_config['user_agent']
self.region = ibm_cf_config['region']
self.endpoint = ibm_cf_config['regions'][self.region]['endpoint']
self.namespace = ibm_cf_config['regions'][self.region]['namespace']
self.namespace_id = ibm_cf_config['regions'][self.region].get(
'namespace_id', None)
self.api_key = ibm_cf_config['regions'][self.region].get(
'api_key', None)
self.iam_api_key = ibm_cf_config.get('iam_api_key', None)
logger.debug("Set IBM CF Namespace to {}".format(self.namespace))
logger.debug("Set IBM CF Endpoint to {}".format(self.endpoint))
if self.api_key:
self.cf_client = CloudFunctionsClient(region=self.region,
endpoint=self.endpoint,
namespace=self.namespace,
api_key=self.api_key,
user_agent=self.user_agent)
elif self.iam_api_key:
token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
token_filename = os.path.join(CACHE_DIR, 'IAM_TOKEN')
if 'token' in self.ibm_cf_config:
logger.debug("Using IBM IAM API Key - Reusing Token")
token_manager._token = self.ibm_cf_config['token']
token_manager._expiry_time = datetime.strptime(
self.ibm_cf_config['token_expiry_time'],
'%Y-%m-%d %H:%M:%S.%f%z')
elif os.path.exists(token_filename):
logger.debug(
"Using IBM IAM API Key - Reusing Token from local cache")
token_data = load_yaml_config(token_filename)
token_manager._token = token_data['token']
token_manager._expiry_time = datetime.strptime(
token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
if token_manager._is_expired() and not is_remote_cluster():
logger.debug(
"Using IBM IAM API Key - Token expired. Requesting new token"
)
token_manager.get_token()
token_data = {}
token_data['token'] = token_manager._token
token_data[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
dump_yaml_config(token_filename, token_data)
ibm_cf_config['token'] = token_manager._token
ibm_cf_config[
'token_expiry_time'] = token_manager._expiry_time.strftime(
'%Y-%m-%d %H:%M:%S.%f%z')
self.cf_client = CloudFunctionsClient(
region=self.region,
endpoint=self.endpoint,
namespace=self.namespace,
namespace_id=self.namespace_id,
token_manager=token_manager,
user_agent=self.user_agent)
log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - '
'Region: {}'.format(__version__, self.namespace,
self.region))
if not self.log_level:
print(log_msg)
logger.debug("IBM CF client created successfully")
def _format_action_name(self, runtime_name, runtime_memory):
runtime_name = runtime_name.replace('/', '_').replace(':', '_')
return '{}_{}MB'.format(runtime_name, runtime_memory)
def _unformat_action_name(self, action_name):
runtime_name, memory = action_name.rsplit('_', 1)
image_name = runtime_name.replace('_', '/', 1)
image_name = image_name.replace('_', ':', -1)
return image_name, int(memory.replace('MB', ''))
def _get_default_runtime_image_name(self):
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
image_name = ibmcf_config.RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
image_name = ibmcf_config.RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
image_name = ibmcf_config.RUNTIME_DEFAULT_37
return image_name
def _create_function_handler_zip(self):
logger.debug("Creating function handler zip in {}".format(
ibmcf_config.FH_ZIP_LOCATION))
def add_folder_to_zip(zip_file, full_dir_path, sub_dir=''):
for file in os.listdir(full_dir_path):
full_path = os.path.join(full_dir_path, file)
if os.path.isfile(full_path):
zip_file.write(
full_path,
os.path.join('pywren_ibm_cloud', sub_dir, file))
elif os.path.isdir(
full_path) and '__pycache__' not in full_path:
add_folder_to_zip(zip_file, full_path,
os.path.join(sub_dir, file))
try:
with zipfile.ZipFile(ibmcf_config.FH_ZIP_LOCATION, 'w',
zipfile.ZIP_DEFLATED) as ibmcf_pywren_zip:
current_location = os.path.dirname(os.path.abspath(__file__))
module_location = os.path.dirname(
os.path.abspath(pywren_ibm_cloud.__file__))
main_file = os.path.join(current_location, 'entry_point.py')
ibmcf_pywren_zip.write(main_file, '__main__.py')
add_folder_to_zip(ibmcf_pywren_zip, module_location)
except Exception as e:
raise Exception('Unable to create the {} package: {}'.format(
ibmcf_config.FH_ZIP_LOCATION, e))
def build_runtime(self, docker_image_name, dockerfile):
"""
Builds a new runtime from a Docker file and pushes it to the Docker hub
"""
logger.info('Creating a new docker image from Dockerfile')
logger.info('Docker image name: {}'.format(docker_image_name))
if dockerfile:
cmd = 'docker build -t {} -f {} .'.format(docker_image_name,
dockerfile)
else:
cmd = 'docker build -t {} .'.format(docker_image_name)
res = os.system(cmd)
if res != 0:
exit()
cmd = 'docker push {}'.format(docker_image_name)
res = os.system(cmd)
if res != 0:
exit()
def create_runtime(self,
docker_image_name,
memory,
timeout=ibmcf_config.RUNTIME_TIMEOUT_DEFAULT):
"""
Creates a new runtime into IBM CF namespace from an already built Docker image
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
runtime_meta = self._generate_runtime_meta(docker_image_name)
logger.info(
'Creating new PyWren runtime based on Docker image {}'.format(
docker_image_name))
self.cf_client.create_package(self.package)
action_name = self._format_action_name(docker_image_name, memory)
self._create_function_handler_zip()
with open(ibmcf_config.FH_ZIP_LOCATION, "rb") as action_zip:
action_bin = action_zip.read()
self.cf_client.create_action(self.package,
action_name,
docker_image_name,
code=action_bin,
memory=memory,
is_binary=True,
timeout=timeout)
return runtime_meta
def delete_runtime(self, docker_image_name, memory):
"""
Deletes a runtime
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
action_name = self._format_action_name(docker_image_name, memory)
self.cf_client.delete_action(self.package, action_name)
def delete_all_runtimes(self):
"""
Deletes all runtimes from all packages
"""
packages = self.cf_client.list_packages()
for pkg in packages:
if 'pywren_v' in pkg['name']:
actions = self.cf_client.list_actions(pkg['name'])
while actions:
for action in actions:
self.cf_client.delete_action(pkg['name'],
action['name'])
actions = self.cf_client.list_actions(pkg['name'])
self.cf_client.delete_package(pkg['name'])
def list_runtimes(self, docker_image_name='all'):
"""
List all the runtimes deployed in the IBM CF service
return: list of tuples (docker_image_name, memory)
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
runtimes = []
actions = self.cf_client.list_actions(self.package)
for action in actions:
action_image_name, memory = self._unformat_action_name(
action['name'])
if docker_image_name == action_image_name or docker_image_name == 'all':
runtimes.append((action_image_name, memory))
return runtimes
def invoke(self, docker_image_name, runtime_memory, payload):
"""
Invoke -- return information about this invocation
"""
exec_id = payload['executor_id']
job_id = payload['job_id']
call_id = payload['call_id']
action_name = self._format_action_name(docker_image_name,
runtime_memory)
start = time.time()
activation_id, exception = self.cf_client.invoke(
self.package, action_name, payload, self.is_remote_cluster)
roundtrip = time.time() - start
resp_time = format(round(roundtrip, 3), '.3f')
if activation_id is None:
log_msg = (
'ExecutorID {} | JobID {} - Function invocation {} failed: '
'{}'.format(exec_id, job_id, call_id, str(exception)))
logger.debug(log_msg)
else:
log_msg = (
'ExecutorID {} | JobID {} - Function invocation {} done! ({}s) - Activation'
' ID: {}'.format(exec_id, job_id, call_id, resp_time,
activation_id))
logger.debug(log_msg)
return activation_id
def invoke_with_result(self,
docker_image_name,
runtime_memory,
payload={}):
"""
Invoke waiting for a result -- return information about this invocation
"""
action_name = self._format_action_name(docker_image_name,
runtime_memory)
return self.cf_client.invoke_with_result(self.package, action_name,
payload)
def get_runtime_key(self, docker_image_name, runtime_memory):
"""
Method that creates and returns the runtime key.
Runtime keys are used to uniquely identify runtimes within the storage,
in order to know which runtimes are installed and which not.
"""
action_name = self._format_action_name(docker_image_name,
runtime_memory)
runtime_key = os.path.join(self.name, self.region, self.namespace,
action_name)
return runtime_key
def _generate_runtime_meta(self, docker_image_name):
"""
Extract installed Python modules from docker image
"""
action_code = """
import sys
import pkgutil
def main(args):
print("Extracting preinstalled Python modules...")
runtime_meta = dict()
mods = list(pkgutil.iter_modules())
runtime_meta["preinstalls"] = [entry for entry in sorted([[mod, is_pkg] for _, mod, is_pkg in mods])]
python_version = sys.version_info
runtime_meta["python_ver"] = str(python_version[0])+"."+str(python_version[1])
print("Done!")
return runtime_meta
"""
runtime_memory = 128
# old_stdout = sys.stdout
# sys.stdout = open(os.devnull, 'w')
action_name = self._format_action_name(docker_image_name,
runtime_memory)
self.cf_client.create_package(self.package)
self.cf_client.create_action(self.package,
action_name,
docker_image_name,
is_binary=False,
code=textwrap.dedent(action_code),
memory=runtime_memory,
timeout=30000)
# sys.stdout = old_stdout
logger.debug("Extracting Python modules list from: {}".format(
docker_image_name))
try:
retry_invoke = True
while retry_invoke:
retry_invoke = False
runtime_meta = self.invoke_with_result(docker_image_name,
runtime_memory)
if 'activationId' in runtime_meta:
retry_invoke = True
except Exception:
raise ("Unable to invoke 'modules' action")
try:
self.delete_runtime(docker_image_name, runtime_memory)
except Exception:
raise Exception("Unable to delete 'modules' action")
if not runtime_meta or 'preinstalls' not in runtime_meta:
raise Exception(runtime_meta)
return runtime_meta
class ComputeBackend:
"""
A wrap-up around IBM Cloud Functions backend.
"""
def __init__(self, ibm_cf_config):
self.log_level = os.getenv('CB_LOG_LEVEL')
self.name = 'ibm_cf'
self.ibm_cf_config = ibm_cf_config
self.package = 'pywren_v' + __version__
self.region = ibm_cf_config['region']
self.cf_client = CloudFunctionsClient(self.ibm_cf_config)
self.is_cf_cluster = is_cf_cluster()
self.namespace = ibm_cf_config[self.region]['namespace']
log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} '
'- Region: {}'.format(__version__, self.namespace,
self.region))
logger.info(log_msg)
if not self.log_level:
print(log_msg)
def _format_action_name(self, runtime_name, runtime_memory):
runtime_name = runtime_name.replace('/', '_').replace(':', '_')
return '{}_{}MB'.format(runtime_name, runtime_memory)
def _unformat_action_name(self, action_name):
runtime_name, memory = action_name.rsplit('_', 1)
image_name = runtime_name.replace('_', '/', 1)
image_name = image_name.replace('_', ':', -1)
return image_name, int(memory.replace('MB', ''))
def _get_default_runtime_image_name(self):
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
image_name = ibm_cf_config.RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
image_name = ibm_cf_config.RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
image_name = ibm_cf_config.RUNTIME_DEFAULT_37
return image_name
def _create_handler_zip(self):
logger.debug(
"Creating function handler zip in {}".format(ZIP_LOCATION))
def add_folder_to_zip(zip_file, full_dir_path, sub_dir=''):
for file in os.listdir(full_dir_path):
full_path = os.path.join(full_dir_path, file)
if os.path.isfile(full_path):
zip_file.write(
full_path,
os.path.join('pywren_ibm_cloud', sub_dir, file),
zipfile.ZIP_DEFLATED)
elif os.path.isdir(
full_path) and '__pycache__' not in full_path:
add_folder_to_zip(zip_file, full_path,
os.path.join(sub_dir, file))
try:
with zipfile.ZipFile(ZIP_LOCATION, 'w') as ibmcf_pywren_zip:
current_location = os.path.dirname(os.path.abspath(__file__))
module_location = os.path.dirname(
os.path.abspath(pywren_ibm_cloud.__file__))
main_file = os.path.join(current_location, 'entry_point.py')
ibmcf_pywren_zip.write(main_file, '__main__.py',
zipfile.ZIP_DEFLATED)
add_folder_to_zip(ibmcf_pywren_zip, module_location)
except Exception as e:
raise Exception('Unable to create the {} package: {}'.format(
ZIP_LOCATION, e))
def build_runtime(self, docker_image_name, dockerfile):
"""
Builds a new runtime from a Docker file and pushes it to the Docker hub
"""
logger.info('Creating a new docker image from Dockerfile')
logger.info('Docker image name: {}'.format(docker_image_name))
if dockerfile:
cmd = 'docker build -t {} -f {} .'.format(docker_image_name,
dockerfile)
else:
cmd = 'docker build -t {} .'.format(docker_image_name)
res = os.system(cmd)
if res != 0:
exit()
cmd = 'docker push {}'.format(docker_image_name)
res = os.system(cmd)
if res != 0:
exit()
def create_runtime(self, docker_image_name, memory, timeout=300000):
"""
Creates a new runtime into IBM CF namespace from an already built Docker image
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
logger.info(
'Creating new PyWren runtime based on Docker image {}'.format(
docker_image_name))
self.cf_client.create_package(self.package)
action_name = self._format_action_name(docker_image_name, memory)
self._create_handler_zip()
with open(ZIP_LOCATION, "rb") as action_zip:
action_bin = action_zip.read()
self.cf_client.create_action(self.package,
action_name,
docker_image_name,
code=action_bin,
memory=memory,
is_binary=True,
timeout=timeout)
return action_name
def delete_runtime(self, docker_image_name, memory):
"""
Deletes a runtime
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
action_name = self._format_action_name(docker_image_name, memory)
self.cf_client.delete_action(self.package, action_name)
def delete_all_runtimes(self):
"""
Deletes all runtimes from all packages
"""
packages = self.cf_client.list_packages()
for pkg in packages:
if 'pywren_v' in pkg['name']:
actions = self.cf_client.list_actions(pkg['name'])
while actions:
for action in actions:
self.cf_client.delete_action(pkg['name'],
action['name'])
actions = self.cf_client.list_actions(pkg['name'])
self.cf_client.delete_package(pkg['name'])
def list_runtimes(self, docker_image_name='all'):
"""
List all the runtimes deployed in the IBM CF service
return: list of tuples [docker_image_name, memory]
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
runtimes = []
actions = self.cf_client.list_actions(self.package)
for action in actions:
action_image_name, memory = self._unformat_action_name(
action['name'])
if docker_image_name == action_image_name or docker_image_name == 'all':
runtimes.append([action_image_name, memory])
return runtimes
def invoke(self, docker_image_name, runtime_memory, payload):
"""
Invoke -- return information about this invocation
"""
exec_id = payload['executor_id']
job_id = payload['job_id']
call_id = payload['call_id']
action_name = self._format_action_name(docker_image_name,
runtime_memory)
start = time.time()
activation_id, exception = self.cf_client.invoke(
self.package, action_name, payload, self.is_cf_cluster)
roundtrip = time.time() - start
resp_time = format(round(roundtrip, 3), '.3f')
if activation_id is None:
log_msg = (
'ExecutorID {} | JobID {} - Function {} invocation failed: {}'.
format(exec_id, job_id, call_id, str(exception)))
logger.debug(log_msg)
else:
log_msg = (
'ExecutorID {} | JobID {} - Function {} invocation done! ({}s) - Activation ID: '
'{}'.format(exec_id, job_id, call_id, resp_time,
activation_id))
logger.debug(log_msg)
return activation_id
def invoke_with_result(self,
docker_image_name,
runtime_memory,
payload={}):
"""
Invoke waiting for a result -- return information about this invocation
"""
action_name = self._format_action_name(docker_image_name,
runtime_memory)
return self.cf_client.invoke_with_result(self.package, action_name,
payload)
def get_runtime_key(self, docker_image_name, runtime_memory):
"""
Method that creates and returns the runtime key.
Runtime keys are used to uniquely identify runtimes within the storage,
in order to know which runtimes are installed and which not.
"""
action_name = self._format_action_name(docker_image_name,
runtime_memory)
runtime_key = os.path.join(self.name, self.region, self.namespace,
action_name)
return runtime_key
def generate_runtime_meta(self, docker_image_name):
"""
Extract installed Python modules from docker image
"""
if docker_image_name == 'default':
docker_image_name = self._get_default_runtime_image_name()
module_location = os.path.dirname(
os.path.abspath(pywren_ibm_cloud.__file__))
action_location = os.path.join(module_location, 'runtime',
'extract_preinstalls_fn.py')
with open(action_location, "r") as action_py:
action_code = action_py.read()
runtime_memory = 130
# old_stdout = sys.stdout
# sys.stdout = open(os.devnull, 'w')
action_name = self._format_action_name(docker_image_name,
runtime_memory)
self.cf_client.create_package(self.package)
self.cf_client.create_action(self.package,
action_name,
docker_image_name,
code=action_code,
memory=runtime_memory,
is_binary=False,
timeout=30000)
# sys.stdout = old_stdout
logger.debug("Extracting Python modules list from: {}".format(
docker_image_name))
try:
runtime_meta = self.invoke_with_result(docker_image_name,
runtime_memory)
except Exception:
raise ("Unable to invoke 'modules' action")
try:
self.delete_runtime(docker_image_name, runtime_memory)
except Exception:
raise ("Unable to delete 'modules' action")
if not runtime_meta or 'preinstalls' not in runtime_meta:
raise Exception(runtime_meta)
return runtime_meta