def runtime_valid(runtime_meta):
"""
Basic checks
"""
logger.debug("Verifying Python versions")
this_version_str = version_str(sys.version_info)
return this_version_str == runtime_meta['python_ver']
def _get_default_runtime_image_name(self):
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
image_name = ibmcf_config.RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
image_name = ibmcf_config.RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
image_name = ibmcf_config.RUNTIME_DEFAULT_37
return image_name
def select_runtime(self, job_id, runtime_memory):
"""
Auxiliary method that selects the runtime to use. To do so it gets the
runtime metadata from the storage. This metadata contains the preinstalled
python modules needed to serialize the local function. If the .metadata
file does not exists in the storage, this means that the runtime is not
installed, so this method will proceed to install it.
"""
log_level = os.getenv('PYWREN_LOGLEVEL')
runtime_name = self.config['pywren']['runtime']
if runtime_memory is None:
runtime_memory = self.config['pywren']['runtime_memory']
runtime_memory = int(runtime_memory)
log_msg = ('ExecutorID {} | JobID {} - Selected Runtime: {} - {}MB'
.format(self.executor_id, job_id, runtime_name, runtime_memory))
logger.info(log_msg)
if not log_level:
print(log_msg, end=' ')
installing = False
for compute_handler in self.compute_handlers:
runtime_key = compute_handler.get_runtime_key(runtime_name, runtime_memory)
runtime_deployed = True
try:
runtime_meta = self.internal_storage.get_runtime_meta(runtime_key)
except Exception:
runtime_deployed = False
if not runtime_deployed:
logger.debug('ExecutorID {} | JobID {} - Runtime {} with {}MB is not yet '
'installed'.format(self.executor_id, job_id, runtime_name, runtime_memory))
if not log_level and not installing:
installing = True
print('(Installing...)')
timeout = self.config['pywren']['runtime_timeout']
logger.debug('Creating runtime: {}, memory: {}MB'.format(runtime_name, runtime_memory))
runtime_meta = compute_handler.create_runtime(runtime_name, runtime_memory, timeout=timeout)
self.internal_storage.put_runtime_meta(runtime_key, runtime_meta)
py_local_version = version_str(sys.version_info)
py_remote_version = runtime_meta['python_ver']
if py_local_version != py_remote_version:
raise Exception(("The indicated runtime '{}' is running Python {} and it "
"is not compatible with the local Python version {}")
.format(runtime_name, py_remote_version, py_local_version))
if not log_level and runtime_deployed:
print()
return runtime_meta
def load_config(config_data):
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_37
if 'workers' not in config_data['pywren'] or \
config_data['pywren']['workers'] > MAX_CONCURRENT_WORKERS:
config_data['pywren']['workers'] = MAX_CONCURRENT_WORKERS
if 'ibm_cf' not in config_data:
raise Exception("ibm_cf section is mandatory in the configuration")
required_parameters_0 = ('endpoint', 'namespace')
required_parameters_1 = ('endpoint', 'namespace', 'api_key')
required_parameters_2 = ('endpoint', 'namespace', 'namespace_id',
'ibm:iam_api_key')
# Check old format. Convert to new format
if set(required_parameters_0) <= set(config_data['ibm_cf']):
endpoint = config_data['ibm_cf'].pop('endpoint')
namespace = config_data['ibm_cf'].pop('namespace')
api_key = config_data['ibm_cf'].pop('api_key', None)
namespace_id = config_data['ibm_cf'].pop('namespace_id', None)
region = endpoint.split('//')[1].split('.')[0].replace('-', '_')
for k in list(config_data['ibm_cf']):
# Delete unnecessary keys
del config_data['ibm_cf'][k]
config_data['ibm_cf']['regions'] = {}
config_data['pywren']['compute_backend_region'] = region
config_data['ibm_cf']['regions'][region] = {
'endpoint': endpoint,
'namespace': namespace
}
if api_key:
config_data['ibm_cf']['regions'][region]['api_key'] = api_key
if namespace_id:
config_data['ibm_cf']['regions'][region][
'namespace_id'] = namespace_id
# -------------------
if 'ibm' in config_data and config_data['ibm'] is not None:
config_data['ibm_cf'].update(config_data['ibm'])
for region in config_data['ibm_cf']['regions']:
if not set(required_parameters_1) <= set(config_data['ibm_cf']['regions'][region]) \
and (not set(required_parameters_0) <= set(config_data['ibm_cf']['regions'][region])
or 'namespace_id' not in config_data['ibm_cf']['regions'][region] or 'iam_api_key' not in config_data['ibm_cf']):
raise Exception('You must provide {} or {} to access to IBM Cloud '
'Functions'.format(required_parameters_1,
required_parameters_2))
cbr = config_data['pywren'].get('compute_backend_region')
if type(cbr) == list:
for region in cbr:
if region not in config_data['ibm_cf']['regions']:
raise Exception(
'Invalid Compute backend region: {}'.format(region))
else:
if cbr is None:
cbr = list(config_data['ibm_cf']['regions'].keys())[0]
config_data['pywren']['compute_backend_region'] = cbr
if cbr not in config_data['ibm_cf']['regions']:
raise Exception('Invalid Compute backend region: {}'.format(cbr))
def default(config_data=None):
"""
First checks .pywren_config
then checks PYWREN_CONFIG_FILE environment variable
then ~/.pywren_config
"""
if not config_data:
if 'PYWREN_CONFIG' in os.environ:
config_data = json.loads(os.environ.get('PYWREN_CONFIG'))
else:
config_filename = get_default_config_filename()
if config_filename is None:
raise ValueError("could not find configuration file")
config_data = load(config_filename)
if not set(('pywren', 'ibm_cf', 'ibm_cos')).issubset(set(config_data)):
raise Exception("pywren, ibm_cf and ibm_cos sections are mandatory in the configuration")
if 'storage_backend' not in config_data['pywren']:
config_data['pywren']['storage_backend'] = STORAGE_BACKEND_DEFAULT
if 'storage_bucket' not in config_data['pywren']:
config_data['pywren']['storage_bucket'] = COS_BUCKET_DEFAULT
if 'storage_prefix' not in config_data['pywren']:
config_data['pywren']['storage_prefix'] = COS_PREFIX_DEFAULT
if 'data_cleaner' not in config_data['pywren']:
config_data['pywren']['data_cleaner'] = DATA_CLEANER_DEFAULT
if 'invocation_retry' not in config_data['pywren']:
config_data['pywren']['invocation_retry'] = INVOCATION_RETRY_DEFAULT
if 'retry_sleeps' not in config_data['pywren']:
config_data['pywren']['retry_sleeps'] = RETRY_SLEEPS_DEFAULT
if 'retries' not in config_data['pywren']:
config_data['pywren']['retries'] = RETRIES_DEFAULT
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'compute_backend' not in config_data['pywren']:
config_data['pywren']['compute_backend'] = COMPUTE_BACKEND_DEFAULT
if 'runtime' not in config_data['pywren']:
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_37
if 'ibm_iam' not in config_data or config_data['ibm_iam'] is None:
config_data['ibm_iam'] = {}
if 'ibm_auth_endpoint' not in config_data['ibm_iam']:
config_data['ibm_iam']['ibm_auth_endpoint'] = IBM_AUTH_ENDPOINT_DEFAULT
if 'api_key' not in config_data['ibm_iam'] and 'api_key' not in config_data['ibm_cf']:
raise Exception("You must provide an IAM api_key, or CF api_key in the configuration")
if 'rabbitmq' not in config_data or not config_data['rabbitmq'] \
or 'amqp_url' not in config_data['rabbitmq']:
config_data['rabbitmq'] = {}
config_data['rabbitmq']['amqp_url'] = None
return config_data