def load_config(config_data):
required_keys = ('docker_user', 'docker_token')
if not set(required_keys) <= set(config_data['knative']):
raise Exception('You must provide {} to access to Knative'.format(required_keys))
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'docker_repo' not in config_data['knative']:
config_data['knative']['docker_repo'] = DOCKER_REPO_DEFAULT
if 'runtime' not in config_data['pywren']:
docker_user = config_data['knative']['docker_user']
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_35.replace('<USER>', docker_user)
elif this_version_str == '3.6':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36.replace('<USER>', docker_user)
elif this_version_str == '3.7':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_37.replace('<USER>', docker_user)
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = CONCURRENT_WORKERS_DEFAULT
def load_config(config_data):
if 'openwhisk' not in config_data:
raise Exception("openwhisk section is mandatory in configuration")
required_keys = ('endpoint', 'namespace', 'api_key')
if not set(required_keys) <= set(config_data['openwhisk']):
raise Exception('You must provide {} to access to openwhisk'.format(required_keys))
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
python_version = version_str(sys.version_info)
try:
config_data['pywren']['runtime'] = RUNTIME_DEFAULT[python_version]
except KeyError:
raise Exception('Unsupported Python version: {}'.format(python_version))
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = CONCURRENT_WORKERS_DEFAULT
if 'ibm_cos' in config_data and 'private_endpoint' in config_data['ibm_cos']:
del config_data['ibm_cos']['private_endpoint']
def runtime_valid(runtime_meta):
"""
Basic checks
"""
logger.debug("Verifying Python versions")
this_version_str = version_str(sys.version_info)
return this_version_str == runtime_meta['python_ver']
def load_config(config_data):
if 'cloudrun' not in config_data:
raise Exception("cloudrun section is mandatory in configuration")
required_keys = ('project_id', 'region')
if not set(required_keys) <= set(config_data['cloudrun']):
raise Exception(
'You must provide {} to access to Cloud Run'.format(required_keys))
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
project_id = config_data['cloudrun']['project_id']
python_version = version_str(sys.version_info).replace('.', '')
revision = 'latest' if 'SNAPSHOT' in __version__ else __version__.replace(
'.', '')
runtime_name = '{}/{}-v{}:{}'.format(project_id, RUNTIME_NAME_DEFAULT,
python_version, revision)
config_data['pywren']['runtime'] = runtime_name
if 'workers' not in config_data['pywren']:
config_data['cloudrun']['workers'] = CONCURRENT_WORKERS_DEFAULT
config_data['pywren']['workers'] = CONCURRENT_WORKERS_DEFAULT
def load_config(config_data):
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_37
if 'docker' not in config_data:
config_data['docker'] = {}
if 'workers' in config_data['pywren']:
config_data['docker']['workers'] = config_data['pywren']['workers']
else:
if 'workers' not in config_data['docker']:
total_cores = multiprocessing.cpu_count()
config_data['pywren']['workers'] = total_cores
config_data['docker']['workers'] = total_cores
else:
config_data['pywren']['workers'] = config_data['docker']['workers']
def load_config(config_data):
if 'knative' not in config_data:
raise Exception("knative section is mandatory in configuration")
required_keys = ('docker_user', 'docker_token')
if not set(required_keys) <= set(config_data['knative']):
raise Exception('You must provide {} to access to Knative'.format(required_keys))
if 'git_url' not in config_data['knative']:
config_data['knative']['git_url'] = BUILD_GIT_URL_DEFAULT
if 'git_rev' not in config_data['knative']:
revision = 'master' if 'SNAPSHOT' in __version__ else __version__
config_data['knative']['git_rev'] = revision
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'docker_repo' not in config_data['knative']:
config_data['knative']['docker_repo'] = DOCKER_REPO_DEFAULT
if 'runtime' not in config_data['pywren']:
docker_user = config_data['knative']['docker_user']
python_version = version_str(sys.version_info).replace('.', '')
revision = 'latest' if 'SNAPSHOT' in __version__ else __version__.replace('.', '')
runtime_name = '{}/{}-v{}:{}'.format(docker_user, RUNTIME_NAME_DEFAULT, python_version, revision)
config_data['pywren']['runtime'] = runtime_name
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = CONCURRENT_WORKERS_DEFAULT
def _get_default_runtime_image_name(self):
project_id = self.cloudrun_config['project_id']
python_version = version_str(sys.version_info).replace('.', '')
revision = 'latest' if 'SNAPSHOT' in __version__ else __version__.replace(
'.', '')
return '{}/{}-v{}:{}'.format(project_id,
cr_config.RUNTIME_NAME_DEFAULT,
python_version, revision)
def _get_default_image_name():
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
image_name = wrenconfig.RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
image_name = wrenconfig.RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
image_name = wrenconfig.RUNTIME_DEFAULT_37
return image_name
def _get_default_dockefile(self):
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
dockefile = docker_config.DOCKERFILE_DEFAULT_35
elif this_version_str == '3.6':
dockefile = docker_config.DOCKERFILE_DEFAULT_36
elif this_version_str == '3.7':
dockefile = docker_config.DOCKERFILE_DEFAULT_37
return dockefile
def _get_default_runtime_image_name(self):
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
image_name = ibmcf_config.RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
image_name = ibmcf_config.RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
image_name = ibmcf_config.RUNTIME_DEFAULT_37
return image_name
def _get_default_runtime_image_name(self):
docker_user = self.knative_config['docker_user']
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
image_name = kconfig.RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
image_name = kconfig.RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
image_name = kconfig.RUNTIME_DEFAULT_37
return image_name.replace('<USER>', docker_user)
def select_runtime(self, job_id, runtime_memory):
"""
Auxiliary method that selects the runtime to use. To do so it gets the
runtime metadata from the storage. This metadata contains the preinstalled
python modules needed to serialize the local function. If the .metadata
file does not exists in the storage, this means that the runtime is not
installed, so this method will proceed to install it.
"""
log_level = os.getenv('PYWREN_LOGLEVEL')
runtime_name = self.pywren_config['pywren']['runtime']
if runtime_memory is None:
runtime_memory = self.pywren_config['pywren']['runtime_memory']
runtime_memory = int(runtime_memory)
log_msg = ('ExecutorID {} | JobID {} - Selected Runtime: {} - {}MB'
.format(self.executor_id, job_id, runtime_name, runtime_memory))
logger.info(log_msg)
if not log_level:
print(log_msg, end=' ')
installing = False
for compute_handler in self.compute_handlers:
runtime_key = compute_handler.get_runtime_key(runtime_name, runtime_memory)
runtime_deployed = True
try:
runtime_meta = self.internal_storage.get_runtime_meta(runtime_key)
except Exception:
runtime_deployed = False
if not runtime_deployed:
logger.debug('ExecutorID {} | JobID {} - Runtime {} with {}MB is not yet '
'installed'.format(self.executor_id, job_id, runtime_name, runtime_memory))
if not log_level and not installing:
installing = True
print('(Installing...)')
timeout = self.pywren_config['pywren']['runtime_timeout']
logger.debug('Creating runtime: {}, memory: {}MB'.format(runtime_name, runtime_memory))
runtime_meta = compute_handler.create_runtime(runtime_name, runtime_memory, timeout=timeout)
self.internal_storage.put_runtime_meta(runtime_key, runtime_meta)
py_local_version = version_str(sys.version_info)
py_remote_version = runtime_meta['python_ver']
if py_local_version != py_remote_version:
raise Exception(("The indicated runtime '{}' is running Python {} and it "
"is not compatible with the local Python version {}")
.format(runtime_name, py_remote_version, py_local_version))
if not log_level and runtime_deployed:
print()
return runtime_meta
def load_config(config_data=None):
if config_data is None:
config_data = {}
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
config_data['pywren']['runtime'] = 'python' + \
version_str(sys.version_info)
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = MAX_CONCURRENT_WORKERS
if config_data['pywren']['runtime_memory'] not in RUNTIME_MEMORY_OPTIONS:
raise Exception('{} MB runtime is not available (Only {} MB)'.format(
config_data['pywren']['runtime_memory'], RUNTIME_MEMORY_OPTIONS))
if config_data['pywren']['runtime_memory'] > RUNTIME_MEMORY_MAX:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_MAX
if config_data['pywren']['runtime_timeout'] > RUNTIME_TIMEOUT_DEFAULT:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'gcp' not in config_data:
raise Exception("'gcp' section is mandatory in the configuration")
config_data['gcp']['retries'] = RETRIES
config_data['gcp']['retry_sleeps'] = RETRY_SLEEPS
# Put storage data into compute backend config dict entry
storage_config = dict()
storage_config['pywren'] = config_data['pywren'].copy()
storage_config['gcp_storage'] = config_data['gcp'].copy()
config_data['gcp']['storage'] = pw_config.extract_storage_config(
storage_config)
required_parameters_0 = ('project_name', 'service_account',
'credentials_path')
if not set(required_parameters_0) <= set(config_data['gcp']):
raise Exception(
"'project_name', 'service_account' and 'credentials_path' \
are mandatory under 'gcp' section")
if not exists(config_data['gcp']['credentials_path']) or not isfile(
config_data['gcp']['credentials_path']):
raise Exception("Path {} must be credentials JSON file.".format(
config_data['gcp']['credentials_path']))
config_data['gcp_functions'] = config_data['gcp'].copy()
if 'region' not in config_data['gcp_functions']:
config_data['gcp_functions']['region'] = config_data['pywren'][
'compute_backend_region']
def _generate_python_meta(self):
"""
Extracts installed Python modules from the local machine
"""
logger.debug("Extracting preinstalled Python modules...")
runtime_meta = dict()
mods = list(pkgutil.iter_modules())
runtime_meta["preinstalls"] = [
entry
for entry in sorted([[mod, is_pkg] for _, mod, is_pkg in mods])
]
runtime_meta["python_ver"] = version_str(sys.version_info)
return runtime_meta
def load_config(config_data=None):
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if config_data['pywren'][
'runtime_memory'] % 64 != 0: # Adjust 64 MB memory increments restriction
mem = config_data['pywren']['runtime_memory']
config_data['pywren']['runtime_memory'] = (mem + (64 - (mem % 64)))
if config_data['pywren']['runtime_memory'] > RUNTIME_MEMORY_MAX:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_MAX
if 'runtime_timeout' not in config_data['pywren'] or \
config_data['pywren']['runtime_timeout'] > RUNTIME_TIMEOUT_DEFAULT:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
config_data['pywren']['runtime'] = 'python' + version_str(
sys.version_info)
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = MAX_CONCURRENT_WORKERS
if 'aws' not in config_data and 'aws_lambda' not in config_data:
raise Exception(
"'aws' and 'aws_lambda' sections are mandatory in the configuration"
)
# Put credential keys to 'aws_lambda' dict entry
config_data['aws_lambda'] = {
**config_data['aws_lambda'],
**config_data['aws']
}
required_parameters_0 = ('access_key_id', 'secret_access_key')
if not set(required_parameters_0) <= set(config_data['aws']):
raise Exception(
"'access_key_id' and 'secret_access_key' are mandatory under 'aws' section"
)
if 'execution_role' not in config_data['aws_lambda']:
raise Exception(
"'execution_role' is mandatory under 'aws_lambda' section")
if 'compute_backend_region' not in config_data['pywren'] \
and 'region_name' not in config_data['aws_lambda']:
raise Exception(
"'compute_backend_region' or 'region_name' not specified")
else:
config_data['aws_lambda']['region'] = config_data['aws_lambda'][
'region_name']
def load_config(config_data=None):
this_version_str = version_str(sys.version_info)
if this_version_str != '3.6':
raise Exception(
'The functions backend Azure Function Apps currently'
' only supports Python version 3.6.X and the local Python'
'version is {}'.format(this_version_str))
if 'runtime' in config_data['pywren']:
print("Ignoring user specified '{}'. The current Azure compute backend"
" does not support custom runtimes.".format('runtime'))
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36
if 'runtime_memory' in config_data['pywren']:
print("Ignoring user specified '{}'. The current Azure compute backend"
" does not support custom runtimes.".format('runtime_memory'))
print('Default runtime memory: {}MB'.format(RUNTIME_MEMORY_DEFAULT))
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' in config_data['pywren']:
print("Ignoring user specified '{}'. The current Azure compute backend"
" does not support custom runtimes.".format('runtime_timeout'))
print('Default runtime timeout: {}ms'.format(RUNTIME_TIMEOUT_DEFAULT))
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = MAX_CONCURRENT_WORKERS
if 'azure_fa' not in config_data:
raise Exception("azure_fa section is mandatory in the configuration")
required_parameters = ('resource_group', 'location', 'account_name',
'account_key')
if set(required_parameters) > set(config_data['azure_fa']):
raise Exception('You must provide {} to access to Azure Function App '\
.format(required_parameters))
if 'functions_version' not in config_data['azure_fa']:
config_data['pywren']['functions_version'] = FUNCTIONS_VERSION_DEFAULT
elif config_data['azure_fa']['functions_version'] not in (2, 3):
raise Exception('You must provide a valid Azure Functions App version {}'\
.format((2, 3)))
def _build_default_runtime(self, default_runtime_img_name):
"""
Builds the default runtime
"""
location = 'https://raw.githubusercontent.com/tomwhite/pywren-ibm-cloud/master/runtime/cloudrun'
python_version = version_str(sys.version_info).replace('.', '')
resp = requests.get('{}/Dockerfile.python{}'.format(
location, python_version))
dockerfile = "Dockerfile"
if resp.status_code == 200:
with open(dockerfile, 'w') as f:
f.write(resp.text)
self.build_runtime(default_runtime_img_name, dockerfile)
os.remove(dockerfile)
else:
msg = 'There was an error fetching the default runtime Dockerfile: {}'.format(
resp.text)
logger.error(msg)
exit()
def load_config(config_data):
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = None
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
python_version = version_str(sys.version_info)
try:
config_data['pywren']['runtime'] = RUNTIME_DEFAULT[python_version]
except KeyError:
raise Exception(
'Unsupported Python version: {}'.format(python_version))
config_data['pywren']['remote_invoker'] = True
if 'storage_backend' not in config_data['pywren']:
config_data['pywren']['storage_backend'] = 'localhost'
if 'docker' not in config_data:
config_data['docker'] = {'host': 'localhost'}
if config_data['docker']['host'] not in ['127.0.0.1', 'localhost']:
if 'ssh_user' not in config_data[
'docker'] or 'ssh_password' not in config_data['docker']:
raise Exception(
'You must provide ssh credentials to access to the remote host'
)
if config_data['pywren']['storage_backend'] == 'localhost':
raise Exception(
'Localhost storage backend is not supported for Docker remote host'
)
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = None
if 'ibm_cos' in config_data and 'private_endpoint' in config_data[
'ibm_cos']:
del config_data['ibm_cos']['private_endpoint']
def _build_default_runtime(self, default_runtime_img_name):
"""
Builds the default runtime
"""
if os.system('docker --version >{} 2>&1'.format(os.devnull)) == 0:
# Build default runtime using local dokcer
python_version = version_str(sys.version_info).replace('.', '')
location = 'https://raw.githubusercontent.com/pywren/pywren-ibm-cloud/master/runtime/knative'
resp = requests.get('{}/Dockerfile.python{}'.format(location, python_version))
dockerfile = "Dockefile.default-kantive-runtime"
if resp.status_code == 200:
with open(dockerfile, 'w') as f:
f.write(resp.text)
self.build_runtime(default_runtime_img_name, dockerfile)
os.remove(dockerfile)
else:
msg = 'There was an error fetching the default runitme Dockerfile: {}'.format(resp.text)
logger.error(msg)
exit()
else:
# Build default runtime using Tekton
self._build_default_runtime_from_git(default_runtime_img_name)
def load_config(config_data):
if 'openwhisk' not in config_data:
raise Exception("openwhisk section is mandatory in configuration")
required_keys = ('endpoint', 'namespace', 'api_key')
if not set(required_keys) <= set(config_data['openwhisk']):
raise Exception(
'You must provide {} to access to openwhisk'.format(required_keys))
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_37
if 'workers' not in config_data['pywren']:
config_data['pywren']['workers'] = CONCURRENT_WORKERS_DEFAULT
def load_config(config_data=None):
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_37
if 'ibm_cf' not in config_data:
raise Exception("ibm_cf section is mandatory in the configuration")
required_parameters_0 = ('endpoint', 'namespace')
required_parameters_1 = ('endpoint', 'namespace', 'api_key')
required_parameters_2 = ('endpoint', 'namespace', 'namespace_id',
'ibm:iam_api_key')
# Check old format. Convert to new format
if set(required_parameters_0) <= set(config_data['ibm_cf']):
endpoint = config_data['ibm_cf'].pop('endpoint')
namespace = config_data['ibm_cf'].pop('namespace')
api_key = config_data['ibm_cf'].pop('api_key', None)
namespace_id = config_data['ibm_cf'].pop('namespace_id', None)
region = endpoint.split('//')[1].split('.')[0].replace('-', '_')
for k in list(config_data['ibm_cf']):
# Delete unnecessary keys
del config_data['ibm_cf'][k]
config_data['ibm_cf']['regions'] = {}
config_data['pywren']['compute_backend_region'] = region
config_data['ibm_cf']['regions'][region] = {
'endpoint': endpoint,
'namespace': namespace
}
if api_key:
config_data['ibm_cf']['regions'][region]['api_key'] = api_key
if namespace_id:
config_data['ibm_cf']['regions'][region][
'namespace_id'] = namespace_id
# -------------------
if 'ibm' in config_data and config_data['ibm'] is not None:
config_data['ibm_cf'].update(config_data['ibm'])
for region in config_data['ibm_cf']['regions']:
if not set(required_parameters_1) <= set(config_data['ibm_cf']['regions'][region]) \
and (not set(required_parameters_0) <= set(config_data['ibm_cf']['regions'][region])
or 'namespace_id' not in config_data['ibm_cf']['regions'][region] or 'iam_api_key' not in config_data['ibm_cf']):
raise Exception('You must provide {} or {} to access to IBM Cloud '
'Functions'.format(required_parameters_1,
required_parameters_2))
if 'compute_backend_region' not in config_data['pywren']:
config_data['pywren']['compute_backend_region'] = list(
config_data['ibm_cf']['regions'].keys())[0]
cbr = config_data['pywren']['compute_backend_region']
if cbr is not None and cbr not in config_data['ibm_cf']['regions']:
raise Exception('Invalid Compute backend region: {}'.format(cbr))
def _get_default_runtime_image_name(self):
python_version = version_str(sys.version_info)
return openwhisk_config.RUNTIME_DEFAULT[python_version]
def default(config_data=None):
"""
First checks .pywren_config
then checks PYWREN_CONFIG_FILE environment variable
then ~/.pywren_config
"""
if not config_data:
if 'PYWREN_CONFIG' in os.environ:
config_data = json.loads(os.environ.get('PYWREN_CONFIG'))
else:
config_filename = get_default_config_filename()
if config_filename is None:
raise ValueError("could not find configuration file")
config_data = load(config_filename)
if not set(('pywren', 'ibm_cf', 'ibm_cos')).issubset(set(config_data)):
raise Exception(
"pywren, ibm_cf and ibm_cos sections are mandatory in the configuration"
)
if 'storage_backend' not in config_data['pywren']:
config_data['pywren']['storage_backend'] = STORAGE_BACKEND_DEFAULT
if 'storage_bucket' not in config_data['pywren']:
config_data['pywren']['storage_bucket'] = COS_BUCKET_DEFAULT
if 'storage_prefix' not in config_data['pywren']:
config_data['pywren']['storage_prefix'] = COS_PREFIX_DEFAULT
if 'data_cleaner' not in config_data['pywren']:
config_data['pywren']['data_cleaner'] = DATA_CLEANER_DEFAULT
if 'invocation_retry' not in config_data['pywren']:
config_data['pywren']['invocation_retry'] = INVOCATION_RETRY_DEFAULT
if 'retry_sleeps' not in config_data['pywren']:
config_data['pywren']['retry_sleeps'] = RETRY_SLEEPS_DEFAULT
if 'retries' not in config_data['pywren']:
config_data['pywren']['retries'] = RETRIES_DEFAULT
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_37
if 'ibm_iam' not in config_data or config_data['ibm_iam'] is None:
config_data['ibm_iam'] = {}
if 'ibm_auth_endpoint' not in config_data['ibm_iam']:
config_data['ibm_iam']['ibm_auth_endpoint'] = IBM_AUTH_ENDPOINT_DEFAULT
if 'api_key' not in config_data[
'ibm_iam'] and 'api_key' not in config_data['ibm_cf']:
raise Exception(
"You must provide an IAM api_key, or CF api_key in the configuration"
)
if 'rabbitmq' not in config_data or not config_data['rabbitmq'] \
or 'amqp_url' not in config_data['rabbitmq']:
config_data['rabbitmq'] = {}
config_data['rabbitmq']['amqp_url'] = None
return config_data
def load_config(config_data):
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
python_version = version_str(sys.version_info)
try:
config_data['pywren']['runtime'] = RUNTIME_DEFAULT[python_version]
except KeyError:
raise Exception(
'Unsupported Python version: {}'.format(python_version))
if 'workers' not in config_data['pywren'] or \
config_data['pywren']['workers'] > MAX_CONCURRENT_WORKERS:
config_data['pywren']['workers'] = MAX_CONCURRENT_WORKERS
if 'ibm_cf' not in config_data:
raise Exception("ibm_cf section is mandatory in the configuration")
required_parameters_0 = ('endpoint', 'namespace')
required_parameters_1 = ('endpoint', 'namespace', 'api_key')
required_parameters_2 = ('endpoint', 'namespace', 'namespace_id',
'ibm:iam_api_key')
# Check old format. Convert to new format
if set(required_parameters_0) <= set(config_data['ibm_cf']):
endpoint = config_data['ibm_cf'].pop('endpoint')
namespace = config_data['ibm_cf'].pop('namespace')
api_key = config_data['ibm_cf'].pop('api_key', None)
namespace_id = config_data['ibm_cf'].pop('namespace_id', None)
region = endpoint.split('//')[1].split('.')[0].replace('-', '_')
for k in list(config_data['ibm_cf']):
# Delete unnecessary keys
del config_data['ibm_cf'][k]
config_data['ibm_cf']['regions'] = {}
config_data['pywren']['compute_backend_region'] = region
config_data['ibm_cf']['regions'][region] = {
'endpoint': endpoint,
'namespace': namespace
}
if api_key:
config_data['ibm_cf']['regions'][region]['api_key'] = api_key
if namespace_id:
config_data['ibm_cf']['regions'][region][
'namespace_id'] = namespace_id
# -------------------
if 'ibm' in config_data and config_data['ibm'] is not None:
config_data['ibm_cf'].update(config_data['ibm'])
for region in config_data['ibm_cf']['regions']:
if not set(required_parameters_1) <= set(config_data['ibm_cf']['regions'][region]) \
and (not set(required_parameters_0) <= set(config_data['ibm_cf']['regions'][region])
or 'namespace_id' not in config_data['ibm_cf']['regions'][region] or 'iam_api_key' not in config_data['ibm_cf']):
raise Exception('You must provide {} or {} to access to IBM Cloud '
'Functions'.format(required_parameters_1,
required_parameters_2))
cbr = config_data['pywren'].get('compute_backend_region')
if type(cbr) == list:
for region in cbr:
if region not in config_data['ibm_cf']['regions']:
raise Exception(
'Invalid Compute backend region: {}'.format(region))
else:
if cbr is None:
cbr = list(config_data['ibm_cf']['regions'].keys())[0]
config_data['pywren']['compute_backend_region'] = cbr
if cbr not in config_data['ibm_cf']['regions']:
raise Exception('Invalid Compute backend region: {}'.format(cbr))
def _build_default_runtime_from_git(self, docker_image_name):
"""
Builds the default runtime and pushes it to the docker container registry
"""
image_name, revision = docker_image_name.split(':')
if self.knative_config['docker_repo'] == 'docker.io' and revision != 'latest':
resp = requests.get('https://index.docker.io/v1/repositories/{}/tags/{}'
.format(docker_image_name, revision))
if resp.status_code == 200:
logger.debug('Docker image docker.io/{}:{} already exists in Dockerhub. '
'Skipping build process.'.format(docker_image_name, revision))
return
logger.debug("Building default PyWren runtime from git with Tekton")
if not {"docker_user", "docker_token"} <= set(self.knative_config):
raise Exception("You must provide 'docker_user' and 'docker_token'"
" to build the default runtime")
task_run = yaml.safe_load(kconfig.task_run)
task_run['spec']['inputs']['params'] = []
python_version = version_str(sys.version_info).replace('.', '')
path_to_dockerfile = {'name': 'pathToDockerFile',
'value': 'pywren_ibm_cloud/compute/backends/knative/tekton/Dockerfile.python{}'.format(python_version)}
task_run['spec']['inputs']['params'].append(path_to_dockerfile)
image_url = {'name': 'imageUrl',
'value': '/'.join([self.knative_config['docker_repo'], image_name])}
task_run['spec']['inputs']['params'].append(image_url)
image_tag = {'name': 'imageTag',
'value': revision}
task_run['spec']['inputs']['params'].append(image_tag)
self._create_account_resources()
self._create_build_resources()
task_run_name = task_run['metadata']['name']
try:
self.api.delete_namespaced_custom_object(
group="tekton.dev",
version="v1alpha1",
name=task_run_name,
namespace=self.namespace,
plural="taskruns",
body=client.V1DeleteOptions()
)
except Exception:
pass
self.api.create_namespaced_custom_object(
group="tekton.dev",
version="v1alpha1",
namespace=self.namespace,
plural="taskruns",
body=task_run
)
logger.debug("Building runtime...")
pod_name = None
w = watch.Watch()
for event in w.stream(self.api.list_namespaced_custom_object, namespace=self.namespace,
group="tekton.dev", version="v1alpha1", plural="taskruns",
field_selector="metadata.name={0}".format(task_run_name)):
if event['object'].get('status'):
pod_name = event['object']['status']['podName']
w.stop()
if pod_name is None:
raise Exception('Unable to get the pod name from the task that is building the runtime')
w = watch.Watch()
for event in w.stream(self.v1.list_namespaced_pod, namespace=self.namespace,
field_selector="metadata.name={0}".format(pod_name)):
if event['object'].status.phase == "Succeeded":
w.stop()
if event['object'].status.phase == "Failed":
w.stop()
logger.debug('Something went wrong building the default PyWren runtime with Tekton')
for container in event['object'].status.container_statuses:
if container.state.terminated.reason == 'Error':
logs = self.v1.read_namespaced_pod_log(name=pod_name,
container=container.name,
namespace=self.namespace)
logger.debug("Tekton container '{}' failed: {}".format(container.name, logs.strip()))
raise Exception('Unable to build the default PyWren runtime with Tekton')
self.api.delete_namespaced_custom_object(
group="tekton.dev",
version="v1alpha1",
name=task_run_name,
namespace=self.namespace,
plural="taskruns",
body=client.V1DeleteOptions()
)
logger.debug('Default PyWren runtime built from git and uploaded to Dockerhub')
def _get_default_runtime_image_name(self):
docker_user = self.knative_config['docker_user']
python_version = version_str(sys.version_info).replace('.', '')
revision = 'latest' if 'SNAPSHOT' in __version__ else __version__.replace('.', '')
return '{}/{}-v{}:{}'.format(docker_user, kconfig.RUNTIME_NAME_DEFAULT, python_version, revision)
def load_config(config_data=None):
if 'runtime_memory' not in config_data['pywren']:
config_data['pywren']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['pywren']:
config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['pywren']:
this_version_str = version_str(sys.version_info)
if this_version_str == '3.5':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_35
elif this_version_str == '3.6':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_36
elif this_version_str == '3.7':
config_data['pywren']['runtime'] = RUNTIME_DEFAULT_37
if 'ibm_cf' not in config_data:
raise Exception("ibm_cf section is mandatory in the configuration")
if 'ibm_iam' in config_data['ibm_cf']:
del config_data['ibm_cf']['ibm_iam']
required_parameters_0 = ('endpoint', 'namespace')
if set(required_parameters_0) <= set(config_data['ibm_cf']):
# old format. convert to new format
endpoint = config_data['ibm_cf'].pop('endpoint')
namespace = config_data['ibm_cf'].pop('namespace')
region = endpoint.split('//')[1].split('.')[0].replace('-', '_')
config_data['pywren']['compute_backend_region'] = region
config_data['ibm_cf'][region] = {
'endpoint': endpoint,
'namespace': namespace
}
if 'api_key' in config_data['ibm_cf']:
config_data['ibm_cf'][region]['api_key'] = config_data[
'ibm_cf'].pop('api_key')
else:
# new format
for region in config_data['ibm_cf']:
required_parameters_1 = ('endpoint', 'namespace', 'api_key')
required_parameters_2 = ('endpoint', 'namespace',
'ibm_iam:api_key')
if set(required_parameters_1) <= set(config_data['ibm_cf'][region]) or \
set(required_parameters_0) <= set(config_data['ibm_cf'][region]) and 'api_key' in config_data['ibm_iam']:
pass
else:
raise Exception(
'You must provide {} or {} to access to IBM Cloud Functions'
.format(required_parameters_1, required_parameters_2))
if 'compute_backend_region' not in config_data['pywren']:
config_data['pywren']['compute_backend_region'] = list(
config_data['ibm_cf'].keys())[0]
cbr = config_data['pywren']['compute_backend_region']
if cbr is not None and cbr not in config_data['ibm_cf']:
raise Exception('Invalid Compute backend region: {}'.format(cbr))
if 'ibm_iam' not in config_data or config_data['ibm_iam'] is None:
config_data['ibm_iam'] = {}
if 'ibm_auth_endpoint' not in config_data['ibm_iam']:
config_data['ibm_iam']['ibm_auth_endpoint'] = IBM_AUTH_ENDPOINT_DEFAULT
config_data['ibm_cf']['ibm_iam'] = config_data['ibm_iam']
from pywren_ibm_cloud.utils import version_str
from pywren_ibm_cloud.config import cloud_logging_config
from pywren_ibm_cloud.runtime.function_handler import function_handler
cloud_logging_config(logging.INFO)
logger = logging.getLogger('__main__')
if __name__ == "__main__":
cmd = sys.argv[1]
if cmd == 'run':
try:
payload_file = sys.argv[2]
with open(payload_file, "r") as f:
json_payload = f.read()
payload = json.loads(json_payload)
function_handler(payload)
except Exception as e:
raise(e)
elif cmd == 'metadata':
runtime_meta = dict()
mods = list(pkgutil.iter_modules())
runtime_meta["preinstalls"] = [entry for entry in sorted([[mod, is_pkg] for _, mod, is_pkg in mods])]
runtime_meta["python_ver"] = version_str(sys.version_info)
print(json.dumps(runtime_meta))
else:
raise ValueError("Command not valid: {}", cmd)
def _get_default_runtime_image_name(self):
return 'python'+version_str(sys.version_info)