def _get_default_runtime_image_name(self):
docker_user = self.knative_config['docker_user']
python_version = version_str(sys.version_info).replace('.', '')
revision = 'latest' if 'SNAPSHOT' in __version__ else __version__.replace(
'.', '')
return '{}/{}-v{}:{}'.format(docker_user, kconfig.RUNTIME_NAME_DEFAULT,
python_version, revision)
def load_config(config_data):
if 'openwhisk' not in config_data:
raise Exception("openwhisk section is mandatory in configuration")
required_keys = ('endpoint', 'namespace', 'api_key')
if not set(required_keys) <= set(config_data['openwhisk']):
raise Exception(
'You must provide {} to access to openwhisk'.format(required_keys))
if 'runtime_memory' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['cloudbutton']:
python_version = version_str(sys.version_info)
try:
config_data['cloudbutton']['runtime'] = RUNTIME_DEFAULT[
python_version]
except KeyError:
raise Exception(
'Unsupported Python version: {}'.format(python_version))
if 'workers' not in config_data['cloudbutton']:
config_data['cloudbutton']['workers'] = CONCURRENT_WORKERS_DEFAULT
if 'ibm_cos' in config_data and 'private_endpoint' in config_data[
'ibm_cos']:
del config_data['ibm_cos']['private_endpoint']
def load_config(config_data=None):
if config_data is None:
config_data = {}
if 'runtime_memory' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime'] = 'python' + \
version_str(sys.version_info)
if 'workers' not in config_data['cloudbutton']:
config_data['cloudbutton']['workers'] = MAX_CONCURRENT_WORKERS
if config_data['cloudbutton'][
'runtime_memory'] not in RUNTIME_MEMORY_OPTIONS:
raise Exception('{} MB runtime is not available (Only {} MB)'.format(
config_data['cloudbutton']['runtime_memory'],
RUNTIME_MEMORY_OPTIONS))
if config_data['cloudbutton']['runtime_memory'] > RUNTIME_MEMORY_MAX:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_MAX
if config_data['cloudbutton']['runtime_timeout'] > RUNTIME_TIMEOUT_DEFAULT:
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'gcp' not in config_data:
raise Exception("'gcp' section is mandatory in the configuration")
config_data['gcp']['retries'] = RETRIES
config_data['gcp']['retry_sleeps'] = RETRY_SLEEPS
# Put storage data into compute backend config dict entry
storage_config = dict()
storage_config['cloudbutton'] = config_data['cloudbutton'].copy()
storage_config['gcp_storage'] = config_data['gcp'].copy()
config_data['gcp']['storage'] = cloudbutton_config.extract_storage_config(
storage_config)
required_parameters_0 = ('project_name', 'service_account',
'credentials_path')
if not set(required_parameters_0) <= set(config_data['gcp']):
raise Exception(
"'project_name', 'service_account' and 'credentials_path' \
are mandatory under 'gcp' section")
if not exists(config_data['gcp']['credentials_path']) or not isfile(
config_data['gcp']['credentials_path']):
raise Exception("Path {} must be credentials JSON file.".format(
config_data['gcp']['credentials_path']))
config_data['gcp_functions'] = config_data['gcp'].copy()
if 'region' not in config_data['gcp_functions']:
config_data['gcp_functions']['region'] = config_data['pywren'][
'compute_backend_region']
def _generate_python_meta(self):
"""
Extracts installed Python modules from the local machine
"""
logger.debug("Extracting preinstalled Python modules...")
runtime_meta = dict()
mods = list(pkgutil.iter_modules())
runtime_meta["preinstalls"] = [
entry
for entry in sorted([[mod, is_pkg] for _, mod, is_pkg in mods])
]
runtime_meta["python_ver"] = version_str(sys.version_info)
return runtime_meta
def load_config(config_data=None):
if 'runtime_memory' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if config_data['cloudbutton'][
'runtime_memory'] % 64 != 0: # Adjust 64 MB memory increments restriction
mem = config_data['cloudbutton']['runtime_memory']
config_data['cloudbutton']['runtime_memory'] = (mem + (64 -
(mem % 64)))
if config_data['cloudbutton']['runtime_memory'] > RUNTIME_MEMORY_MAX:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_MAX
if 'runtime_timeout' not in config_data['cloudbutton'] or \
config_data['cloudbutton']['runtime_timeout'] > RUNTIME_TIMEOUT_DEFAULT:
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime'] = 'python' + version_str(
sys.version_info)
if 'workers' not in config_data['cloudbutton']:
config_data['cloudbutton']['workers'] = MAX_CONCURRENT_WORKERS
if 'aws' not in config_data and 'aws_lambda' not in config_data:
raise Exception(
"'aws' and 'aws_lambda' sections are mandatory in the configuration"
)
# Put credential keys to 'aws_lambda' dict entry
config_data['aws_lambda'] = {
**config_data['aws_lambda'],
**config_data['aws']
}
required_parameters_0 = ('access_key_id', 'secret_access_key')
if not set(required_parameters_0) <= set(config_data['aws']):
raise Exception(
"'access_key_id' and 'secret_access_key' are mandatory under 'aws' section"
)
if 'execution_role' not in config_data['aws_lambda']:
raise Exception(
"'execution_role' is mandatory under 'aws_lambda' section")
if 'compute_backend_region' not in config_data['cloudbutton'] \
and 'region_name' not in config_data['aws_lambda']:
raise Exception(
"'compute_backend_region' or 'region_name' not specified")
else:
config_data['aws_lambda']['region'] = config_data['aws_lambda'][
'region_name']
def load_config(config_data=None):
this_version_str = version_str(sys.version_info)
if this_version_str != '3.6':
raise Exception(
'The functions backend Azure Function Apps currently'
' only supports Python version 3.6.X and the local Python'
'version is {}'.format(this_version_str))
if 'runtime' in config_data['cloudbutton']:
print("Ignoring user specified '{}'. The current Azure compute backend"
" does not support custom runtimes.".format('runtime'))
config_data['cloudbutton']['runtime'] = RUNTIME_DEFAULT_36
if 'runtime_memory' in config_data['cloudbutton']:
print("Ignoring user specified '{}'. The current Azure compute backend"
" does not support custom runtimes.".format('runtime_memory'))
print('Default runtime memory: {}MB'.format(RUNTIME_MEMORY_DEFAULT))
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' in config_data['cloudbutton']:
print("Ignoring user specified '{}'. The current Azure compute backend"
" does not support custom runtimes.".format('runtime_timeout'))
print('Default runtime timeout: {}ms'.format(RUNTIME_TIMEOUT_DEFAULT))
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'workers' not in config_data['cloudbutton']:
config_data['cloudbutton']['workers'] = MAX_CONCURRENT_WORKERS
if 'azure_fa' not in config_data:
raise Exception("azure_fa section is mandatory in the configuration")
required_parameters = ('resource_group', 'location', 'account_name',
'account_key')
if set(required_parameters) > set(config_data['azure_fa']):
raise Exception('You must provide {} to access to Azure Function App '\
.format(required_parameters))
if 'functions_version' not in config_data['azure_fa']:
config_data['cloudbutton'][
'functions_version'] = FUNCTIONS_VERSION_DEFAULT
elif config_data['azure_fa']['functions_version'] not in (2, 3):
raise Exception('You must provide a valid Azure Functions App version {}'\
.format((2, 3)))
def load_config(config_data=None):
this_version_str = version_str(sys.version_info)
if this_version_str != '3.6':
raise Exception(
'The functions backend Aliyun Function Compute currently'
' only supports Python version 3.6.X and the local Python'
'version is {}'.format(this_version_str))
if 'runtime' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime'] = 'default'
if 'runtime_memory' in config_data['cloudbutton']:
if config_data['cloudbutton']['runtime_memory'] > RUNTIME_MEMORY_MAX:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_MAX
else:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' in config_data['cloudbutton']:
if config_data['cloudbutton']['runtime_timeout'] > RUNTIME_TIMEOUT_MAX:
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_MAX
else:
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'workers' in config_data['cloudbutton']:
if config_data['cloudbutton']['workers'] > MAX_CONCURRENT_WORKERS:
config_data['cloudbutton']['workers'] = MAX_CONCURRENT_WORKERS
else:
config_data['cloudbutton']['workers'] = MAX_CONCURRENT_WORKERS
if 'aliyun_fc' not in config_data:
raise Exception("aliyun_fc section is mandatory in the configuration")
required_parameters = ('public_endpoint', 'access_key_id',
'access_key_secret')
if set(required_parameters) > set(config_data['aliyun_fc']):
raise Exception('You must provide {} to access to Aliyun Function Compute '\
.format(required_parameters))
def load_config(config_data):
if 'knative' not in config_data:
raise Exception("knative section is mandatory in configuration")
required_keys = ('docker_user', 'docker_token')
if not set(required_keys) <= set(config_data['knative']):
raise Exception('You must provide {} to access to Knative'.format(required_keys))
if 'git_url' not in config_data['knative']:
config_data['knative']['git_url'] = BUILD_GIT_URL_DEFAULT
if 'git_rev' not in config_data['knative']:
revision = 'master' if 'SNAPSHOT' in __version__ else __version__
config_data['knative']['git_rev'] = revision
if 'cpu' not in config_data['knative']:
config_data['knative']['cpu'] = RUNTIME_CPU_DEFAULT
if 'runtime_memory' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'docker_repo' not in config_data['knative']:
config_data['knative']['docker_repo'] = DOCKER_REPO_DEFAULT
if 'runtime' not in config_data['cloudbutton']:
docker_user = config_data['knative']['docker_user']
python_version = version_str(sys.version_info).replace('.', '')
revision = 'latest' if 'SNAPSHOT' in __version__ else __version__.replace('.', '')
runtime_name = '{}/{}-v{}:{}'.format(docker_user, RUNTIME_NAME_DEFAULT, python_version, revision)
config_data['cloudbutton']['runtime'] = runtime_name
if 'workers' not in config_data['cloudbutton']:
config_data['cloudbutton']['workers'] = CONCURRENT_WORKERS_DEFAULT
if 'ibm_cos' in config_data and 'private_endpoint' in config_data['ibm_cos']:
del config_data['ibm_cos']['private_endpoint']
def _build_default_runtime(self, default_runtime_img_name):
"""
Builds the default runtime
"""
if os.system('docker --version >{} 2>&1'.format(os.devnull)) == 0:
# Build default runtime using local dokcer
python_version = version_str(sys.version_info).replace('.', '')
location = 'https://raw.githubusercontent.com/pywren/pywren-ibm-cloud/master/runtime/knative'
resp = requests.get('{}/Dockerfile.python{}'.format(
location, python_version))
dockerfile = "Dockefile.default-kantive-runtime"
if resp.status_code == 200:
with open(dockerfile, 'w') as f:
f.write(resp.text)
self.build_runtime(default_runtime_img_name, dockerfile)
os.remove(dockerfile)
else:
msg = 'There was an error fetching the default runitme Dockerfile: {}'.format(
resp.text)
logger.error(msg)
exit()
else:
# Build default runtime using Tekton
self._build_default_runtime_from_git(default_runtime_img_name)
def load_config(config_data):
if 'runtime_memory' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_memory'] = RUNTIME_MEMORY_DEFAULT
if 'runtime_timeout' not in config_data['cloudbutton']:
config_data['cloudbutton']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT
if 'runtime' not in config_data['cloudbutton']:
python_version = version_str(sys.version_info)
try:
config_data['cloudbutton']['runtime'] = RUNTIME_DEFAULT[
python_version]
except KeyError:
raise Exception(
'Unsupported Python version: {}'.format(python_version))
if 'workers' not in config_data['cloudbutton'] or \
config_data['cloudbutton']['workers'] > MAX_CONCURRENT_WORKERS:
config_data['cloudbutton']['workers'] = MAX_CONCURRENT_WORKERS
if 'ibm_cf' not in config_data:
raise Exception("ibm_cf section is mandatory in the configuration")
required_parameters_0 = ('endpoint', 'namespace')
required_parameters_1 = ('endpoint', 'namespace', 'api_key')
required_parameters_2 = ('endpoint', 'namespace', 'namespace_id',
'ibm:iam_api_key')
# Check old format. Convert to new format
if set(required_parameters_0) <= set(config_data['ibm_cf']):
endpoint = config_data['ibm_cf'].pop('endpoint')
namespace = config_data['ibm_cf'].pop('namespace')
api_key = config_data['ibm_cf'].pop('api_key', None)
namespace_id = config_data['ibm_cf'].pop('namespace_id', None)
region = endpoint.split('//')[1].split('.')[0].replace('-', '_')
for k in list(config_data['ibm_cf']):
# Delete unnecessary keys
del config_data['ibm_cf'][k]
config_data['ibm_cf']['regions'] = {}
config_data['cloudbutton']['compute_backend_region'] = region
config_data['ibm_cf']['regions'][region] = {
'endpoint': endpoint,
'namespace': namespace
}
if api_key:
config_data['ibm_cf']['regions'][region]['api_key'] = api_key
if namespace_id:
config_data['ibm_cf']['regions'][region][
'namespace_id'] = namespace_id
# -------------------
if 'ibm' in config_data and config_data['ibm'] is not None:
config_data['ibm_cf'].update(config_data['ibm'])
for region in config_data['ibm_cf']['regions']:
if not set(required_parameters_1) <= set(config_data['ibm_cf']['regions'][region]) \
and (not set(required_parameters_0) <= set(config_data['ibm_cf']['regions'][region])
or 'namespace_id' not in config_data['ibm_cf']['regions'][region] or 'iam_api_key' not in config_data['ibm_cf']):
raise Exception('You must provide {} or {} to access to IBM Cloud '
'Functions'.format(required_parameters_1,
required_parameters_2))
cbr = config_data['cloudbutton'].get('compute_backend_region')
if type(cbr) == list:
for region in cbr:
if region not in config_data['ibm_cf']['regions']:
raise Exception(
'Invalid Compute backend region: {}'.format(region))
else:
if cbr is None:
cbr = list(config_data['ibm_cf']['regions'].keys())[0]
config_data['cloudbutton']['compute_backend_region'] = cbr
if cbr not in config_data['ibm_cf']['regions']:
raise Exception('Invalid Compute backend region: {}'.format(cbr))
def _get_default_runtime_image_name(self):
python_version = version_str(sys.version_info)
return openwhisk_config.RUNTIME_DEFAULT[python_version]
def _get_default_runtime_image_name(self):
return 'python' + version_str(sys.version_info)
def select_runtime(self, job_id, runtime_memory):
"""
Auxiliary method that selects the runtime to use. To do so it gets the
runtime metadata from the storage. This metadata contains the preinstalled
python modules needed to serialize the local function. If the .metadata
file does not exists in the storage, this means that the runtime is not
installed, so this method will proceed to install it.
"""
log_level = os.getenv('CLOUDBUTTON_LOGLEVEL')
runtime_name = self.config['cloudbutton']['runtime']
if runtime_memory is None:
runtime_memory = self.config['cloudbutton']['runtime_memory']
if runtime_memory:
runtime_memory = int(runtime_memory)
log_msg = ('ExecutorID {} | JobID {} - Selected Runtime: {} - {}MB'
.format(self.executor_id, job_id, runtime_name,
runtime_memory))
else:
log_msg = (
'ExecutorID {} | JobID {} - Selected Runtime: {}'.format(
self.executor_id, job_id, runtime_name))
print(log_msg, end=' ') if not self.log_level else logger.info(log_msg)
installing = False
for compute_handler in self.compute_handlers:
runtime_key = compute_handler.get_runtime_key(
runtime_name, runtime_memory)
runtime_deployed = True
try:
runtime_meta = self.internal_storage.get_runtime_meta(
runtime_key)
except Exception:
runtime_deployed = False
if not runtime_deployed:
logger.debug(
'ExecutorID {} | JobID {} - Runtime {} with {}MB is not yet '
'installed'.format(self.executor_id, job_id, runtime_name,
runtime_memory))
if not log_level and not installing:
installing = True
print('(Installing...)')
timeout = self.config['cloudbutton']['runtime_timeout']
logger.debug('Creating runtime: {}, memory: {}MB'.format(
runtime_name, runtime_memory))
runtime_meta = compute_handler.create_runtime(runtime_name,
runtime_memory,
timeout=timeout)
self.internal_storage.put_runtime_meta(runtime_key,
runtime_meta)
py_local_version = version_str(sys.version_info)
py_remote_version = runtime_meta['python_ver']
if py_local_version != py_remote_version:
raise Exception(
("The indicated runtime '{}' is running Python {} and it "
"is not compatible with the local Python version {}"
).format(runtime_name, py_remote_version,
py_local_version))
if not log_level and runtime_deployed:
print()
return runtime_meta
def _build_default_runtime_from_git(self, docker_image_name):
"""
Builds the default runtime and pushes it to the docker container registry
"""
image_name, revision = docker_image_name.split(':')
if self.knative_config[
'docker_repo'] == 'docker.io' and revision != 'latest':
resp = requests.get(
'https://index.docker.io/v1/repositories/{}/tags/{}'.format(
docker_image_name, revision))
if resp.status_code == 200:
logger.debug(
'Docker image docker.io/{}:{} already exists in Dockerhub. '
'Skipping build process.'.format(docker_image_name,
revision))
return
logger.debug("Building default PyWren runtime from git with Tekton")
if not {"docker_user", "docker_token"} <= set(self.knative_config):
raise Exception("You must provide 'docker_user' and 'docker_token'"
" to build the default runtime")
task_run = yaml.safe_load(kconfig.task_run)
task_run['spec']['inputs']['params'] = []
python_version = version_str(sys.version_info).replace('.', '')
path_to_dockerfile = {
'name':
'pathToDockerFile',
'value':
'pywren_ibm_cloud/compute/backends/knative/tekton/Dockerfile.python{}'
.format(python_version)
}
task_run['spec']['inputs']['params'].append(path_to_dockerfile)
image_url = {
'name': 'imageUrl',
'value': '/'.join([self.knative_config['docker_repo'], image_name])
}
task_run['spec']['inputs']['params'].append(image_url)
image_tag = {'name': 'imageTag', 'value': revision}
task_run['spec']['inputs']['params'].append(image_tag)
self._create_account_resources()
self._create_build_resources()
task_run_name = task_run['metadata']['name']
try:
self.api.delete_namespaced_custom_object(
group="tekton.dev",
version="v1alpha1",
name=task_run_name,
namespace=self.namespace,
plural="taskruns",
body=client.V1DeleteOptions())
except Exception:
pass
self.api.create_namespaced_custom_object(group="tekton.dev",
version="v1alpha1",
namespace=self.namespace,
plural="taskruns",
body=task_run)
logger.debug("Building runtime...")
pod_name = None
w = watch.Watch()
for event in w.stream(
self.api.list_namespaced_custom_object,
namespace=self.namespace,
group="tekton.dev",
version="v1alpha1",
plural="taskruns",
field_selector="metadata.name={0}".format(task_run_name)):
if event['object'].get('status'):
pod_name = event['object']['status']['podName']
w.stop()
if pod_name is None:
raise Exception(
'Unable to get the pod name from the task that is building the runtime'
)
w = watch.Watch()
for event in w.stream(
self.v1.list_namespaced_pod,
namespace=self.namespace,
field_selector="metadata.name={0}".format(pod_name)):
if event['object'].status.phase == "Succeeded":
w.stop()
if event['object'].status.phase == "Failed":
w.stop()
logger.debug(
'Something went wrong building the default PyWren runtime with Tekton'
)
for container in event['object'].status.container_statuses:
if container.state.terminated.reason == 'Error':
logs = self.v1.read_namespaced_pod_log(
name=pod_name,
container=container.name,
namespace=self.namespace)
logger.debug("Tekton container '{}' failed: {}".format(
container.name, logs.strip()))
raise Exception(
'Unable to build the default PyWren runtime with Tekton')
self.api.delete_namespaced_custom_object(group="tekton.dev",
version="v1alpha1",
name=task_run_name,
namespace=self.namespace,
plural="taskruns",
body=client.V1DeleteOptions())
logger.debug(
'Default PyWren runtime built from git and uploaded to Dockerhub')