def get_runtime(id, session=None):
model = models.Runtime
filters = sa.and_(
model.id == id,
sa.or_(model.project_id == context.get_ctx().projectid,
model.is_public),
)
runtime = db_base.model_query(model).filter(filters).first()
if not runtime:
raise exc.DBEntityNotFoundError("Runtime not found [id=%s]" % id)
return runtime
def put(self, id, runtime):
"""Update runtime.
Currently, we only support update name, description, image. When
updating image, send message to engine for asynchronous handling.
"""
acl.enforce('runtime:update', context.get_ctx())
values = {}
for key in UPDATE_ALLOWED:
if runtime.to_dict().get(key) is not None:
values.update({key: runtime.to_dict()[key]})
LOG.info('Update resource, params: %s',
values,
resource={
'type': self.type,
'id': id
})
with db_api.transaction():
if 'image' in values:
pre_runtime = db_api.get_runtime(id)
if pre_runtime.status != status.AVAILABLE:
raise exc.RuntimeNotAvailableException(
'Runtime %s is not available.' % id)
pre_image = pre_runtime.image
if pre_image != values['image']:
# Ensure there is no function running in the runtime.
db_funcs = db_api.get_functions(insecure=True,
fields=['id'],
runtime_id=id)
func_ids = [func.id for func in db_funcs]
for id in func_ids:
if etcd_util.get_service_url(id):
raise exc.NotAllowedException(
'Runtime %s is still in use by functions.' %
id)
values['status'] = status.UPGRADING
self.engine_client.update_runtime(id,
image=values['image'],
pre_image=pre_image)
else:
values.pop('image')
runtime_db = db_api.update_runtime(id, values)
return resources.Runtime.from_db_obj(runtime_db)
def get_keystone_client(use_session=True):
if use_session:
session = _get_user_keystone_session()
keystone = ks_client.Client(session=session)
else:
ctx = context.get_ctx()
auth_url = CONF.keystone_authtoken.auth_uri
keystone = ks_client.Client(user_id=ctx.user,
token=ctx.auth_token,
tenant_id=ctx.projectid,
auth_url=auth_url)
keystone.management_url = auth_url
return keystone
def post(self, body):
ctx = context.get_ctx()
acl.enforce('execution:create', ctx)
params = body.to_dict()
if not (params.get("function_id") or params.get("function_alias")):
raise exc.InputException(
'Either function_alias or function_id must be provided.')
LOG.info("Creating %s. [params=%s]", self.type, params)
db_model = executions.create_execution(self.engine_client, params)
return resources.Execution.from_db_obj(db_model)
def detach(self, id):
"""Detach the function from its underlying workers.
This is admin only operation, which gives admin user a safe way to
clean up the underlying resources allocated for the function.
"""
acl.enforce('function:detach', context.get_ctx())
db_api.get_function(id)
LOG.info('Starting to detach function %s', id)
# Delete allocated resources in orchestrator and etcd keys.
self.engine_client.delete_function(id)
etcd_util.delete_function(id)
def delete(self, function_id, version):
"""Delete a specific function version.
- The version should not being used by any job
- The version should not being used by any webhook
- Admin user can not delete normal user's version
"""
ctx = context.get_ctx()
acl.enforce('function_version:delete', ctx)
LOG.info("Deleting version %s of function %s.", version, function_id)
with db_api.transaction():
version_db = db_api.get_function_version(function_id, version,
insecure=False)
latest_version = version_db.function.latest_version
version_jobs = db_api.get_jobs(
function_id=version_db.function_id,
function_version=version_db.version_number,
status={'nin': ['done', 'cancelled']}
)
if len(version_jobs) > 0:
raise exc.NotAllowedException(
'The function version is still associated with running '
'job(s).'
)
version_webhook = db_api.get_webhooks(
function_id=version_db.function_id,
function_version=version_db.version_number,
)
if len(version_webhook) > 0:
raise exc.NotAllowedException(
'The function versioin is still associated with webhook.'
)
# Delete resources for function version
self.engine_client.delete_function(function_id, version=version)
etcd_util.delete_function(function_id, version=version)
self.storage_provider.delete(ctx.projectid, function_id, None,
version=version)
db_api.delete_function_version(function_id, version)
if latest_version == version:
version_db.function.latest_version = latest_version - 1
LOG.info("Version %s of function %s deleted.", version, function_id)
def scale_up(self, id, scale):
"""Scale up the containers for function execution.
This is admin only operation. The load monitoring of function execution
depends on the monitoring solution of underlying orchestrator.
"""
acl.enforce('function:scale_up', context.get_ctx())
func_db = db_api.get_function(id)
params = scale.to_dict()
LOG.info('Starting to scale up function %s, params: %s', id, params)
self.engine_client.scaleup_function(id,
runtime_id=func_db.runtime_id,
count=params['count'])
def get_all(self, function_id, function_version=0):
acl.enforce('function_worker:get_all', context.get_ctx())
LOG.info("Getting workers for function %s(version %s).", function_id,
function_version)
workers = etcd_util.get_workers(function_id, version=function_version)
workers = [
resources.FunctionWorker.from_dict({
'function_id': function_id,
'function_version': function_version,
'worker_name': w
}) for w in workers
]
return resources.FunctionWorkers(workers=workers)
def post(self, webhook):
acl.enforce('webhook:create', context.get_ctx())
params = webhook.to_dict()
if not POST_REQUIRED.issubset(set(params.keys())):
raise exc.InputException(
'Required param is missing. Required: %s' % POST_REQUIRED)
LOG.info("Creating %s, params: %s", self.type, params)
# Even admin user can not expose normal user's function
db_api.get_function(params['function_id'], insecure=False)
webhook_d = db_api.create_webhook(params).to_dict()
return resources.Webhook.from_dict(
self._add_webhook_url(webhook_d['id'], webhook_d))
def get_all(self, all_projects=False, project_id=None):
project_id, all_projects = rest_utils.get_project_params(
project_id, all_projects)
if all_projects:
acl.enforce('webhook:get_all:all_projects', context.get_ctx())
filters = rest_utils.get_filters(project_id=project_id, )
LOG.info("Get all %ss. filters=%s", self.type, filters)
webhooks = []
for i in db_api.get_webhooks(insecure=all_projects, **filters):
webhooks.append(
resources.Webhook.from_dict(
self._add_webhook_url(i.id, i.to_dict())))
return resources.Webhooks(webhooks=webhooks)
def scale_down(self, id, scale):
"""Scale down the containers for function execution.
This is admin only operation. The load monitoring of function execution
depends on the monitoring solution of underlying orchestrator.
"""
acl.enforce('function:scale_down', context.get_ctx())
db_api.get_function(id)
workers = etcd_util.get_workers(id)
params = scale.to_dict()
if len(workers) <= 1:
LOG.info('No need to scale down function %s', id)
return
LOG.info('Starting to scale down function %s, params: %s', id, params)
self.engine_client.scaledown_function(id, count=params['count'])
def post(self, runtime):
acl.enforce('runtime:create', context.get_ctx())
params = runtime.to_dict()
if not POST_REQUIRED.issubset(set(params.keys())):
raise exc.InputException(
'Required param is missing. Required: %s' % POST_REQUIRED)
LOG.info("Creating %s, params: %s", self.type, params)
params.update({'status': status.CREATING})
db_model = db_api.create_runtime(params)
self.engine_client.create_runtime(db_model.id)
return resources.Runtime.from_db_obj(db_model)
def get_all(self, function_id):
"""Get all the versions of the given function.
Admin user can get all versions for the normal user's function.
"""
acl.enforce('function_version:get_all', context.get_ctx())
LOG.info("Getting versions for function %s.", function_id)
# Getting function and versions needs to happen in a db transaction
with db_api.transaction():
func_db = db_api.get_function(function_id)
db_versions = func_db.versions
versions = [resources.FunctionVersion.from_db_obj(v)
for v in db_versions]
return resources.FunctionVersions(function_versions=versions)
def delete(self, id):
acl.enforce('runtime:delete', context.get_ctx())
LOG.info("Delete resource.", resource={'type': self.type, 'id': id})
with db_api.transaction():
runtime_db = db_api.get_runtime(id)
# Runtime can not be deleted if still associate with functions.
funcs = db_api.get_functions(insecure=True, runtime_id={'eq': id})
if len(funcs):
raise exc.NotAllowedException('Runtime %s is still in use.' %
id)
runtime_db.status = status.DELETING
# Clean related resources asynchronously
self.engine_client.delete_runtime(id)
def detach(self, function_id, version):
"""Detach the function version from its underlying workers.
This is admin only operation, which gives admin user a safe way to
clean up the underlying resources allocated for the function version.
"""
acl.enforce('function_version:detach', context.get_ctx())
db_api.get_function(function_id)
# If version=0, it's equivalent to /functions/<funcion-id>/detach
if version > 0:
db_api.get_function_version(function_id, version)
LOG.info('Starting to detach function %s(version %s)', function_id,
version)
# Delete allocated resources in orchestrator and etcd keys.
self.engine_client.delete_function(function_id, version=version)
etcd_util.delete_function(function_id, version=version)
def delete(self, id):
"""Delete the specified function."""
LOG.info("Delete resource.", resource={'type': self.type, 'id': id})
with db_api.transaction():
func_db = db_api.get_function(id)
if len(func_db.jobs) > 0:
raise exc.NotAllowedException(
'The function is still associated with running job(s).')
source = func_db.code['source']
if source == 'package':
self.storage_provider.delete(context.get_ctx().projectid, id)
# Delete all resources created by orchestrator asynchronously.
self.engine_client.delete_function(id)
# This will also delete function service mapping as well.
db_api.delete_function(id)
def pool(self, id):
"""Get the pool information for the runtime.
This operation should be admin only.
We don't check the runtime existence, because this function
also helps us to check the underlying pool even after the runtime
is already deleted.
"""
acl.enforce('runtime_pool:get_all', context.get_ctx())
LOG.info("Getting pool information for runtime %s.", id)
capacity = self.engine_client.get_runtime_pool(id)
pool_capacity = resources.RuntimePoolCapacity.from_dict(capacity)
return resources.RuntimePool.from_dict({
"name": id,
"capacity": pool_capacity
})
def get(self, id):
"""Get function information or download function package.
This method can support HTTP request using either
'Accept:application/json' or no 'Accept' header.
"""
ctx = context.get_ctx()
acl.enforce('function:get', ctx)
download = strutils.bool_from_string(
pecan.request.GET.get('download', False)
)
func_db = db_api.get_function(id)
if not download:
LOG.info("Getting function %s.", id)
pecan.override_template('json')
return resources.Function.from_db_obj(func_db).to_dict()
LOG.info("Downloading function %s", id)
source = func_db.code['source']
if source == constants.PACKAGE_FUNCTION:
f = self.storage_provider.retrieve(func_db.project_id, id,
func_db.code['md5sum'])
elif source == constants.SWIFT_FUNCTION:
container = func_db.code['swift']['container']
obj = func_db.code['swift']['object']
f = swift_util.download_object(container, obj)
else:
msg = 'Download image function is not allowed.'
pecan.abort(
status_code=405,
detail=msg,
headers={'Server-Error-Message': msg}
)
pecan.response.app_iter = (f if isinstance(f, collections.Iterable)
else FileIter(f))
pecan.response.headers['Content-Type'] = 'application/zip'
pecan.response.headers['Content-Disposition'] = (
'attachment; filename="%s"' % id
)
def put(self, id, webhook):
"""Update webhook.
Currently, we only support update function_id.
"""
acl.enforce('webhook:update', context.get_ctx())
values = {}
for key in UPDATE_ALLOWED:
if webhook.to_dict().get(key) is not None:
values.update({key: webhook.to_dict()[key]})
LOG.info('Update %s %s, params: %s', self.type, id, values)
if 'function_id' in values:
# Even admin user can not expose normal user's function
db_api.get_function(values['function_id'], insecure=False)
webhook = db_api.update_webhook(id, values).to_dict()
return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
def get_request_data(conf, function_id, version, execution_id, rlimit, input,
entry, trust_id, qinling_endpoint):
"""Prepare the request body should send to the worker."""
ctx = context.get_ctx()
if version == 0:
download_url = (
'%s/%s/functions/%s?download=true' %
(qinling_endpoint.strip('/'), constants.CURRENT_VERSION,
function_id)
)
else:
download_url = (
'%s/%s/functions/%s/versions/%s?download=true' %
(qinling_endpoint.strip('/'), constants.CURRENT_VERSION,
function_id, version)
)
data = {
'execution_id': execution_id,
'cpu': rlimit['cpu'],
'memory_size': rlimit['memory_size'],
'input': input,
'function_id': function_id,
'function_version': version,
'entry': entry,
'download_url': download_url,
'request_id': ctx.request_id,
}
if conf.pecan.auth_enable:
data.update(
{
'token': ctx.auth_token,
'auth_url': conf.keystone_authtoken.www_authenticate_uri,
'username': conf.keystone_authtoken.username,
'password': conf.keystone_authtoken.password,
'trust_id': trust_id
}
)
return data
def get(self, id):
LOG.info("Fetch function [id=%s]", id)
download = strutils.bool_from_string(
pecan.request.GET.get('download', False))
func_db = db_api.get_function(id)
ctx = context.get_ctx()
if not download:
pecan.override_template('json')
return resources.Function.from_dict(func_db.to_dict()).to_dict()
else:
f = self.storage_provider.retrieve(
ctx.projectid,
id,
)
pecan.response.app_iter = FileIter(f)
pecan.response.headers['Content-Type'] = 'application/zip'
pecan.response.headers['Content-Disposition'] = (
'attachment; filename="%s"' % os.path.basename(f.name))
def post(self, function_id, body):
"""Create a new version for the function.
Only allow to create version for package type function.
The supported boy params:
- description: Optional. The description of the new version.
"""
ctx = context.get_ctx()
acl.enforce('function_version:create', ctx)
params = body.to_dict()
values = {
'description': params.get('description'),
}
# Try to create a new function version within lock and db transaction
version = self._create_function_version(ctx.project_id, function_id,
**values)
return resources.FunctionVersion.from_db_obj(version)
def get_all(self, all_projects=False, project_id=None):
"""Return a list of functions.
:param project_id: Optional. Admin user can query other projects
resources, the param is ignored for normal user.
:param all_projects: Optional. Get resources of all projects.
"""
project_id, all_projects = rest_utils.get_project_params(
project_id, all_projects)
if all_projects:
acl.enforce('function:get_all:all_projects', context.get_ctx())
filters = rest_utils.get_filters(project_id=project_id, )
LOG.info("Get all functions. filters=%s", filters)
db_functions = db_api.get_functions(insecure=all_projects, **filters)
functions = [
resources.Function.from_db_obj(db_model)
for db_model in db_functions
]
return resources.Functions(functions=functions)
def put(self, id, webhook):
acl.enforce('webhook:update', context.get_ctx())
values = {}
for key in UPDATE_ALLOWED:
if webhook.to_dict().get(key) is not None:
values.update({key: webhook.to_dict()[key]})
LOG.info('Update %s %s, params: %s', self.type, id, values)
# Even admin user can not expose normal user's function
webhook_db = db_api.get_webhook(id, insecure=False)
pre_alias = webhook_db.function_alias
pre_function_id = webhook_db.function_id
pre_version = webhook_db.function_version
new_alias = values.get("function_alias")
new_function_id = values.get("function_id", pre_function_id)
new_version = values.get("function_version", pre_version)
function_id = pre_function_id
version = pre_version
if new_alias and new_alias != pre_alias:
alias_db = db_api.get_function_alias(new_alias)
function_id = alias_db.function_id
version = alias_db.function_version
# If function_alias is provided, we don't store either functin id
# or function version.
values.update({'function_id': None, 'function_version': None})
elif new_function_id != pre_function_id or new_version != pre_version:
function_id = new_function_id
version = new_version
values.update({"function_alias": None})
db_api.get_function(function_id, insecure=False)
if version and version > 0:
db_api.get_function_version(function_id, version)
webhook = db_api.update_webhook(id, values).to_dict()
return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
def put(self, alias_name, body):
"""Update alias for the specified function.
The supported body params:
- function_id: Optional. Function id the alias point to.
- function_version: Optional. Version number the alias point to.
- description: Optional. The description of the alias.
"""
ctx = context.get_ctx()
acl.enforce('function_alias:update', ctx)
params = body.to_dict()
values = {}
for key in UPDATE_ALLOWED:
if params.get(key) is not None:
values.update({key: params[key]})
LOG.info("Updating Alias %s, params: %s", alias_name, values)
alias = db_api.update_function_alias(alias_name, **values)
LOG.info("Alias %s updated.", alias_name)
return resources.FunctionAlias.from_db_obj(alias)
def scale_up(self, function_id, version, scale):
"""Scale up the workers for function version execution.
This is admin only operation. The load monitoring of execution
depends on the monitoring solution of underlying orchestrator.
"""
acl.enforce('function_version:scale_up', context.get_ctx())
func_db = db_api.get_function(function_id)
# If version=0, it's equivalent to /functions/<funcion-id>/scale_up
if version > 0:
db_api.get_function_version(function_id, version)
params = scale.to_dict()
LOG.info('Starting to scale up function %s(version %s), params: %s',
function_id, version, params)
self.engine_client.scaleup_function(function_id,
runtime_id=func_db.runtime_id,
version=version,
count=params['count'])
def get(self, function_id, version):
"""Get function version or download function version package.
This method can support HTTP request using either
'Accept:application/json' or no 'Accept' header.
"""
ctx = context.get_ctx()
acl.enforce('function_version:get', ctx)
download = strutils.bool_from_string(
pecan.request.GET.get('download', False)
)
version = int(version)
version_db = db_api.get_function_version(function_id, version)
if not download:
LOG.info("Getting version %s for function %s.", version,
function_id)
pecan.override_template('json')
return resources.FunctionVersion.from_db_obj(version_db).to_dict()
LOG.info("Downloading version %s for function %s.", version,
function_id)
f = self.storage_provider.retrieve(version_db.project_id, function_id,
None, version=version)
if isinstance(f, collections.Iterable):
pecan.response.app_iter = f
else:
pecan.response.app_iter = FileIter(f)
pecan.response.headers['Content-Type'] = 'application/zip'
pecan.response.headers['Content-Disposition'] = (
'attachment; filename="%s_%s"' % (function_id, version)
)
def _prepare_pod(self,
pod,
deployment_name,
function_id,
labels=None,
entry=None,
actual_function=None):
"""Pod preparation.
1. Update pod labels.
2. Expose service and trigger package download.
"""
name = pod.metadata.name
actual_function = actual_function or function_id
LOG.info('Prepare pod %s in deployment %s for function %s', name,
deployment_name, function_id)
# Update pod label.
pod_labels = self._update_pod_label(pod, {'function_id': function_id})
# Create service for the chosen pod.
service_name = "service-%s" % function_id
labels.update({'function_id': function_id})
service_body = self.service_template.render({
"service_name": service_name,
"labels": labels,
"selector": pod_labels
})
ret = self.v1.create_namespaced_service(self.conf.kubernetes.namespace,
yaml.safe_load(service_body))
node_port = ret.spec.ports[0].node_port
LOG.debug(
'Service created for pod %s, service name: %s, node port: %s',
name, service_name, node_port)
# Get external ip address for an arbitrary node.
ret = self.v1.list_node()
addresses = ret.items[0].status.addresses
node_ip = None
for addr in addresses:
if addr.type == 'ExternalIP':
node_ip = addr.address
# FIXME: test purpose using minikube
if not node_ip:
for addr in addresses:
if addr.type == 'InternalIP':
node_ip = addr.address
# Download code package into container.
pod_service_url = 'http://%s:%s' % (node_ip, node_port)
request_url = '%s/download' % pod_service_url
download_url = ('http://%s:%s/v1/functions/%s?download=true' %
(self.conf.kubernetes.qinling_service_address,
self.conf.api.port, actual_function))
data = {
'download_url': download_url,
'function_id': actual_function,
'entry': entry,
'token': context.get_ctx().auth_token,
}
LOG.debug('Send request to pod %s, request_url: %s, data: %s', name,
request_url, data)
# TODO(kong): Here we sleep some time to avoid 'Failed to establish a
# new connection' error for some reason. Needs to find a better
# solution.
time.sleep(1)
r = requests.post(request_url, json=data)
if r.status_code != requests.codes.ok:
raise exc.OrchestratorException(
'Failed to download function code package.')
return name, pod_service_url
def get_project_id():
return context.get_ctx().projectid
def delete_function(self, id, version=0):
return self._client.prepare(topic=self.topic,
server=None).cast(ctx.get_ctx(),
'delete_function',
function_id=id,
function_version=version)