def put(self, id, webhook):
acl.enforce('webhook:update', context.get_ctx())
values = {}
for key in UPDATE_ALLOWED:
if webhook.to_dict().get(key) is not None:
values.update({key: webhook.to_dict()[key]})
LOG.info('Update %s %s, params: %s', self.type, id, values)
# Even admin user can not expose normal user's function
webhook_db = db_api.get_webhook(id, insecure=False)
pre_alias = webhook_db.function_alias
pre_function_id = webhook_db.function_id
pre_version = webhook_db.function_version
new_alias = values.get("function_alias")
new_function_id = values.get("function_id", pre_function_id)
new_version = values.get("function_version", pre_version)
function_id = pre_function_id
version = pre_version
if new_alias and new_alias != pre_alias:
alias_db = db_api.get_function_alias(new_alias)
function_id = alias_db.function_id
version = alias_db.function_version
# If function_alias is provided, we don't store either functin id
# or function version.
values.update({'function_id': None, 'function_version': None})
elif new_function_id != pre_function_id or new_version != pre_version:
function_id = new_function_id
version = new_version
values.update({"function_alias": None})
db_api.get_function(function_id, insecure=False)
if version and version > 0:
db_api.get_function_version(function_id, version)
webhook = db_api.update_webhook(id, values).to_dict()
return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
def scaledown_function(self, ctx, function_id, count=1):
func_db = db_api.get_function(function_id)
worker_deleted_num = (count if len(func_db.workers) > count else
len(func_db.workers) - 1)
workers = func_db.workers[:worker_deleted_num]
with db_api.transaction():
for worker in workers:
LOG.debug('Removing worker %s', worker.worker_name)
self.orchestrator.delete_worker(worker.worker_name, )
db_api.delete_function_worker(worker.worker_name)
LOG.info('Finished scaling up function %s.', function_id)
def post(self, job):
"""Creates a new job."""
params = job.to_dict()
if not (params.get("function_id") or params.get("function_alias")):
raise exc.InputException(
'Either function_alias or function_id must be provided.')
# Check the input params.
first_time, next_time, count = jobs.validate_job(params)
LOG.info("Creating %s, params: %s", self.type, params)
version = params.get('function_version', 0)
# if function_alias provided
function_alias = params.get('function_alias')
if function_alias:
alias_db = db_api.get_function_alias(function_alias)
function_id = alias_db.function_id
version = alias_db.function_version
params.update({'function_id': function_id, 'version': version})
with db_api.transaction():
db_api.get_function(params['function_id'])
if version > 0:
db_api.get_function_version(params['function_id'], version)
values = {
'name': params.get('name'),
'pattern': params.get('pattern'),
'first_execution_time': first_time,
'next_execution_time': next_time,
'count': count,
'function_id': params['function_id'],
'function_version': version,
'function_input': params.get('function_input'),
'status': status.RUNNING
}
db_job = db_api.create_job(values)
return resources.Job.from_db_obj(db_job)
def test_job_handler(self, mock_get_next):
db_func = self.create_function()
function_id = db_func.id
self.assertEqual(0, db_func.count)
now = datetime.utcnow()
db_job = self.create_job(
function_id,
status=status.RUNNING,
next_execution_time=now,
count=2
)
job_id = db_job.id
e_client = mock.Mock()
mock_get_next.return_value = now + timedelta(seconds=1)
periodics.handle_job(e_client)
context.set_ctx(self.ctx)
db_job = db_api.get_job(job_id)
self.assertEqual(1, db_job.count)
db_func = db_api.get_function(function_id)
self.assertEqual(1, db_func.count)
db_execs = db_api.get_executions(function_id=function_id)
self.assertEqual(1, len(db_execs))
periodics.handle_job(e_client)
context.set_ctx(self.ctx)
db_job = db_api.get_job(job_id)
self.assertEqual(0, db_job.count)
self.assertEqual(status.DONE, db_job.status)
db_func = db_api.get_function(function_id)
self.assertEqual(2, db_func.count)
db_execs = db_api.get_executions(function_id=function_id)
self.assertEqual(2, len(db_execs))
def post(self, webhook):
acl.enforce('webhook:create', context.get_ctx())
params = webhook.to_dict()
if not POST_REQUIRED.issubset(set(params.keys())):
raise exc.InputException(
'Required param is missing. Required: %s' % POST_REQUIRED
)
LOG.info("Creating %s, params: %s", self.type, params)
# Even admin user can not expose normal user's function
db_api.get_function(params['function_id'], insecure=False)
version = params.get('function_version', 0)
if version > 0:
db_api.get_function_version(params['function_id'], version)
webhook_d = db_api.create_webhook(params).to_dict()
return resources.Webhook.from_dict(
self._add_webhook_url(webhook_d['id'], webhook_d)
)
def scale_up(self, id, scale):
"""Scale up the containers for function execution.
This is admin only operation. The load monitoring of function execution
depends on the monitoring solution of underlying orchestrator.
"""
func_db = db_api.get_function(id)
params = scale.to_dict()
LOG.info('Starting to scale up function %s, params: %s', id, params)
self.engine_client.scaleup_function(id,
runtime_id=func_db.runtime_id,
count=params['count'])
def _create_function_version(self, project_id, function_id, **kwargs):
with etcd_util.get_function_version_lock(function_id) as lock:
if not lock.is_acquired():
return False
with db_api.transaction():
# Get latest function package md5 and version number
func_db = db_api.get_function(function_id, insecure=False)
if func_db.code['source'] != constants.PACKAGE_FUNCTION:
raise exc.NotAllowedException(
"Function versioning only allowed for %s type "
"function." %
constants.PACKAGE_FUNCTION
)
l_md5 = func_db.code['md5sum']
l_version = func_db.latest_version
if len(func_db.versions) >= constants.MAX_VERSION_NUMBER:
raise exc.NotAllowedException(
'Can not exceed maximum number(%s) of versions' %
constants.MAX_VERSION_NUMBER
)
# Check if the latest package changed since last version
changed = self.storage_provider.changed_since(project_id,
function_id,
l_md5,
l_version)
if not changed:
raise exc.NotAllowedException(
'Function package not changed since the latest '
'version %s.' % l_version
)
LOG.info("Creating %s, function_id: %s, old_version: %d",
self.type, function_id, l_version)
# Create new version and copy package.
self.storage_provider.copy(project_id, function_id, l_md5,
l_version)
version = db_api.increase_function_version(function_id,
l_version,
**kwargs)
func_db.latest_version = l_version + 1
LOG.info("New version %d for function %s created.", l_version + 1,
function_id)
return version
def post(self, job):
"""Creates a new job."""
params = job.to_dict()
if not POST_REQUIRED.issubset(set(params.keys())):
raise exc.InputException(
'Required param is missing. Required: %s' % POST_REQUIRED)
# Check the input params.
first_time, next_time, count = jobs.validate_job(params)
LOG.info("Creating %s, params: %s", self.type, params)
with db_api.transaction():
db_api.get_function(params['function_id'])
values = {
'name': params.get('name'),
'pattern': params.get('pattern'),
'first_execution_time': first_time,
'next_execution_time': next_time,
'count': count,
'function_id': params['function_id'],
'function_input': params.get('function_input') or {},
'status': status.RUNNING
}
if cfg.CONF.pecan.auth_enable:
values['trust_id'] = keystone_util.create_trust().id
try:
db_job = db_api.create_job(values)
except Exception:
# Delete trust before raising exception.
keystone_util.delete_trust(values.get('trust_id'))
raise
return resources.Job.from_dict(db_job.to_dict())
def scale_down(self, id, scale):
"""Scale down the containers for function execution.
This is admin only operation. The load monitoring of function execution
depends on the monitoring solution of underlying orchestrator.
"""
func_db = db_api.get_function(id)
params = scale.to_dict()
if len(func_db.workers) <= 1:
LOG.info('No need to scale down function %s', id)
return
LOG.info('Starting to scale down function %s, params: %s', id, params)
self.engine_client.scaledown_function(id, count=params['count'])
def scaleup_function(self, ctx, function_id, runtime_id, count=1):
function = db_api.get_function(function_id)
worker_names = self.orchestrator.scaleup_function(
function_id,
identifier=runtime_id,
entry=function.entry,
count=count)
with db_api.transaction():
for name in worker_names:
worker = {'function_id': function_id, 'worker_name': name}
db_api.create_function_worker(worker)
LOG.info('Finished scaling up function %s.', function_id)
def get_all(self, function_id):
"""Get all the versions of the given function.
Admin user can get all versions for the normal user's function.
"""
acl.enforce('function_version:get_all', context.get_ctx())
LOG.info("Getting versions for function %s.", function_id)
# Getting function and versions needs to happen in a db transaction
with db_api.transaction():
func_db = db_api.get_function(function_id)
db_versions = func_db.versions
versions = [resources.FunctionVersion.from_db_obj(v)
for v in db_versions]
return resources.FunctionVersions(function_versions=versions)
def delete(self, id):
"""Delete the specified function."""
LOG.info("Delete resource.", resource={'type': self.type, 'id': id})
with db_api.transaction():
func_db = db_api.get_function(id)
if len(func_db.jobs) > 0:
raise exc.NotAllowedException(
'The function is still associated with running job(s).')
source = func_db.code['source']
if source == 'package':
self.storage_provider.delete(context.get_ctx().projectid, id)
# Delete all resources created by orchestrator asynchronously.
self.engine_client.delete_function(id)
# This will also delete function service mapping as well.
db_api.delete_function(id)
def get(self, id):
"""Get function information or download function package.
This method can support HTTP request using either
'Accept:application/json' or no 'Accept' header.
"""
ctx = context.get_ctx()
acl.enforce('function:get', ctx)
download = strutils.bool_from_string(
pecan.request.GET.get('download', False)
)
func_db = db_api.get_function(id)
if not download:
LOG.info("Getting function %s.", id)
pecan.override_template('json')
return resources.Function.from_db_obj(func_db).to_dict()
LOG.info("Downloading function %s", id)
source = func_db.code['source']
if source == constants.PACKAGE_FUNCTION:
f = self.storage_provider.retrieve(func_db.project_id, id,
func_db.code['md5sum'])
elif source == constants.SWIFT_FUNCTION:
container = func_db.code['swift']['container']
obj = func_db.code['swift']['object']
f = swift_util.download_object(container, obj)
else:
msg = 'Download image function is not allowed.'
pecan.abort(
status_code=405,
detail=msg,
headers={'Server-Error-Message': msg}
)
pecan.response.app_iter = (f if isinstance(f, collections.Iterable)
else FileIter(f))
pecan.response.headers['Content-Type'] = 'application/zip'
pecan.response.headers['Content-Disposition'] = (
'attachment; filename="%s"' % id
)
def post(self, execution):
params = execution.to_dict()
LOG.info("Creating execution. [execution=%s]", params)
function_id = params['function_id']
# Check if the service url is existing.
try:
mapping = db_api.get_function_service_mapping(function_id)
LOG.debug('Found Service url for function: %s', function_id)
func_url = '%s/execute' % mapping.service_url
LOG.info('Invoke function %s, url: %s', function_id, func_url)
r = requests.post(func_url, data=params.get('input'))
params.update({
'status': 'success',
'output': {
'result': r.json()
}
})
db_model = db_api.create_execution(params)
return resources.Execution.from_dict(db_model.to_dict())
except exc.DBEntityNotFoundError:
pass
func = db_api.get_function(function_id)
runtime_id = func.runtime_id
params.update({'status': 'running'})
db_model = db_api.create_execution(params)
self.engine_client.create_execution(db_model.id,
function_id,
runtime_id,
input=params.get('input'))
updated_db = db_api.get_execution(db_model.id)
return resources.Execution.from_dict(updated_db.to_dict())
def test_delete(self, mock_package_delete, mock_engine_delete,
mock_etcd_delete):
db_api.increase_function_version(self.func_id,
0,
description="version 1")
resp = self.app.delete('/v1/functions/%s/versions/1' % self.func_id)
self.assertEqual(204, resp.status_int)
mock_engine_delete.assert_called_once_with(self.func_id, version=1)
mock_etcd_delete.assert_called_once_with(self.func_id, version=1)
mock_package_delete.assert_called_once_with(
unit_base.DEFAULT_PROJECT_ID, self.func_id, None, version=1)
# We need to set context as it was removed after the API call
context.set_ctx(self.ctx)
with db_api.transaction():
func_db = db_api.get_function(self.func_id)
self.assertEqual(0, len(func_db.versions))
self.assertEqual(0, func_db.latest_version)
def get(self, id):
LOG.info("Fetch function [id=%s]", id)
download = strutils.bool_from_string(
pecan.request.GET.get('download', False))
func_db = db_api.get_function(id)
ctx = context.get_ctx()
if not download:
pecan.override_template('json')
return resources.Function.from_dict(func_db.to_dict()).to_dict()
else:
f = self.storage_provider.retrieve(
ctx.projectid,
id,
)
pecan.response.app_iter = FileIter(f)
pecan.response.headers['Content-Type'] = 'application/zip'
pecan.response.headers['Content-Disposition'] = (
'attachment; filename="%s"' % os.path.basename(f.name))
def invoke(self, id, **kwargs):
with db_api.transaction():
# The webhook url can be accessed without authentication, so
# insecure is used here
webhook_db = db_api.get_webhook(id, insecure=True)
function_alias = webhook_db.function_alias
if function_alias:
alias = db_api.get_function_alias(function_alias,
insecure=True)
function_id = alias.function_id
function_version = alias.function_version
function_db = db_api.get_function(function_id, insecure=True)
else:
function_db = webhook_db.function
function_id = webhook_db.function_id
function_version = webhook_db.function_version
trust_id = function_db.trust_id
project_id = function_db.project_id
LOG.info('Invoking function %s(version %s) by webhook %s', function_id,
function_version, id)
# Setup user context
ctx = keystone_utils.create_trust_context(trust_id, project_id)
context.set_ctx(ctx)
params = {
'function_id': function_id,
'function_version': function_version,
'sync': False,
'input': json.dumps(kwargs),
'description': constants.EXECUTION_BY_WEBHOOK % id
}
execution = executions.create_execution(self.engine_client, params)
pecan.response.status = 202
return {'execution_id': execution.id}
def test_create_execution_prepare_execution_exception(
self,
etcd_util_get_service_url_mock
):
"""test_create_execution_prepare_execution_exception
Create execution for image type function, prepare_execution method
raises exception.
"""
function = self.create_function()
function_id = function.id
runtime_id = function.runtime_id
db_api.update_function(
function_id,
{
'code': {
'source': constants.IMAGE_FUNCTION,
'image': self.rand_name('image', prefix=self.prefix)
}
}
)
function = db_api.get_function(function_id)
execution = self.create_execution(function_id=function_id)
execution_id = execution.id
prepare_execution = self.orchestrator.prepare_execution
prepare_execution.side_effect = exc.OrchestratorException(
'Exception in prepare_execution'
)
etcd_util_get_service_url_mock.return_value = None
self.default_engine.create_execution(
mock.Mock(), execution_id, function_id, 0, runtime_id)
execution = db_api.get_execution(execution_id)
self.assertEqual(status.ERROR, execution.status)
self.assertEqual('', execution.logs)
self.assertEqual({'output': 'Function execution failed.'},
execution.result)
def scale_up(self, function_id, version, scale):
"""Scale up the workers for function version execution.
This is admin only operation. The load monitoring of execution
depends on the monitoring solution of underlying orchestrator.
"""
acl.enforce('function_version:scale_up', context.get_ctx())
func_db = db_api.get_function(function_id)
# If version=0, it's equivalent to /functions/<funcion-id>/scale_up
if version > 0:
db_api.get_function_version(function_id, version)
params = scale.to_dict()
LOG.info('Starting to scale up function %s(version %s), params: %s',
function_id, version, params)
self.engine_client.scaleup_function(function_id,
runtime_id=func_db.runtime_id,
version=version,
count=params['count'])
def create_execution(engine_client, params):
function_id = params['function_id']
is_sync = params.get('sync', True)
input = params.get('input')
version = params.get('function_version', 0)
func_db = db_api.get_function(function_id)
runtime_id = func_db.runtime_id
# Image type function does not need runtime
if runtime_id:
runtime_db = db_api.get_runtime(runtime_id)
if runtime_db and runtime_db.status != status.AVAILABLE:
raise exc.RuntimeNotAvailableException(
'Runtime %s is not available.' % func_db.runtime_id)
if version > 0:
if func_db.code['source'] != constants.PACKAGE_FUNCTION:
raise exc.InputException(
"Can not specify version for %s type function." %
constants.PACKAGE_FUNCTION)
# update version count
version_db = db_api.get_function_version(function_id, version)
pre_version_count = version_db.count
_update_function_version_db(version_db.id, pre_version_count)
else:
pre_count = func_db.count
_update_function_db(function_id, pre_count)
# input in params should be a string.
if input:
try:
params['input'] = json.loads(input)
except ValueError:
params['input'] = {'__function_input': input}
params.update({'status': status.RUNNING})
db_model = db_api.create_execution(params)
try:
engine_client.create_execution(db_model.id,
function_id,
version,
runtime_id,
input=params.get('input'),
is_sync=is_sync)
except exc.QinlingException:
# Catch RPC errors for executions:
# - for RemoteError in an RPC call, the execution status would be
# handled in the engine side;
# - for other exceptions in an RPC call or cast, the execution status
# would remain RUNNING so we should update it.
db_model = db_api.get_execution(db_model.id)
if db_model.status == status.RUNNING:
db_model = db_api.update_execution(db_model.id,
{'status': status.ERROR})
return db_model
if is_sync:
# The execution should already be updated by engine service for sync
# execution.
db_model = db_api.get_execution(db_model.id)
return db_model
def handle_job(engine_client):
"""Execute job task with no db transactions."""
for job in db_api.get_next_jobs(timeutils.utcnow() + timedelta(seconds=3)):
job_id = job.id
func_id = job.function_id
LOG.debug("Processing job: %s, function: %s", job_id, func_id)
func_db = db_api.get_function(func_id, insecure=True)
trust_id = func_db.trust_id
try:
# Setup context before schedule job.
ctx = keystone_utils.create_trust_context(trust_id, job.project_id)
context.set_ctx(ctx)
if (job.count is not None and job.count > 0):
job.count -= 1
# Job delete/update is done using UPDATE ... FROM ... WHERE
# non-locking clause.
if job.count == 0:
modified = db_api.conditional_update(
models.Job,
{
'status': status.DONE,
'count': 0
},
{
'id': job_id,
'status': status.RUNNING
},
insecure=True,
)
else:
next_time = jobs.get_next_execution_time(
job.pattern, job.next_execution_time)
modified = db_api.conditional_update(
models.Job,
{
'next_execution_time': next_time,
'count': job.count
},
{
'id': job_id,
'next_execution_time': job.next_execution_time
},
insecure=True,
)
if not modified:
LOG.warning(
'Job %s has been already handled by another periodic '
'task.', job_id)
continue
LOG.debug("Starting to execute function %s by job %s", func_id,
job_id)
params = {
'function_id': func_id,
'input': job.function_input,
'sync': False,
'description': constants.EXECUTION_BY_JOB % job_id
}
executions.create_execution(engine_client, params)
except Exception:
LOG.exception("Failed to process job %s", job_id)
finally:
context.set_ctx(None)
def create_execution(self,
ctx,
execution_id,
function_id,
runtime_id,
input=None):
LOG.info(
'Creating execution. execution_id=%s, function_id=%s, '
'runtime_id=%s, input=%s', execution_id, function_id, runtime_id,
input)
function = db_api.get_function(function_id)
source = function.code['source']
image = None
identifier = None
labels = None
svc_url = None
# Auto scale workers if needed
if source != constants.IMAGE_FUNCTION:
svc_url = self.function_load_check(function_id, runtime_id)
temp_url = etcd_util.get_service_url(function_id)
svc_url = svc_url or temp_url
if svc_url:
func_url = '%s/execute' % svc_url
LOG.debug(
'Found service url for function: %s, execution: %s, url: %s',
function_id, execution_id, func_url)
data = utils.get_request_data(CONF, function_id, execution_id,
input, function.entry,
function.trust_id,
self.qinling_endpoint)
success, res = utils.url_request(self.session, func_url, body=data)
success = success and res.pop('success')
LOG.debug('Finished execution %s, success: %s', execution_id,
success)
db_api.update_execution(
execution_id, {
'status': status.SUCCESS if success else status.FAILED,
'logs': res.pop('logs', ''),
'result': res
})
return
if source == constants.IMAGE_FUNCTION:
image = function.code['image']
identifier = (
'%s-%s' %
(common.generate_unicode_uuid(dashed=False), function_id))[:63]
labels = {'function_id': function_id}
else:
identifier = runtime_id
labels = {'runtime_id': runtime_id}
_, svc_url = self.orchestrator.prepare_execution(
function_id,
image=image,
identifier=identifier,
labels=labels,
input=input,
)
success, res = self.orchestrator.run_execution(
execution_id,
function_id,
input=input,
identifier=identifier,
service_url=svc_url,
entry=function.entry,
trust_id=function.trust_id)
logs = ''
# Execution log is only available for non-image source execution.
if svc_url:
logs = res.pop('logs', '')
success = success and res.pop('success')
else:
# If the function is created from docker image, the result is
# direct output, here we convert to a dict to fit into the db
# schema.
res = {'output': res}
LOG.debug('Finished execution %s, success: %s', execution_id, success)
db_api.update_execution(
execution_id, {
'status': status.SUCCESS if success else status.FAILED,
'logs': logs,
'result': res
})
def create_execution(self,
ctx,
execution_id,
function_id,
runtime_id,
input=None):
LOG.info(
'Creating execution. execution_id=%s, function_id=%s, '
'runtime_id=%s, input=%s', execution_id, function_id, runtime_id,
input)
# FIXME(kong): Make the transaction range smaller.
with db_api.transaction():
execution = db_api.get_execution(execution_id)
function = db_api.get_function(function_id)
if function.service:
func_url = '%s/execute' % function.service.service_url
LOG.debug('Found service url for function: %s, url: %s',
function_id, func_url)
data = {'input': input, 'execution_id': execution_id}
r = self.session.post(func_url, json=data)
res = r.json()
LOG.debug('Finished execution %s', execution_id)
success = res.pop('success')
execution.status = status.SUCCESS if success else status.FAILED
execution.logs = res.pop('logs', '')
execution.output = res
return
source = function.code['source']
image = None
identifier = None
labels = None
if source == constants.IMAGE_FUNCTION:
image = function.code['image']
identifier = ('%s-%s' %
(common.generate_unicode_uuid(dashed=False),
function_id))[:63]
labels = {'function_id': function_id}
else:
identifier = runtime_id
labels = {'runtime_id': runtime_id}
worker_name, service_url = self.orchestrator.prepare_execution(
function_id,
image=image,
identifier=identifier,
labels=labels,
input=input,
entry=function.entry,
trust_id=function.trust_id)
output = self.orchestrator.run_execution(
execution_id,
function_id,
input=input,
identifier=identifier,
service_url=service_url,
)
logs = ''
# Execution log is only available for non-image source execution.
if service_url:
logs = output.pop('logs', '')
success = output.pop('success')
else:
# If the function is created from docker image, the output is
# direct output, here we convert to a dict to fit into the db
# schema.
output = {'output': output}
success = True
LOG.debug('Finished execution. execution_id=%s, output=%s',
execution_id, output)
execution.output = output
execution.logs = logs
execution.status = status.SUCCESS if success else status.FAILED
# No service is created in orchestrator for single container.
if not image:
mapping = {
'function_id': function_id,
'service_url': service_url,
}
db_api.create_function_service_mapping(mapping)
worker = {
'function_id': function_id,
'worker_name': worker_name
}
db_api.create_function_worker(worker)
def delete(self, id):
"""Delete the specified function.
Delete function will also delete all its versions.
"""
LOG.info("Delete function %s.", id)
with db_api.transaction():
func_db = db_api.get_function(id)
if len(func_db.jobs) > 0:
raise exc.NotAllowedException(
'The function is still associated with running job(s).'
)
if len(func_db.webhooks) > 0:
raise exc.NotAllowedException(
'The function is still associated with webhook(s).'
)
# Even admin user can not delete other project's function because
# the trust associated can only be removed by function owner.
if func_db.project_id != context.get_ctx().projectid:
raise exc.NotAllowedException(
'Function can only be deleted by its owner.'
)
# Delete trust if needed
if func_db.trust_id:
keystone_util.delete_trust(func_db.trust_id)
for version_db in func_db.versions:
# Delete all resources created by orchestrator asynchronously.
self.engine_client.delete_function(
id,
version=version_db.version_number
)
# Delete etcd keys
etcd_util.delete_function(
id,
version=version_db.version_number
)
# Delete function version packages. Versions is only supported
# for package type function.
self.storage_provider.delete(
func_db.project_id,
id,
None,
version=version_db.version_number
)
# Delete resources for function version 0(func_db.versions==[])
self.engine_client.delete_function(id)
etcd_util.delete_function(id)
source = func_db.code['source']
if source == constants.PACKAGE_FUNCTION:
self.storage_provider.delete(func_db.project_id, id,
func_db.code['md5sum'])
# This will also delete function service mapping and function
# versions as well.
db_api.delete_function(id)
def test_create_execution_image_type_function(self, mock_svc_url):
"""Create 2 executions for an image type function."""
function = self.create_function()
function_id = function.id
runtime_id = function.runtime_id
db_api.update_function(
function_id, {
'code': {
'source': constants.IMAGE_FUNCTION,
'image': self.rand_name('image', prefix=self.prefix)
}
})
function = db_api.get_function(function_id)
execution_1 = self.create_execution(function_id=function_id)
execution_1_id = execution_1.id
execution_2 = self.create_execution(function_id=function_id)
execution_2_id = execution_2.id
mock_svc_url.return_value = None
self.orchestrator.prepare_execution.return_value = (mock.Mock(), None)
self.orchestrator.run_execution.side_effect = [(True, {
'duration': 5,
'logs': 'fake log'
}), (False, {
'duration': 0,
'error': 'Function execution failed.'
})]
# Create two executions, with different results
self.default_engine.create_execution(mock.Mock(), execution_1_id,
function_id, 0, runtime_id)
self.default_engine.create_execution(mock.Mock(),
execution_2_id,
function_id,
0,
runtime_id,
input='input')
get_service_url_calls = [
mock.call(function_id, 0),
mock.call(function_id, 0)
]
mock_svc_url.assert_has_calls(get_service_url_calls)
prepare_calls = [
mock.call(function_id,
0,
rlimit=self.rlimit,
image=function.code['image'],
identifier=mock.ANY,
labels=None,
input=None),
mock.call(function_id,
0,
rlimit=self.rlimit,
image=function.code['image'],
identifier=mock.ANY,
labels=None,
input='input')
]
self.orchestrator.prepare_execution.assert_has_calls(prepare_calls)
run_calls = [
mock.call(execution_1_id,
function_id,
0,
rlimit=None,
input=None,
identifier=mock.ANY,
service_url=None,
entry=function.entry,
trust_id=function.trust_id),
mock.call(execution_2_id,
function_id,
0,
rlimit=None,
input='input',
identifier=mock.ANY,
service_url=None,
entry=function.entry,
trust_id=function.trust_id)
]
self.orchestrator.run_execution.assert_has_calls(run_calls)
execution_1 = db_api.get_execution(execution_1_id)
execution_2 = db_api.get_execution(execution_2_id)
self.assertEqual(status.SUCCESS, execution_1.status)
self.assertEqual('fake log', execution_1.logs)
self.assertEqual({"duration": 5}, execution_1.result)
self.assertEqual(status.FAILED, execution_2.status)
self.assertEqual('', execution_2.logs)
self.assertEqual({
'duration': 0,
'error': 'Function execution failed.'
}, execution_2.result)
def put(self, id, **kwargs):
"""Update function.
- Function can not being used by job.
- Function can not being executed.
- (TODO)Function status should be changed so no execution will create
when function is updating.
"""
values = {}
try:
for key in UPDATE_ALLOWED:
if kwargs.get(key) is not None:
if key == "code":
kwargs[key] = json.loads(kwargs[key])
values.update({key: kwargs[key]})
except Exception as e:
raise exc.InputException("Invalid input, %s" % str(e))
LOG.info('Update function %s, params: %s', id, values)
ctx = context.get_ctx()
if values.get('timeout'):
common.validate_int_in_range('timeout', values['timeout'],
CONF.resource_limits.min_timeout,
CONF.resource_limits.max_timeout)
db_update_only = set(['name', 'description', 'timeout'])
if set(values.keys()).issubset(db_update_only):
func_db = db_api.update_function(id, values)
else:
source = values.get('code', {}).get('source')
md5sum = values.get('code', {}).get('md5sum')
cpu = values.get('cpu')
memory_size = values.get('memory_size')
# Check cpu and memory_size values when updating.
if cpu is not None:
common.validate_int_in_range('cpu', values['cpu'],
CONF.resource_limits.min_cpu,
CONF.resource_limits.max_cpu)
if memory_size is not None:
common.validate_int_in_range('memory', values['memory_size'],
CONF.resource_limits.min_memory,
CONF.resource_limits.max_memory)
with db_api.transaction():
pre_func = db_api.get_function(id)
if len(pre_func.jobs) > 0:
raise exc.NotAllowedException(
'The function is still associated with running job(s).'
)
pre_source = pre_func.code['source']
pre_md5sum = pre_func.code.get('md5sum')
if source and source != pre_source:
raise exc.InputException(
"The function code type can not be changed.")
if pre_source == constants.IMAGE_FUNCTION:
raise exc.InputException(
"The image type function code can not be changed.")
# Package type function. 'code' and 'entry' make sense only if
# 'package' is provided
package_updated = False
if (pre_source == constants.PACKAGE_FUNCTION
and values.get('package') is not None):
if md5sum and md5sum == pre_md5sum:
raise exc.InputException(
"The function code checksum is not changed.")
# Update the package data.
data = values['package'].file.read()
package_updated, md5sum = self.storage_provider.store(
ctx.projectid, id, data, md5sum=md5sum)
values.setdefault('code', {}).update({
"md5sum": md5sum,
"source": pre_source
})
values.pop('package')
# Swift type function
if (pre_source == constants.SWIFT_FUNCTION
and "swift" in values.get('code', {})):
swift_info = values['code']["swift"]
if not (swift_info.get('container')
or swift_info.get('object')):
raise exc.InputException(
"Either container or object must be provided for "
"swift type function update.")
new_swift_info = pre_func.code['swift']
new_swift_info.update(swift_info)
self._check_swift(new_swift_info.get('container'),
new_swift_info.get('object'))
values['code'] = {
"source": pre_source,
"swift": new_swift_info
}
# Delete allocated resources in orchestrator and etcd.
self.engine_client.delete_function(id)
etcd_util.delete_function(id)
func_db = db_api.update_function(id, values)
# Delete the old function package if needed
if package_updated:
self.storage_provider.delete(ctx.projectid, id, pre_md5sum)
pecan.response.status = 200
return resources.Function.from_db_obj(func_db).to_dict()
def create_execution(self,
ctx,
execution_id,
function_id,
function_version,
runtime_id,
input=None):
LOG.info(
'Creating execution. execution_id=%s, function_id=%s, '
'function_version=%s, runtime_id=%s, input=%s', execution_id,
function_id, function_version, runtime_id, input)
function = db_api.get_function(function_id)
source = function.code['source']
rlimit = {'cpu': function.cpu, 'memory_size': function.memory_size}
image = None
identifier = None
labels = None
svc_url = None
is_image_source = source == constants.IMAGE_FUNCTION
# Auto scale workers if needed
if not is_image_source:
try:
svc_url = self.function_load_check(function_id,
function_version,
runtime_id)
except exc.OrchestratorException as e:
utils.handle_execution_exception(execution_id, str(e))
return
temp_url = etcd_util.get_service_url(function_id, function_version)
svc_url = svc_url or temp_url
if svc_url:
func_url = '%s/execute' % svc_url
LOG.debug(
'Found service url for function: %s(version %s), execution: '
'%s, url: %s', function_id, function_version, execution_id,
func_url)
data = utils.get_request_data(CONF, function_id, function_version,
execution_id, rlimit, input,
function.entry, function.trust_id,
self.qinling_endpoint)
success, res = utils.url_request(self.session, func_url, body=data)
utils.finish_execution(execution_id,
success,
res,
is_image_source=is_image_source)
return
if source == constants.IMAGE_FUNCTION:
image = function.code['image']
# Be consistent with k8s naming convention
identifier = (
'%s-%s' %
(common.generate_unicode_uuid(dashed=False), function_id))[:63]
else:
identifier = runtime_id
labels = {'runtime_id': runtime_id}
try:
# For image function, it will be executed inside this method; for
# package type function it only sets up underlying resources and
# get a service url. If the service url is already created
# beforehand, nothing happens.
_, svc_url = self.orchestrator.prepare_execution(
function_id,
function_version,
rlimit=rlimit,
image=image,
identifier=identifier,
labels=labels,
input=input,
)
except exc.OrchestratorException as e:
utils.handle_execution_exception(execution_id, str(e))
return
# For image type function, read the worker log; For package type
# function, invoke and get log
success, res = self.orchestrator.run_execution(
execution_id,
function_id,
function_version,
rlimit=rlimit if svc_url else None,
input=input,
identifier=identifier,
service_url=svc_url,
entry=function.entry,
trust_id=function.trust_id)
utils.finish_execution(execution_id,
success,
res,
is_image_source=is_image_source)
