def function_load_check(self, function_id, version, runtime_id):
"""Check function load and scale the workers if needed.
:return: None if no need to scale up otherwise return the service url
"""
with etcd_util.get_worker_lock(function_id, version) as lock:
if not lock.is_acquired():
raise exc.EtcdLockException(
'Etcd: failed to get worker lock for function %s'
'(version %s).' % (function_id, version))
workers = etcd_util.get_workers(function_id, version)
running_execs = db_api.get_executions(function_id=function_id,
function_version=version,
status=status.RUNNING)
concurrency = (len(running_execs) or 1) / (len(workers) or 1)
if (len(workers) == 0
or concurrency > CONF.engine.function_concurrency):
LOG.info(
'Scale up function %s(version %s). Current concurrency: '
'%s, execution number %s, worker number %s', function_id,
version, concurrency, len(running_execs), len(workers))
# NOTE(kong): The increase step could be configurable
return self.scaleup_function(None, function_id, version,
runtime_id, 1)
def scale_down(self, function_id, version, scale):
"""Scale down the workers for function version execution.
This is admin only operation. The load monitoring of execution
depends on the monitoring solution of underlying orchestrator.
"""
acl.enforce('function_version:scale_down', context.get_ctx())
db_api.get_function(function_id)
params = scale.to_dict()
# If version=0, it's equivalent to /functions/<funcion-id>/scale_down
if version > 0:
db_api.get_function_version(function_id, version)
workers = etcd_util.get_workers(function_id, version=version)
if len(workers) <= 1:
LOG.info('No need to scale down function %s(version %s)',
function_id, version)
return
LOG.info('Starting to scale down function %s(version %s), params: %s',
function_id, version, params)
self.engine_client.scaledown_function(function_id, version=version,
count=params['count'])
def scaledown_function(self, ctx, function_id, count=1):
workers = etcd_util.get_workers(function_id)
worker_deleted_num = (count if len(workers) > count else len(workers) -
1)
workers = workers[:worker_deleted_num]
for worker in workers:
LOG.debug('Removing worker %s', worker)
self.orchestrator.delete_worker(worker)
LOG.info('Finished scaling up function %s.', function_id)
def get_all(self, function_id):
acl.enforce('function_worker:get_all', context.get_ctx())
LOG.info("Get workers for function %s.", function_id)
workers = etcd_util.get_workers(function_id, CONF)
workers = [
resources.FunctionWorker.from_dict({
'function_id': function_id,
'worker_name': w
}) for w in workers
]
return resources.FunctionWorkers(workers=workers)
def get_all(self, function_id, function_version=0):
acl.enforce('function_worker:get_all', context.get_ctx())
LOG.info("Getting workers for function %s(version %s).", function_id,
function_version)
workers = etcd_util.get_workers(function_id, version=function_version)
workers = [
resources.FunctionWorker.from_dict({
'function_id': function_id,
'function_version': function_version,
'worker_name': w
}) for w in workers
]
return resources.FunctionWorkers(workers=workers)
def scale_down(self, id, scale):
"""Scale down the containers for function execution.
This is admin only operation. The load monitoring of function execution
depends on the monitoring solution of underlying orchestrator.
"""
acl.enforce('function:scale_down', context.get_ctx())
db_api.get_function(id)
workers = etcd_util.get_workers(id)
params = scale.to_dict()
if len(workers) <= 1:
LOG.info('No need to scale down function %s', id)
return
LOG.info('Starting to scale down function %s, params: %s', id, params)
self.engine_client.scaledown_function(id, count=params['count'])
def function_load_check(self, function_id, runtime_id):
with etcd_util.get_worker_lock() as lock:
if not lock.is_acquired():
return False
workers = etcd_util.get_workers(function_id)
running_execs = db_api.get_executions(function_id=function_id,
status=status.RUNNING)
concurrency = (len(running_execs) or 1) / (len(workers) or 1)
if (len(workers) == 0
or concurrency > CONF.engine.function_concurrency):
LOG.info(
'Scale up function %s. Current concurrency: %s, execution '
'number %s, worker number %s', function_id, concurrency,
len(running_execs), len(workers))
# NOTE(kong): The increase step could be configurable
return self.scaleup_function(None, function_id, runtime_id, 1)