def function_load_check(self, function_id, version, runtime_id): """Check function load and scale the workers if needed. :return: None if no need to scale up otherwise return the service url """ with etcd_util.get_worker_lock(function_id, version) as lock: if not lock.is_acquired(): raise exc.EtcdLockException( 'Etcd: failed to get worker lock for function %s' '(version %s).' % (function_id, version)) workers = etcd_util.get_workers(function_id, version) running_execs = db_api.get_executions(function_id=function_id, function_version=version, status=status.RUNNING) concurrency = (len(running_execs) or 1) / (len(workers) or 1) if (len(workers) == 0 or concurrency > CONF.engine.function_concurrency): LOG.info( 'Scale up function %s(version %s). Current concurrency: ' '%s, execution number %s, worker number %s', function_id, version, concurrency, len(running_execs), len(workers)) # NOTE(kong): The increase step could be configurable return self.scaleup_function(None, function_id, version, runtime_id, 1)
def scale_down(self, function_id, version, scale): """Scale down the workers for function version execution. This is admin only operation. The load monitoring of execution depends on the monitoring solution of underlying orchestrator. """ acl.enforce('function_version:scale_down', context.get_ctx()) db_api.get_function(function_id) params = scale.to_dict() # If version=0, it's equivalent to /functions/<funcion-id>/scale_down if version > 0: db_api.get_function_version(function_id, version) workers = etcd_util.get_workers(function_id, version=version) if len(workers) <= 1: LOG.info('No need to scale down function %s(version %s)', function_id, version) return LOG.info('Starting to scale down function %s(version %s), params: %s', function_id, version, params) self.engine_client.scaledown_function(function_id, version=version, count=params['count'])
def scaledown_function(self, ctx, function_id, count=1): workers = etcd_util.get_workers(function_id) worker_deleted_num = (count if len(workers) > count else len(workers) - 1) workers = workers[:worker_deleted_num] for worker in workers: LOG.debug('Removing worker %s', worker) self.orchestrator.delete_worker(worker) LOG.info('Finished scaling up function %s.', function_id)
def get_all(self, function_id): acl.enforce('function_worker:get_all', context.get_ctx()) LOG.info("Get workers for function %s.", function_id) workers = etcd_util.get_workers(function_id, CONF) workers = [ resources.FunctionWorker.from_dict({ 'function_id': function_id, 'worker_name': w }) for w in workers ] return resources.FunctionWorkers(workers=workers)
def get_all(self, function_id, function_version=0): acl.enforce('function_worker:get_all', context.get_ctx()) LOG.info("Getting workers for function %s(version %s).", function_id, function_version) workers = etcd_util.get_workers(function_id, version=function_version) workers = [ resources.FunctionWorker.from_dict({ 'function_id': function_id, 'function_version': function_version, 'worker_name': w }) for w in workers ] return resources.FunctionWorkers(workers=workers)
def scale_down(self, id, scale): """Scale down the containers for function execution. This is admin only operation. The load monitoring of function execution depends on the monitoring solution of underlying orchestrator. """ acl.enforce('function:scale_down', context.get_ctx()) db_api.get_function(id) workers = etcd_util.get_workers(id) params = scale.to_dict() if len(workers) <= 1: LOG.info('No need to scale down function %s', id) return LOG.info('Starting to scale down function %s, params: %s', id, params) self.engine_client.scaledown_function(id, count=params['count'])
def function_load_check(self, function_id, runtime_id): with etcd_util.get_worker_lock() as lock: if not lock.is_acquired(): return False workers = etcd_util.get_workers(function_id) running_execs = db_api.get_executions(function_id=function_id, status=status.RUNNING) concurrency = (len(running_execs) or 1) / (len(workers) or 1) if (len(workers) == 0 or concurrency > CONF.engine.function_concurrency): LOG.info( 'Scale up function %s. Current concurrency: %s, execution ' 'number %s, worker number %s', function_id, concurrency, len(running_execs), len(workers)) # NOTE(kong): The increase step could be configurable return self.scaleup_function(None, function_id, runtime_id, 1)