def scale_up(self, function_id, version, scale): """Scale up the workers for function version execution. This is admin only operation. The load monitoring of execution depends on the monitoring solution of underlying orchestrator. """ acl.enforce('function_version:scale_up', context.get_ctx()) func_db = db_api.get_function(function_id) # If version=0, it's equivalent to /functions/<funcion-id>/scale_up if version > 0: db_api.get_function_version(function_id, version) params = scale.to_dict() LOG.info('Starting to scale up function %s(version %s), params: %s', function_id, version, params) self.engine_client.scaleup_function( function_id, runtime_id=func_db.runtime_id, version=version, count=params['count'] )
def scale_down(self, function_id, version, scale): """Scale down the workers for function version execution. This is admin only operation. The load monitoring of execution depends on the monitoring solution of underlying orchestrator. """ acl.enforce('function_version:scale_down', context.get_ctx()) db_api.get_function(function_id) params = scale.to_dict() # If version=0, it's equivalent to /functions/<funcion-id>/scale_down if version > 0: db_api.get_function_version(function_id, version) workers = etcd_util.get_workers(function_id, version=version) if len(workers) <= 1: LOG.info('No need to scale down function %s(version %s)', function_id, version) return LOG.info('Starting to scale down function %s(version %s), params: %s', function_id, version, params) self.engine_client.scaledown_function(function_id, version=version, count=params['count'])
def put(self, id, webhook): """Update webhook. Currently, only function_id and function_version are allowed to update. """ acl.enforce('webhook:update', context.get_ctx()) values = {} for key in UPDATE_ALLOWED: if webhook.to_dict().get(key) is not None: values.update({key: webhook.to_dict()[key]}) LOG.info('Update %s %s, params: %s', self.type, id, values) # Even admin user can not expose normal user's function webhook_db = db_api.get_webhook(id, insecure=False) pre_function_id = webhook_db.function_id pre_version = webhook_db.function_version new_function_id = values.get("function_id", pre_function_id) new_version = values.get("function_version", pre_version) db_api.get_function(new_function_id, insecure=False) if new_version > 0: db_api.get_function_version(new_function_id, new_version) webhook = db_api.update_webhook(id, values).to_dict() return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
def post(self, webhook): acl.enforce('webhook:create', context.get_ctx()) params = webhook.to_dict() if not (params.get("function_id") or params.get("function_alias")): raise exc.InputException( 'Either function_alias or function_id must be provided.') # if function_alias provided function_alias = params.get('function_alias') if function_alias: alias_db = db_api.get_function_alias(function_alias) function_id = alias_db.function_id version = alias_db.function_version params.update({ 'function_id': function_id, 'function_version': version }) LOG.info("Creating %s, params: %s", self.type, params) # Even admin user can not expose normal user's function db_api.get_function(params['function_id'], insecure=False) version = params.get('function_version', 0) if version > 0: db_api.get_function_version(params['function_id'], version) webhook_d = db_api.create_webhook(params).to_dict() return resources.Webhook.from_dict( self._add_webhook_url(webhook_d['id'], webhook_d))
def post(self, job): """Creates a new job.""" params = job.to_dict() if not POST_REQUIRED.issubset(set(params.keys())): raise exc.InputException( 'Required param is missing. Required: %s' % POST_REQUIRED) # Check the input params. first_time, next_time, count = jobs.validate_job(params) LOG.info("Creating %s, params: %s", self.type, params) version = params.get('function_version', 0) with db_api.transaction(): db_api.get_function(params['function_id']) if version > 0: db_api.get_function_version(params['function_id'], version) values = { 'name': params.get('name'), 'pattern': params.get('pattern'), 'first_execution_time': first_time, 'next_execution_time': next_time, 'count': count, 'function_id': params['function_id'], 'function_version': version, 'function_input': params.get('function_input'), 'status': status.RUNNING } db_job = db_api.create_job(values) return resources.Job.from_db_obj(db_job)
def test_job_handler_with_version(self, mock_next_time): db_func = self.create_function() function_id = db_func.id new_version = db_api.increase_function_version(function_id, 0) self.assertEqual(0, new_version.count) now = datetime.utcnow() db_job = self.create_job( function_id, function_version=1, status=status.RUNNING, next_execution_time=now, count=2 ) job_id = db_job.id e_client = mock.Mock() # It doesn't matter what's the returned value, but need to be in # datetime type. mock_next_time.return_value = now + timedelta(seconds=1) periodics.handle_job(e_client) context.set_ctx(self.ctx) db_job = db_api.get_job(job_id) self.assertEqual(1, db_job.count) db_func = db_api.get_function(function_id) self.assertEqual(0, db_func.count) db_version = db_api.get_function_version(function_id, 1) self.assertEqual(1, db_version.count) db_execs = db_api.get_executions(function_id=function_id, function_version=1) self.assertEqual(1, len(db_execs)) periodics.handle_job(e_client) context.set_ctx(self.ctx) db_job = db_api.get_job(job_id) self.assertEqual(0, db_job.count) self.assertEqual(status.DONE, db_job.status) db_func = db_api.get_function(function_id) self.assertEqual(0, db_func.count) db_version = db_api.get_function_version(function_id, 1) self.assertEqual(2, db_version.count) db_execs = db_api.get_executions(function_id=function_id, function_version=1) self.assertEqual(2, len(db_execs))
def test_job_handler_with_alias(self, mock_next_time): e_client = mock.Mock() now = datetime.utcnow() # It doesn't matter what's the returned value, but need to be in # datetime type. mock_next_time.return_value = now + timedelta(seconds=1) # Create a alias for a function. alias_name = self.rand_name(name="alias", prefix=self.prefix) db_func = self.create_function() function_id = db_func.id db_api.create_function_alias(name=alias_name, function_id=function_id) self.create_job( function_alias=alias_name, status=status.RUNNING, next_execution_time=now, ) periodics.handle_job(e_client) context.set_ctx(self.ctx) # Create function version 1 and update the alias. db_api.increase_function_version(function_id, 0) db_api.update_function_alias(alias_name, function_version=1) periodics.handle_job(e_client) context.set_ctx(self.ctx) db_func = db_api.get_function(function_id) self.assertEqual(1, db_func.count) db_version = db_api.get_function_version(function_id, 1) self.assertEqual(1, db_version.count) db_execs = db_api.get_executions(function_id=function_id) self.assertEqual(2, len(db_execs))
def delete(self, function_id, version): """Delete a specific function version. - The version should not being used by any job - The version should not being used by any webhook - Admin user can not delete normal user's version """ ctx = context.get_ctx() acl.enforce('function_version:delete', ctx) LOG.info("Deleting version %s of function %s.", version, function_id) with db_api.transaction(): version_db = db_api.get_function_version(function_id, version, insecure=False) latest_version = version_db.function.latest_version version_jobs = db_api.get_jobs( function_id=version_db.function_id, function_version=version_db.version_number, status={'nin': ['done', 'cancelled']}) if len(version_jobs) > 0: raise exc.NotAllowedException( 'The function version is still associated with running ' 'job(s).') version_webhook = db_api.get_webhooks( function_id=version_db.function_id, function_version=version_db.version_number, ) if len(version_webhook) > 0: raise exc.NotAllowedException( 'The function version is still associated with webhook.') filters = rest_utils.get_filters( function_id=version_db.function_id, function_version=version_db.version_number) version_aliases = db_api.get_function_aliases(**filters) if len(version_aliases) > 0: raise exc.NotAllowedException( 'The function version is still associated with alias.') # Delete resources for function version self.engine_client.delete_function(function_id, version=version) etcd_util.delete_function(function_id, version=version) self.storage_provider.delete(ctx.projectid, function_id, None, version=version) db_api.delete_function_version(function_id, version) if latest_version == version: version_db.function.latest_version = latest_version - 1 LOG.info("Version %s of function %s deleted.", version, function_id)
def detach(self, function_id, version): """Detach the function version from its underlying workers. This is admin only operation, which gives admin user a safe way to clean up the underlying resources allocated for the function version. """ acl.enforce('function_version:detach', context.get_ctx()) db_api.get_function(function_id) # If version=0, it's equivalent to /functions/<funcion-id>/detach if version > 0: db_api.get_function_version(function_id, version) LOG.info('Starting to detach function %s(version %s)', function_id, version) # Delete allocated resources in orchestrator and etcd keys. self.engine_client.delete_function(function_id, version=version) etcd_util.delete_function(function_id, version=version)
def post(self, job): """Creates a new job.""" params = job.to_dict() if not (params.get("function_id") or params.get("function_alias")): raise exc.InputException( 'Either function_alias or function_id must be provided.') LOG.info("Creating %s, params: %s", self.type, params) # Check the input params. first_time, next_time, count = jobs.validate_job(params) version = params.get('function_version', 0) function_alias = params.get('function_alias') if function_alias: # Check if the alias exists. db_api.get_function_alias(function_alias) else: # Check the function(version) exists. db_api.get_function(params['function_id']) if version > 0: # Check if the version exists. db_api.get_function_version(params['function_id'], version) values = { 'name': params.get('name'), 'pattern': params.get('pattern'), 'first_execution_time': first_time, 'next_execution_time': next_time, 'count': count, 'function_alias': function_alias, 'function_id': params.get("function_id"), 'function_version': version, 'function_input': params.get('function_input'), 'status': status.RUNNING } db_job = db_api.create_job(values) return resources.Job.from_db_obj(db_job)
def put(self, id, webhook): acl.enforce('webhook:update', context.get_ctx()) values = {} for key in UPDATE_ALLOWED: if webhook.to_dict().get(key) is not None: values.update({key: webhook.to_dict()[key]}) LOG.info('Update %s %s, params: %s', self.type, id, values) # Even admin user can not expose normal user's function webhook_db = db_api.get_webhook(id, insecure=False) pre_alias = webhook_db.function_alias pre_function_id = webhook_db.function_id pre_version = webhook_db.function_version new_alias = values.get("function_alias") new_function_id = values.get("function_id", pre_function_id) new_version = values.get("function_version", pre_version) function_id = pre_function_id version = pre_version if new_alias and new_alias != pre_alias: alias_db = db_api.get_function_alias(new_alias) function_id = alias_db.function_id version = alias_db.function_version # If function_alias is provided, we don't store either functin id # or function version. values.update({'function_id': None, 'function_version': None}) elif new_function_id != pre_function_id or new_version != pre_version: function_id = new_function_id version = new_version values.update({"function_alias": None}) db_api.get_function(function_id, insecure=False) if version and version > 0: db_api.get_function_version(function_id, version) webhook = db_api.update_webhook(id, values).to_dict() return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
def post(self, webhook): acl.enforce('webhook:create', context.get_ctx()) params = webhook.to_dict() if not POST_REQUIRED.issubset(set(params.keys())): raise exc.InputException( 'Required param is missing. Required: %s' % POST_REQUIRED ) LOG.info("Creating %s, params: %s", self.type, params) # Even admin user can not expose normal user's function db_api.get_function(params['function_id'], insecure=False) version = params.get('function_version', 0) if version > 0: db_api.get_function_version(params['function_id'], version) webhook_d = db_api.create_webhook(params).to_dict() return resources.Webhook.from_dict( self._add_webhook_url(webhook_d['id'], webhook_d) )
def get(self, function_id, version): """Get function version or download function version package. This method can support HTTP request using either 'Accept:application/json' or no 'Accept' header. """ ctx = context.get_ctx() acl.enforce('function_version:get', ctx) download = strutils.bool_from_string( pecan.request.GET.get('download', False) ) version = int(version) version_db = db_api.get_function_version(function_id, version) if not download: LOG.info("Getting version %s for function %s.", version, function_id) pecan.override_template('json') return resources.FunctionVersion.from_db_obj(version_db).to_dict() LOG.info("Downloading version %s for function %s.", version, function_id) f = self.storage_provider.retrieve(version_db.project_id, function_id, None, version=version) if isinstance(f, collections.Iterable): pecan.response.app_iter = f else: pecan.response.app_iter = FileIter(f) pecan.response.headers['Content-Type'] = 'application/zip' pecan.response.headers['Content-Disposition'] = ( 'attachment; filename="%s_%s"' % (function_id, version) )
def create_execution(engine_client, params): function_id = params['function_id'] is_sync = params.get('sync', True) input = params.get('input') version = params.get('function_version', 0) func_db = db_api.get_function(function_id) runtime_id = func_db.runtime_id # Image type function does not need runtime if runtime_id: runtime_db = db_api.get_runtime(runtime_id) if runtime_db and runtime_db.status != status.AVAILABLE: raise exc.RuntimeNotAvailableException( 'Runtime %s is not available.' % func_db.runtime_id) if version > 0: if func_db.code['source'] != constants.PACKAGE_FUNCTION: raise exc.InputException( "Can not specify version for %s type function." % constants.PACKAGE_FUNCTION) # update version count version_db = db_api.get_function_version(function_id, version) pre_version_count = version_db.count _update_function_version_db(version_db.id, pre_version_count) else: pre_count = func_db.count _update_function_db(function_id, pre_count) # input in params should be a string. if input: try: params['input'] = json.loads(input) except ValueError: params['input'] = {'__function_input': input} params.update({'status': status.RUNNING}) db_model = db_api.create_execution(params) try: engine_client.create_execution(db_model.id, function_id, version, runtime_id, input=params.get('input'), is_sync=is_sync) except exc.QinlingException: # Catch RPC errors for executions: # - for RemoteError in an RPC call, the execution status would be # handled in the engine side; # - for other exceptions in an RPC call or cast, the execution status # would remain RUNNING so we should update it. db_model = db_api.get_execution(db_model.id) if db_model.status == status.RUNNING: db_model = db_api.update_execution(db_model.id, {'status': status.ERROR}) return db_model if is_sync: # The execution should already be updated by engine service for sync # execution. db_model = db_api.get_execution(db_model.id) return db_model