def put(self, id, webhook): acl.enforce('webhook:update', context.get_ctx()) values = {} for key in UPDATE_ALLOWED: if webhook.to_dict().get(key) is not None: values.update({key: webhook.to_dict()[key]}) LOG.info('Update %s %s, params: %s', self.type, id, values) # Even admin user can not expose normal user's function webhook_db = db_api.get_webhook(id, insecure=False) pre_alias = webhook_db.function_alias pre_function_id = webhook_db.function_id pre_version = webhook_db.function_version new_alias = values.get("function_alias") new_function_id = values.get("function_id", pre_function_id) new_version = values.get("function_version", pre_version) function_id = pre_function_id version = pre_version if new_alias and new_alias != pre_alias: alias_db = db_api.get_function_alias(new_alias) function_id = alias_db.function_id version = alias_db.function_version # If function_alias is provided, we don't store either functin id # or function version. values.update({'function_id': None, 'function_version': None}) elif new_function_id != pre_function_id or new_version != pre_version: function_id = new_function_id version = new_version values.update({"function_alias": None}) db_api.get_function(function_id, insecure=False) if version and version > 0: db_api.get_function_version(function_id, version) webhook = db_api.update_webhook(id, values).to_dict() return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
def scaledown_function(self, ctx, function_id, count=1): func_db = db_api.get_function(function_id) worker_deleted_num = (count if len(func_db.workers) > count else len(func_db.workers) - 1) workers = func_db.workers[:worker_deleted_num] with db_api.transaction(): for worker in workers: LOG.debug('Removing worker %s', worker.worker_name) self.orchestrator.delete_worker(worker.worker_name, ) db_api.delete_function_worker(worker.worker_name) LOG.info('Finished scaling up function %s.', function_id)
def post(self, job): """Creates a new job.""" params = job.to_dict() if not (params.get("function_id") or params.get("function_alias")): raise exc.InputException( 'Either function_alias or function_id must be provided.') # Check the input params. first_time, next_time, count = jobs.validate_job(params) LOG.info("Creating %s, params: %s", self.type, params) version = params.get('function_version', 0) # if function_alias provided function_alias = params.get('function_alias') if function_alias: alias_db = db_api.get_function_alias(function_alias) function_id = alias_db.function_id version = alias_db.function_version params.update({'function_id': function_id, 'version': version}) with db_api.transaction(): db_api.get_function(params['function_id']) if version > 0: db_api.get_function_version(params['function_id'], version) values = { 'name': params.get('name'), 'pattern': params.get('pattern'), 'first_execution_time': first_time, 'next_execution_time': next_time, 'count': count, 'function_id': params['function_id'], 'function_version': version, 'function_input': params.get('function_input'), 'status': status.RUNNING } db_job = db_api.create_job(values) return resources.Job.from_db_obj(db_job)
def test_job_handler(self, mock_get_next): db_func = self.create_function() function_id = db_func.id self.assertEqual(0, db_func.count) now = datetime.utcnow() db_job = self.create_job( function_id, status=status.RUNNING, next_execution_time=now, count=2 ) job_id = db_job.id e_client = mock.Mock() mock_get_next.return_value = now + timedelta(seconds=1) periodics.handle_job(e_client) context.set_ctx(self.ctx) db_job = db_api.get_job(job_id) self.assertEqual(1, db_job.count) db_func = db_api.get_function(function_id) self.assertEqual(1, db_func.count) db_execs = db_api.get_executions(function_id=function_id) self.assertEqual(1, len(db_execs)) periodics.handle_job(e_client) context.set_ctx(self.ctx) db_job = db_api.get_job(job_id) self.assertEqual(0, db_job.count) self.assertEqual(status.DONE, db_job.status) db_func = db_api.get_function(function_id) self.assertEqual(2, db_func.count) db_execs = db_api.get_executions(function_id=function_id) self.assertEqual(2, len(db_execs))
def post(self, webhook): acl.enforce('webhook:create', context.get_ctx()) params = webhook.to_dict() if not POST_REQUIRED.issubset(set(params.keys())): raise exc.InputException( 'Required param is missing. Required: %s' % POST_REQUIRED ) LOG.info("Creating %s, params: %s", self.type, params) # Even admin user can not expose normal user's function db_api.get_function(params['function_id'], insecure=False) version = params.get('function_version', 0) if version > 0: db_api.get_function_version(params['function_id'], version) webhook_d = db_api.create_webhook(params).to_dict() return resources.Webhook.from_dict( self._add_webhook_url(webhook_d['id'], webhook_d) )
def scale_up(self, id, scale): """Scale up the containers for function execution. This is admin only operation. The load monitoring of function execution depends on the monitoring solution of underlying orchestrator. """ func_db = db_api.get_function(id) params = scale.to_dict() LOG.info('Starting to scale up function %s, params: %s', id, params) self.engine_client.scaleup_function(id, runtime_id=func_db.runtime_id, count=params['count'])
def _create_function_version(self, project_id, function_id, **kwargs): with etcd_util.get_function_version_lock(function_id) as lock: if not lock.is_acquired(): return False with db_api.transaction(): # Get latest function package md5 and version number func_db = db_api.get_function(function_id, insecure=False) if func_db.code['source'] != constants.PACKAGE_FUNCTION: raise exc.NotAllowedException( "Function versioning only allowed for %s type " "function." % constants.PACKAGE_FUNCTION ) l_md5 = func_db.code['md5sum'] l_version = func_db.latest_version if len(func_db.versions) >= constants.MAX_VERSION_NUMBER: raise exc.NotAllowedException( 'Can not exceed maximum number(%s) of versions' % constants.MAX_VERSION_NUMBER ) # Check if the latest package changed since last version changed = self.storage_provider.changed_since(project_id, function_id, l_md5, l_version) if not changed: raise exc.NotAllowedException( 'Function package not changed since the latest ' 'version %s.' % l_version ) LOG.info("Creating %s, function_id: %s, old_version: %d", self.type, function_id, l_version) # Create new version and copy package. self.storage_provider.copy(project_id, function_id, l_md5, l_version) version = db_api.increase_function_version(function_id, l_version, **kwargs) func_db.latest_version = l_version + 1 LOG.info("New version %d for function %s created.", l_version + 1, function_id) return version
def post(self, job): """Creates a new job.""" params = job.to_dict() if not POST_REQUIRED.issubset(set(params.keys())): raise exc.InputException( 'Required param is missing. Required: %s' % POST_REQUIRED) # Check the input params. first_time, next_time, count = jobs.validate_job(params) LOG.info("Creating %s, params: %s", self.type, params) with db_api.transaction(): db_api.get_function(params['function_id']) values = { 'name': params.get('name'), 'pattern': params.get('pattern'), 'first_execution_time': first_time, 'next_execution_time': next_time, 'count': count, 'function_id': params['function_id'], 'function_input': params.get('function_input') or {}, 'status': status.RUNNING } if cfg.CONF.pecan.auth_enable: values['trust_id'] = keystone_util.create_trust().id try: db_job = db_api.create_job(values) except Exception: # Delete trust before raising exception. keystone_util.delete_trust(values.get('trust_id')) raise return resources.Job.from_dict(db_job.to_dict())
def scale_down(self, id, scale): """Scale down the containers for function execution. This is admin only operation. The load monitoring of function execution depends on the monitoring solution of underlying orchestrator. """ func_db = db_api.get_function(id) params = scale.to_dict() if len(func_db.workers) <= 1: LOG.info('No need to scale down function %s', id) return LOG.info('Starting to scale down function %s, params: %s', id, params) self.engine_client.scaledown_function(id, count=params['count'])
def scaleup_function(self, ctx, function_id, runtime_id, count=1): function = db_api.get_function(function_id) worker_names = self.orchestrator.scaleup_function( function_id, identifier=runtime_id, entry=function.entry, count=count) with db_api.transaction(): for name in worker_names: worker = {'function_id': function_id, 'worker_name': name} db_api.create_function_worker(worker) LOG.info('Finished scaling up function %s.', function_id)
def get_all(self, function_id): """Get all the versions of the given function. Admin user can get all versions for the normal user's function. """ acl.enforce('function_version:get_all', context.get_ctx()) LOG.info("Getting versions for function %s.", function_id) # Getting function and versions needs to happen in a db transaction with db_api.transaction(): func_db = db_api.get_function(function_id) db_versions = func_db.versions versions = [resources.FunctionVersion.from_db_obj(v) for v in db_versions] return resources.FunctionVersions(function_versions=versions)
def delete(self, id): """Delete the specified function.""" LOG.info("Delete resource.", resource={'type': self.type, 'id': id}) with db_api.transaction(): func_db = db_api.get_function(id) if len(func_db.jobs) > 0: raise exc.NotAllowedException( 'The function is still associated with running job(s).') source = func_db.code['source'] if source == 'package': self.storage_provider.delete(context.get_ctx().projectid, id) # Delete all resources created by orchestrator asynchronously. self.engine_client.delete_function(id) # This will also delete function service mapping as well. db_api.delete_function(id)
def get(self, id): """Get function information or download function package. This method can support HTTP request using either 'Accept:application/json' or no 'Accept' header. """ ctx = context.get_ctx() acl.enforce('function:get', ctx) download = strutils.bool_from_string( pecan.request.GET.get('download', False) ) func_db = db_api.get_function(id) if not download: LOG.info("Getting function %s.", id) pecan.override_template('json') return resources.Function.from_db_obj(func_db).to_dict() LOG.info("Downloading function %s", id) source = func_db.code['source'] if source == constants.PACKAGE_FUNCTION: f = self.storage_provider.retrieve(func_db.project_id, id, func_db.code['md5sum']) elif source == constants.SWIFT_FUNCTION: container = func_db.code['swift']['container'] obj = func_db.code['swift']['object'] f = swift_util.download_object(container, obj) else: msg = 'Download image function is not allowed.' pecan.abort( status_code=405, detail=msg, headers={'Server-Error-Message': msg} ) pecan.response.app_iter = (f if isinstance(f, collections.Iterable) else FileIter(f)) pecan.response.headers['Content-Type'] = 'application/zip' pecan.response.headers['Content-Disposition'] = ( 'attachment; filename="%s"' % id )
def post(self, execution): params = execution.to_dict() LOG.info("Creating execution. [execution=%s]", params) function_id = params['function_id'] # Check if the service url is existing. try: mapping = db_api.get_function_service_mapping(function_id) LOG.debug('Found Service url for function: %s', function_id) func_url = '%s/execute' % mapping.service_url LOG.info('Invoke function %s, url: %s', function_id, func_url) r = requests.post(func_url, data=params.get('input')) params.update({ 'status': 'success', 'output': { 'result': r.json() } }) db_model = db_api.create_execution(params) return resources.Execution.from_dict(db_model.to_dict()) except exc.DBEntityNotFoundError: pass func = db_api.get_function(function_id) runtime_id = func.runtime_id params.update({'status': 'running'}) db_model = db_api.create_execution(params) self.engine_client.create_execution(db_model.id, function_id, runtime_id, input=params.get('input')) updated_db = db_api.get_execution(db_model.id) return resources.Execution.from_dict(updated_db.to_dict())
def test_delete(self, mock_package_delete, mock_engine_delete, mock_etcd_delete): db_api.increase_function_version(self.func_id, 0, description="version 1") resp = self.app.delete('/v1/functions/%s/versions/1' % self.func_id) self.assertEqual(204, resp.status_int) mock_engine_delete.assert_called_once_with(self.func_id, version=1) mock_etcd_delete.assert_called_once_with(self.func_id, version=1) mock_package_delete.assert_called_once_with( unit_base.DEFAULT_PROJECT_ID, self.func_id, None, version=1) # We need to set context as it was removed after the API call context.set_ctx(self.ctx) with db_api.transaction(): func_db = db_api.get_function(self.func_id) self.assertEqual(0, len(func_db.versions)) self.assertEqual(0, func_db.latest_version)
def get(self, id): LOG.info("Fetch function [id=%s]", id) download = strutils.bool_from_string( pecan.request.GET.get('download', False)) func_db = db_api.get_function(id) ctx = context.get_ctx() if not download: pecan.override_template('json') return resources.Function.from_dict(func_db.to_dict()).to_dict() else: f = self.storage_provider.retrieve( ctx.projectid, id, ) pecan.response.app_iter = FileIter(f) pecan.response.headers['Content-Type'] = 'application/zip' pecan.response.headers['Content-Disposition'] = ( 'attachment; filename="%s"' % os.path.basename(f.name))
def invoke(self, id, **kwargs): with db_api.transaction(): # The webhook url can be accessed without authentication, so # insecure is used here webhook_db = db_api.get_webhook(id, insecure=True) function_alias = webhook_db.function_alias if function_alias: alias = db_api.get_function_alias(function_alias, insecure=True) function_id = alias.function_id function_version = alias.function_version function_db = db_api.get_function(function_id, insecure=True) else: function_db = webhook_db.function function_id = webhook_db.function_id function_version = webhook_db.function_version trust_id = function_db.trust_id project_id = function_db.project_id LOG.info('Invoking function %s(version %s) by webhook %s', function_id, function_version, id) # Setup user context ctx = keystone_utils.create_trust_context(trust_id, project_id) context.set_ctx(ctx) params = { 'function_id': function_id, 'function_version': function_version, 'sync': False, 'input': json.dumps(kwargs), 'description': constants.EXECUTION_BY_WEBHOOK % id } execution = executions.create_execution(self.engine_client, params) pecan.response.status = 202 return {'execution_id': execution.id}
def test_create_execution_prepare_execution_exception( self, etcd_util_get_service_url_mock ): """test_create_execution_prepare_execution_exception Create execution for image type function, prepare_execution method raises exception. """ function = self.create_function() function_id = function.id runtime_id = function.runtime_id db_api.update_function( function_id, { 'code': { 'source': constants.IMAGE_FUNCTION, 'image': self.rand_name('image', prefix=self.prefix) } } ) function = db_api.get_function(function_id) execution = self.create_execution(function_id=function_id) execution_id = execution.id prepare_execution = self.orchestrator.prepare_execution prepare_execution.side_effect = exc.OrchestratorException( 'Exception in prepare_execution' ) etcd_util_get_service_url_mock.return_value = None self.default_engine.create_execution( mock.Mock(), execution_id, function_id, 0, runtime_id) execution = db_api.get_execution(execution_id) self.assertEqual(status.ERROR, execution.status) self.assertEqual('', execution.logs) self.assertEqual({'output': 'Function execution failed.'}, execution.result)
def scale_up(self, function_id, version, scale): """Scale up the workers for function version execution. This is admin only operation. The load monitoring of execution depends on the monitoring solution of underlying orchestrator. """ acl.enforce('function_version:scale_up', context.get_ctx()) func_db = db_api.get_function(function_id) # If version=0, it's equivalent to /functions/<funcion-id>/scale_up if version > 0: db_api.get_function_version(function_id, version) params = scale.to_dict() LOG.info('Starting to scale up function %s(version %s), params: %s', function_id, version, params) self.engine_client.scaleup_function(function_id, runtime_id=func_db.runtime_id, version=version, count=params['count'])
def create_execution(engine_client, params): function_id = params['function_id'] is_sync = params.get('sync', True) input = params.get('input') version = params.get('function_version', 0) func_db = db_api.get_function(function_id) runtime_id = func_db.runtime_id # Image type function does not need runtime if runtime_id: runtime_db = db_api.get_runtime(runtime_id) if runtime_db and runtime_db.status != status.AVAILABLE: raise exc.RuntimeNotAvailableException( 'Runtime %s is not available.' % func_db.runtime_id) if version > 0: if func_db.code['source'] != constants.PACKAGE_FUNCTION: raise exc.InputException( "Can not specify version for %s type function." % constants.PACKAGE_FUNCTION) # update version count version_db = db_api.get_function_version(function_id, version) pre_version_count = version_db.count _update_function_version_db(version_db.id, pre_version_count) else: pre_count = func_db.count _update_function_db(function_id, pre_count) # input in params should be a string. if input: try: params['input'] = json.loads(input) except ValueError: params['input'] = {'__function_input': input} params.update({'status': status.RUNNING}) db_model = db_api.create_execution(params) try: engine_client.create_execution(db_model.id, function_id, version, runtime_id, input=params.get('input'), is_sync=is_sync) except exc.QinlingException: # Catch RPC errors for executions: # - for RemoteError in an RPC call, the execution status would be # handled in the engine side; # - for other exceptions in an RPC call or cast, the execution status # would remain RUNNING so we should update it. db_model = db_api.get_execution(db_model.id) if db_model.status == status.RUNNING: db_model = db_api.update_execution(db_model.id, {'status': status.ERROR}) return db_model if is_sync: # The execution should already be updated by engine service for sync # execution. db_model = db_api.get_execution(db_model.id) return db_model
def handle_job(engine_client): """Execute job task with no db transactions.""" for job in db_api.get_next_jobs(timeutils.utcnow() + timedelta(seconds=3)): job_id = job.id func_id = job.function_id LOG.debug("Processing job: %s, function: %s", job_id, func_id) func_db = db_api.get_function(func_id, insecure=True) trust_id = func_db.trust_id try: # Setup context before schedule job. ctx = keystone_utils.create_trust_context(trust_id, job.project_id) context.set_ctx(ctx) if (job.count is not None and job.count > 0): job.count -= 1 # Job delete/update is done using UPDATE ... FROM ... WHERE # non-locking clause. if job.count == 0: modified = db_api.conditional_update( models.Job, { 'status': status.DONE, 'count': 0 }, { 'id': job_id, 'status': status.RUNNING }, insecure=True, ) else: next_time = jobs.get_next_execution_time( job.pattern, job.next_execution_time) modified = db_api.conditional_update( models.Job, { 'next_execution_time': next_time, 'count': job.count }, { 'id': job_id, 'next_execution_time': job.next_execution_time }, insecure=True, ) if not modified: LOG.warning( 'Job %s has been already handled by another periodic ' 'task.', job_id) continue LOG.debug("Starting to execute function %s by job %s", func_id, job_id) params = { 'function_id': func_id, 'input': job.function_input, 'sync': False, 'description': constants.EXECUTION_BY_JOB % job_id } executions.create_execution(engine_client, params) except Exception: LOG.exception("Failed to process job %s", job_id) finally: context.set_ctx(None)
def create_execution(self, ctx, execution_id, function_id, runtime_id, input=None): LOG.info( 'Creating execution. execution_id=%s, function_id=%s, ' 'runtime_id=%s, input=%s', execution_id, function_id, runtime_id, input) function = db_api.get_function(function_id) source = function.code['source'] image = None identifier = None labels = None svc_url = None # Auto scale workers if needed if source != constants.IMAGE_FUNCTION: svc_url = self.function_load_check(function_id, runtime_id) temp_url = etcd_util.get_service_url(function_id) svc_url = svc_url or temp_url if svc_url: func_url = '%s/execute' % svc_url LOG.debug( 'Found service url for function: %s, execution: %s, url: %s', function_id, execution_id, func_url) data = utils.get_request_data(CONF, function_id, execution_id, input, function.entry, function.trust_id, self.qinling_endpoint) success, res = utils.url_request(self.session, func_url, body=data) success = success and res.pop('success') LOG.debug('Finished execution %s, success: %s', execution_id, success) db_api.update_execution( execution_id, { 'status': status.SUCCESS if success else status.FAILED, 'logs': res.pop('logs', ''), 'result': res }) return if source == constants.IMAGE_FUNCTION: image = function.code['image'] identifier = ( '%s-%s' % (common.generate_unicode_uuid(dashed=False), function_id))[:63] labels = {'function_id': function_id} else: identifier = runtime_id labels = {'runtime_id': runtime_id} _, svc_url = self.orchestrator.prepare_execution( function_id, image=image, identifier=identifier, labels=labels, input=input, ) success, res = self.orchestrator.run_execution( execution_id, function_id, input=input, identifier=identifier, service_url=svc_url, entry=function.entry, trust_id=function.trust_id) logs = '' # Execution log is only available for non-image source execution. if svc_url: logs = res.pop('logs', '') success = success and res.pop('success') else: # If the function is created from docker image, the result is # direct output, here we convert to a dict to fit into the db # schema. res = {'output': res} LOG.debug('Finished execution %s, success: %s', execution_id, success) db_api.update_execution( execution_id, { 'status': status.SUCCESS if success else status.FAILED, 'logs': logs, 'result': res })
def create_execution(self, ctx, execution_id, function_id, runtime_id, input=None): LOG.info( 'Creating execution. execution_id=%s, function_id=%s, ' 'runtime_id=%s, input=%s', execution_id, function_id, runtime_id, input) # FIXME(kong): Make the transaction range smaller. with db_api.transaction(): execution = db_api.get_execution(execution_id) function = db_api.get_function(function_id) if function.service: func_url = '%s/execute' % function.service.service_url LOG.debug('Found service url for function: %s, url: %s', function_id, func_url) data = {'input': input, 'execution_id': execution_id} r = self.session.post(func_url, json=data) res = r.json() LOG.debug('Finished execution %s', execution_id) success = res.pop('success') execution.status = status.SUCCESS if success else status.FAILED execution.logs = res.pop('logs', '') execution.output = res return source = function.code['source'] image = None identifier = None labels = None if source == constants.IMAGE_FUNCTION: image = function.code['image'] identifier = ('%s-%s' % (common.generate_unicode_uuid(dashed=False), function_id))[:63] labels = {'function_id': function_id} else: identifier = runtime_id labels = {'runtime_id': runtime_id} worker_name, service_url = self.orchestrator.prepare_execution( function_id, image=image, identifier=identifier, labels=labels, input=input, entry=function.entry, trust_id=function.trust_id) output = self.orchestrator.run_execution( execution_id, function_id, input=input, identifier=identifier, service_url=service_url, ) logs = '' # Execution log is only available for non-image source execution. if service_url: logs = output.pop('logs', '') success = output.pop('success') else: # If the function is created from docker image, the output is # direct output, here we convert to a dict to fit into the db # schema. output = {'output': output} success = True LOG.debug('Finished execution. execution_id=%s, output=%s', execution_id, output) execution.output = output execution.logs = logs execution.status = status.SUCCESS if success else status.FAILED # No service is created in orchestrator for single container. if not image: mapping = { 'function_id': function_id, 'service_url': service_url, } db_api.create_function_service_mapping(mapping) worker = { 'function_id': function_id, 'worker_name': worker_name } db_api.create_function_worker(worker)
def delete(self, id): """Delete the specified function. Delete function will also delete all its versions. """ LOG.info("Delete function %s.", id) with db_api.transaction(): func_db = db_api.get_function(id) if len(func_db.jobs) > 0: raise exc.NotAllowedException( 'The function is still associated with running job(s).' ) if len(func_db.webhooks) > 0: raise exc.NotAllowedException( 'The function is still associated with webhook(s).' ) # Even admin user can not delete other project's function because # the trust associated can only be removed by function owner. if func_db.project_id != context.get_ctx().projectid: raise exc.NotAllowedException( 'Function can only be deleted by its owner.' ) # Delete trust if needed if func_db.trust_id: keystone_util.delete_trust(func_db.trust_id) for version_db in func_db.versions: # Delete all resources created by orchestrator asynchronously. self.engine_client.delete_function( id, version=version_db.version_number ) # Delete etcd keys etcd_util.delete_function( id, version=version_db.version_number ) # Delete function version packages. Versions is only supported # for package type function. self.storage_provider.delete( func_db.project_id, id, None, version=version_db.version_number ) # Delete resources for function version 0(func_db.versions==[]) self.engine_client.delete_function(id) etcd_util.delete_function(id) source = func_db.code['source'] if source == constants.PACKAGE_FUNCTION: self.storage_provider.delete(func_db.project_id, id, func_db.code['md5sum']) # This will also delete function service mapping and function # versions as well. db_api.delete_function(id)
def test_create_execution_image_type_function(self, mock_svc_url): """Create 2 executions for an image type function.""" function = self.create_function() function_id = function.id runtime_id = function.runtime_id db_api.update_function( function_id, { 'code': { 'source': constants.IMAGE_FUNCTION, 'image': self.rand_name('image', prefix=self.prefix) } }) function = db_api.get_function(function_id) execution_1 = self.create_execution(function_id=function_id) execution_1_id = execution_1.id execution_2 = self.create_execution(function_id=function_id) execution_2_id = execution_2.id mock_svc_url.return_value = None self.orchestrator.prepare_execution.return_value = (mock.Mock(), None) self.orchestrator.run_execution.side_effect = [(True, { 'duration': 5, 'logs': 'fake log' }), (False, { 'duration': 0, 'error': 'Function execution failed.' })] # Create two executions, with different results self.default_engine.create_execution(mock.Mock(), execution_1_id, function_id, 0, runtime_id) self.default_engine.create_execution(mock.Mock(), execution_2_id, function_id, 0, runtime_id, input='input') get_service_url_calls = [ mock.call(function_id, 0), mock.call(function_id, 0) ] mock_svc_url.assert_has_calls(get_service_url_calls) prepare_calls = [ mock.call(function_id, 0, rlimit=self.rlimit, image=function.code['image'], identifier=mock.ANY, labels=None, input=None), mock.call(function_id, 0, rlimit=self.rlimit, image=function.code['image'], identifier=mock.ANY, labels=None, input='input') ] self.orchestrator.prepare_execution.assert_has_calls(prepare_calls) run_calls = [ mock.call(execution_1_id, function_id, 0, rlimit=None, input=None, identifier=mock.ANY, service_url=None, entry=function.entry, trust_id=function.trust_id), mock.call(execution_2_id, function_id, 0, rlimit=None, input='input', identifier=mock.ANY, service_url=None, entry=function.entry, trust_id=function.trust_id) ] self.orchestrator.run_execution.assert_has_calls(run_calls) execution_1 = db_api.get_execution(execution_1_id) execution_2 = db_api.get_execution(execution_2_id) self.assertEqual(status.SUCCESS, execution_1.status) self.assertEqual('fake log', execution_1.logs) self.assertEqual({"duration": 5}, execution_1.result) self.assertEqual(status.FAILED, execution_2.status) self.assertEqual('', execution_2.logs) self.assertEqual({ 'duration': 0, 'error': 'Function execution failed.' }, execution_2.result)
def put(self, id, **kwargs): """Update function. - Function can not being used by job. - Function can not being executed. - (TODO)Function status should be changed so no execution will create when function is updating. """ values = {} try: for key in UPDATE_ALLOWED: if kwargs.get(key) is not None: if key == "code": kwargs[key] = json.loads(kwargs[key]) values.update({key: kwargs[key]}) except Exception as e: raise exc.InputException("Invalid input, %s" % str(e)) LOG.info('Update function %s, params: %s', id, values) ctx = context.get_ctx() if values.get('timeout'): common.validate_int_in_range('timeout', values['timeout'], CONF.resource_limits.min_timeout, CONF.resource_limits.max_timeout) db_update_only = set(['name', 'description', 'timeout']) if set(values.keys()).issubset(db_update_only): func_db = db_api.update_function(id, values) else: source = values.get('code', {}).get('source') md5sum = values.get('code', {}).get('md5sum') cpu = values.get('cpu') memory_size = values.get('memory_size') # Check cpu and memory_size values when updating. if cpu is not None: common.validate_int_in_range('cpu', values['cpu'], CONF.resource_limits.min_cpu, CONF.resource_limits.max_cpu) if memory_size is not None: common.validate_int_in_range('memory', values['memory_size'], CONF.resource_limits.min_memory, CONF.resource_limits.max_memory) with db_api.transaction(): pre_func = db_api.get_function(id) if len(pre_func.jobs) > 0: raise exc.NotAllowedException( 'The function is still associated with running job(s).' ) pre_source = pre_func.code['source'] pre_md5sum = pre_func.code.get('md5sum') if source and source != pre_source: raise exc.InputException( "The function code type can not be changed.") if pre_source == constants.IMAGE_FUNCTION: raise exc.InputException( "The image type function code can not be changed.") # Package type function. 'code' and 'entry' make sense only if # 'package' is provided package_updated = False if (pre_source == constants.PACKAGE_FUNCTION and values.get('package') is not None): if md5sum and md5sum == pre_md5sum: raise exc.InputException( "The function code checksum is not changed.") # Update the package data. data = values['package'].file.read() package_updated, md5sum = self.storage_provider.store( ctx.projectid, id, data, md5sum=md5sum) values.setdefault('code', {}).update({ "md5sum": md5sum, "source": pre_source }) values.pop('package') # Swift type function if (pre_source == constants.SWIFT_FUNCTION and "swift" in values.get('code', {})): swift_info = values['code']["swift"] if not (swift_info.get('container') or swift_info.get('object')): raise exc.InputException( "Either container or object must be provided for " "swift type function update.") new_swift_info = pre_func.code['swift'] new_swift_info.update(swift_info) self._check_swift(new_swift_info.get('container'), new_swift_info.get('object')) values['code'] = { "source": pre_source, "swift": new_swift_info } # Delete allocated resources in orchestrator and etcd. self.engine_client.delete_function(id) etcd_util.delete_function(id) func_db = db_api.update_function(id, values) # Delete the old function package if needed if package_updated: self.storage_provider.delete(ctx.projectid, id, pre_md5sum) pecan.response.status = 200 return resources.Function.from_db_obj(func_db).to_dict()
def create_execution(self, ctx, execution_id, function_id, function_version, runtime_id, input=None): LOG.info( 'Creating execution. execution_id=%s, function_id=%s, ' 'function_version=%s, runtime_id=%s, input=%s', execution_id, function_id, function_version, runtime_id, input) function = db_api.get_function(function_id) source = function.code['source'] rlimit = {'cpu': function.cpu, 'memory_size': function.memory_size} image = None identifier = None labels = None svc_url = None is_image_source = source == constants.IMAGE_FUNCTION # Auto scale workers if needed if not is_image_source: try: svc_url = self.function_load_check(function_id, function_version, runtime_id) except exc.OrchestratorException as e: utils.handle_execution_exception(execution_id, str(e)) return temp_url = etcd_util.get_service_url(function_id, function_version) svc_url = svc_url or temp_url if svc_url: func_url = '%s/execute' % svc_url LOG.debug( 'Found service url for function: %s(version %s), execution: ' '%s, url: %s', function_id, function_version, execution_id, func_url) data = utils.get_request_data(CONF, function_id, function_version, execution_id, rlimit, input, function.entry, function.trust_id, self.qinling_endpoint) success, res = utils.url_request(self.session, func_url, body=data) utils.finish_execution(execution_id, success, res, is_image_source=is_image_source) return if source == constants.IMAGE_FUNCTION: image = function.code['image'] # Be consistent with k8s naming convention identifier = ( '%s-%s' % (common.generate_unicode_uuid(dashed=False), function_id))[:63] else: identifier = runtime_id labels = {'runtime_id': runtime_id} try: # For image function, it will be executed inside this method; for # package type function it only sets up underlying resources and # get a service url. If the service url is already created # beforehand, nothing happens. _, svc_url = self.orchestrator.prepare_execution( function_id, function_version, rlimit=rlimit, image=image, identifier=identifier, labels=labels, input=input, ) except exc.OrchestratorException as e: utils.handle_execution_exception(execution_id, str(e)) return # For image type function, read the worker log; For package type # function, invoke and get log success, res = self.orchestrator.run_execution( execution_id, function_id, function_version, rlimit=rlimit if svc_url else None, input=input, identifier=identifier, service_url=svc_url, entry=function.entry, trust_id=function.trust_id) utils.finish_execution(execution_id, success, res, is_image_source=is_image_source)