def put(self, job_id): # we use HTTP/HTTPS PUT method to redo a job. Regularly PUT method # requires a request body, but considering the job redo operation # doesn't need more information other than job id, we will issue # this request without a request body. context = t_context.extract_context_from_environ() if not policy.enforce(context, policy.ADMIN_API_JOB_REDO): return utils.format_api_error(403, _('Unauthorized to redo a job')) try: db_api.get_job_from_log(context, job_id) return utils.format_api_error( 400, _('Job %(job_id)s is from job log') % {'job_id': job_id}) except Exception: try: job = db_api.get_job(context, job_id) except t_exc.ResourceNotFound: return utils.format_api_error( 404, _('Job %(job_id)s not found') % {'job_id': job_id}) try: # if status = RUNNING, notify user this new one and then exit if job['status'] == constants.JS_Running: return utils.format_api_error( 400, (_("Can't redo job %(job_id)s which is running") % { 'job_id': job['id'] })) # if status = SUCCESS, notify user this new one and then exit elif job['status'] == constants.JS_Success: msg = ( _("Can't redo job %(job_id)s which had run successfully") % { 'job_id': job['id'] }) return utils.format_api_error(400, msg) # if job status = FAIL or job status = NEW, redo it immediately self.xjob_handler.invoke_method(context, job['project_id'], constants.job_handles[job['type']], job['type'], job['resource_id']) except Exception as e: LOG.exception('Failed to redo the job: ' '%(exception)s ', {'exception': e}) return utils.format_api_error(500, _('Failed to redo the job'))
def delete(self, job_id): # delete a job from the database. If the job is running, the delete # operation will fail. In other cases, job will be deleted directly. context = t_context.extract_context_from_environ() if not policy.enforce(context, policy.ADMIN_API_JOB_DELETE): return utils.format_api_error(403, _('Unauthorized to delete a job')) try: db_api.get_job_from_log(context, job_id) return utils.format_api_error( 400, _('Job %(job_id)s is from job log') % {'job_id': job_id}) except Exception: try: job = db_api.get_job(context, job_id) except t_exc.ResourceNotFound: return utils.format_api_error( 404, _('Job %(job_id)s not found') % {'job_id': job_id}) try: # if job status = RUNNING, notify user this new one, delete # operation fails. if job['status'] == constants.JS_Running: return utils.format_api_error( 400, (_('Failed to delete the running job %(job_id)s') % { "job_id": job_id })) # if job status = SUCCESS, move the job entry to job log table, # then delete it from job table. elif job['status'] == constants.JS_Success: db_api.finish_job(context, job_id, True, timeutils.utcnow()) pecan.response.status = 200 return {} db_api.delete_job(context, job_id) pecan.response.status = 200 return {} except Exception as e: LOG.exception('Failed to delete the job: ' '%(exception)s ', {'exception': e}) return utils.format_api_error(500, _('Failed to delete the job'))
def put(self, job_id): # we use HTTP/HTTPS PUT method to redo a job. Regularly PUT method # requires a request body, but considering the job redo operation # doesn't need more information other than job id, we will issue # this request without a request body. context = t_context.extract_context_from_environ() if not policy.enforce(context, policy.ADMIN_API_JOB_REDO): return utils.format_api_error( 403, _('Unauthorized to redo a job')) try: db_api.get_job_from_log(context, job_id) return utils.format_api_error( 400, _('Job %(job_id)s is from job log') % {'job_id': job_id}) except Exception: try: job = db_api.get_job(context, job_id) except t_exc.ResourceNotFound: return utils.format_api_error( 404, _('Job %(job_id)s not found') % {'job_id': job_id}) try: # if status = RUNNING, notify user this new one and then exit if job['status'] == constants.JS_Running: return utils.format_api_error( 400, (_("Can't redo job %(job_id)s which is running") % {'job_id': job['id']})) # if status = SUCCESS, notify user this new one and then exit elif job['status'] == constants.JS_Success: msg = (_("Can't redo job %(job_id)s which had run successfully" ) % {'job_id': job['id']}) return utils.format_api_error(400, msg) # if job status = FAIL or job status = NEW, redo it immediately self.xjob_handler.invoke_method(context, job['project_id'], constants.job_handles[job['type']], job['type'], job['resource_id']) except Exception as e: LOG.exception('Failed to redo the job: ' '%(exception)s ', {'exception': e}) return utils.format_api_error( 500, _('Failed to redo the job'))
def get_one(self, id, **kwargs): """the return value may vary according to the value of id :param id: 1) if id = 'schemas', return job schemas 2) if id = 'detail', return all jobs 3) if id = $job_id, return detailed single job info :return: return value is decided by id parameter """ context = t_context.extract_context_from_environ() job_resource_map = constants.job_resource_map if not policy.enforce(context, policy.ADMIN_API_JOB_SCHEMA_LIST): return utils.format_api_error( 403, _('Unauthorized to show job information')) if id == 'schemas': job_schemas = [] for job_type in job_resource_map.keys(): job = {} resource = [] for resource_type, resource_id in job_resource_map[job_type]: resource.append(resource_id) job['resource'] = resource job['type'] = job_type job_schemas.append(job) return {'schemas': job_schemas} if id == 'detail': return self.get_all(**kwargs) try: job = db_api.get_job(context, id) return {'job': self._get_more_readable_job(job)} except Exception: try: job = db_api.get_job_from_log(context, id) return {'job': self._get_more_readable_job(job)} except t_exc.ResourceNotFound: return utils.format_api_error( 404, _('Resource not found'))
def delete(self, job_id): # delete a job from the database. If the job is running, the delete # operation will fail. In other cases, job will be deleted directly. context = t_context.extract_context_from_environ() if not policy.enforce(context, policy.ADMIN_API_JOB_DELETE): return utils.format_api_error( 403, _('Unauthorized to delete a job')) try: db_api.get_job_from_log(context, job_id) return utils.format_api_error( 400, _('Job %(job_id)s is from job log') % {'job_id': job_id}) except Exception: try: job = db_api.get_job(context, job_id) except t_exc.ResourceNotFound: return utils.format_api_error( 404, _('Job %(job_id)s not found') % {'job_id': job_id}) try: # if job status = RUNNING, notify user this new one, delete # operation fails. if job['status'] == constants.JS_Running: return utils.format_api_error( 400, (_('Failed to delete the running job %(job_id)s') % {"job_id": job_id})) # if job status = SUCCESS, move the job entry to job log table, # then delete it from job table. elif job['status'] == constants.JS_Success: db_api.finish_job(context, job_id, True, timeutils.utcnow()) pecan.response.status = 200 return {} db_api.delete_job(context, job_id) pecan.response.status = 200 return {} except Exception as e: LOG.exception('Failed to delete the job: ' '%(exception)s ', {'exception': e}) return utils.format_api_error( 500, _('Failed to delete the job'))
def get_all(self, **kwargs): """Get all the jobs. Using filters, only get a subset of jobs. :param kwargs: job filters :return: a list of jobs """ context = t_context.extract_context_from_environ() if not policy.enforce(context, policy.ADMIN_API_JOB_LIST): return utils.format_api_error( 403, _('Unauthorized to show all jobs')) # check limit and marker, default value -1 means no pagination _limit = kwargs.pop('limit', -1) try: limit = int(_limit) limit = utils.get_pagination_limit(limit) except ValueError as e: LOG.exception('Failed to convert pagination limit to an integer: ' '%(exception)s ', {'exception': e}) msg = (_("Limit should be an integer or a valid literal " "for int() rather than '%s'") % _limit) return utils.format_api_error(400, msg) marker = kwargs.pop('marker', None) sorts = [('timestamp', 'desc'), ('id', 'desc')] is_valid_filter, filters = self._get_filters(kwargs) if not is_valid_filter: msg = (_('Unsupported filter type: %(filters)s') % { 'filters': ', '.join( [filter_name for filter_name in filters]) }) return utils.format_api_error(400, msg) # project ID from client should be equal to the one from # context, since only the project ID in which the user # is authorized will be used as the filter. filters['project_id'] = context.project_id filters = [{'key': key, 'comparator': 'eq', 'value': value} for key, value in six.iteritems(filters)] try: if marker is not None: try: # verify whether the marker is effective db_api.get_job(context, marker) jobs = db_api.list_jobs(context, filters, sorts, limit, marker) jobs_from_log = [] if len(jobs) < limit: jobs_from_log = db_api.list_jobs_from_log( context, filters, sorts, limit - len(jobs), None) job_collection = jobs + jobs_from_log except t_exc.ResourceNotFound: try: db_api.get_job_from_log(context, marker) jobs_from_log = db_api.list_jobs_from_log( context, filters, sorts, limit, marker) job_collection = jobs_from_log except t_exc.ResourceNotFound: msg = (_('Invalid marker: %(marker)s') % {'marker': marker}) return utils.format_api_error(400, msg) else: jobs = db_api.list_jobs(context, filters, sorts, limit, marker) jobs_from_log = [] if len(jobs) < limit: jobs_from_log = db_api.list_jobs_from_log( context, filters, sorts, limit - len(jobs), None) job_collection = jobs + jobs_from_log # add link links = [] if len(job_collection) >= limit: marker = job_collection[-1]['id'] base = constants.JOB_PATH link = "%s?limit=%s&marker=%s" % (base, limit, marker) links.append({"rel": "next", "href": link}) result = {'jobs': [self._get_more_readable_job(job) for job in job_collection]} if links: result['jobs_links'] = links return result except Exception as e: LOG.exception('Failed to show all asynchronous jobs: ' '%(exception)s ', {'exception': e}) return utils.format_api_error( 500, _('Failed to show all asynchronous jobs'))
def test_redo_job(self): for job_type in self.all_job_types: job = self._prepare_job_element(job_type) jobs = [ # create an entirely new job { "job": job, "expected_error": 200 }, ] self._test_and_check(jobs) response = self.app.get('/v1.0/jobs') return_job = response.json jobs = return_job['jobs'] # redo a new job for job in jobs: response_1 = self.app.put('/v1.0/jobs/%(id)s' % {'id': job['id']}, expect_errors=True) self.assertEqual(response_1.status_int, 200) response_2 = self.app.put('/v1.0/jobs/123', expect_errors=True) self.assertEqual(response_2.status_int, 404) # redo a running job job_type_3 = constants.JT_NETWORK_UPDATE job_3 = self._prepare_job_element(job_type_3) resource_id_3 = '#'.join([job_3['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type_3]]) job_running_3 = db_api.register_job(self.context, job_3['project_id'], job_type_3, resource_id_3) self.assertEqual(constants.JS_Running, job_running_3['status']) response_3 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': job_running_3['id']}, expect_errors=True) self.assertEqual(response_3.status_int, 400) # redo a failed job job_type_4 = constants.JT_NETWORK_UPDATE job_4 = self._prepare_job_element(job_type_4) job_dict_4 = { "job": job_4, "expected_error": 200 } response_4 = self.app.post_json('/v1.0/jobs', dict(job=job_dict_4['job']), expect_errors=True) return_job_4 = response_4.json self.assertEqual(response_4.status_int, 200) db_api.finish_job(self.context, return_job_4['job']['id'], False, timeutils.utcnow()) job_fail_4 = db_api.get_job(self.context, return_job_4['job']['id']) self.assertEqual(constants.JS_Fail, job_fail_4['status']) response_5 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': return_job_4['job']['id']}, expect_errors=True) self.assertEqual(response_5.status_int, 200) # redo a successful job job_type_6 = constants.JT_NETWORK_UPDATE job_6 = self._prepare_job_element(job_type_6) job_dict_6 = { "job": job_6, "expected_error": 200 } response_6 = self.app.post_json('/v1.0/jobs', dict(job=job_dict_6['job']), expect_errors=True) return_job_6 = response_6.json with self.context.session.begin(): job_dict = {'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid()} core.update_resource(self.context, models.AsyncJob, return_job_6['job']['id'], job_dict) job_succ_6 = db_api.get_job(self.context, return_job_6['job']['id']) self.assertEqual(constants.JS_Success, job_succ_6['status']) response_7 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': return_job_6['job']['id']}, expect_errors=True) self.assertEqual(response_7.status_int, 400)
def test_post(self, mock_context): mock_context.return_value = self.context # cover all job types for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) kw_job = {'job': job} # failure case, only admin can create the job self.context.is_admin = False res = self.controller.post(**kw_job) self._validate_error_code(res, 403) self.context.is_admin = True # failure case, request body not found kw_job_1 = {'job_1': job} res = self.controller.post(**kw_job_1) self._validate_error_code(res, 400) # failure case, wrong job type parameter job_type_backup = job.pop('type') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = '' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = job_type_backup + '_1' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = job_type_backup # failure case, wrong resource parameter job_resource_backup = job.pop('resource') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['resource'] = copy.deepcopy(job_resource_backup) job['resource'].popitem() res = self.controller.post(**kw_job) self._validate_error_code(res, 400) fake_resource = 'fake_resource' job['resource'][fake_resource] = fake_resource res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['resource'] = job_resource_backup # failure case, wrong project id parameter project_id_backup = job.pop('project_id') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = '' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = uuidutils.generate_uuid() res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = project_id_backup # successful case, create an entirely new job. Because the job # status returned from controller has been formatted, so we not # only validate the database records, but also validate the return # value of the controller. job_1 = self.controller.post(**kw_job)['job'] job_in_db_1 = db_api.get_job(self.context, job_1['id']) self.assertEqual(job_type, job_in_db_1['type']) self.assertEqual(job['project_id'], job_in_db_1['project_id']) self.assertEqual(constants.JS_New, job_in_db_1['status']) self.assertEqual('NEW', job_1['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_1['resource'])) self.assertFalse('resource_id' in job_1) self.assertFalse('extra_id' in job_1) db_api.delete_job(self.context, job_1['id']) # successful case, target job already exists in the job table # and its status is NEW, then this newer job will be picked by # job handler. job_2 = self.controller.post(**kw_job)['job'] job_in_db_2 = db_api.get_job(self.context, job_2['id']) job_3 = self.controller.post(**kw_job)['job'] job_in_db_3 = db_api.get_job(self.context, job_3['id']) self.assertEqual(job_type, job_in_db_2['type']) self.assertEqual(job['project_id'], job_in_db_2['project_id']) self.assertEqual(constants.JS_New, job_in_db_2['status']) self.assertEqual('NEW', job_2['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_2['resource'])) self.assertFalse('resource_id' in job_2) self.assertFalse('extra_id' in job_2) self.assertEqual(job_type, job_in_db_3['type']) self.assertEqual(job['project_id'], job_in_db_3['project_id']) self.assertEqual(constants.JS_New, job_in_db_3['status']) self.assertEqual('NEW', job_3['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_3['resource'])) self.assertFalse('resource_id' in job_3) self.assertFalse('extra_id' in job_3) db_api.finish_job(self.context, job_3['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_3['id'])
def test_delete(self, mock_context): mock_context.return_value = self.context # cover all job types. # each 'for' loop adds one item in job log table, we set count variable # to record dynamic total job entries in job log table. count = 1 for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) resource_id = '#'.join([ job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type] ]) # failure case, only admin can delete the job job_1 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.context.is_admin = False res = self.controller.delete(job_1['id']) self._validate_error_code(res, 403) self.context.is_admin = True db_api.delete_job(self.context, job_1['id']) # failure case, job not found res = self.controller.delete(-123) self._validate_error_code(res, 404) # failure case, delete a running job job_2 = db_api.register_job(self.context, job['project_id'], job_type, resource_id) job = db_api.get_job(self.context, job_2['id']) res = self.controller.delete(job_2['id']) self._validate_error_code(res, 400) # finish the job and delete it db_api.finish_job(self.context, job_2['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_2['id']) # successful case, delete a successful job. successful job from # job log can't be deleted, here this successful job is from # job table. job_3 = self._prepare_job_element(job_type) resource_id_3 = '#'.join([ job_3['resource'][resource_id_3] for resource_type_3, resource_id_3 in self.job_resource_map[job_type] ]) job_4 = db_api.new_job(self.context, job_3['project_id'], job_type, resource_id_3) with self.context.session.begin(): job_dict = { 'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid() } core.update_resource(self.context, models.AsyncJob, job_4['id'], job_dict) job_4_succ = db_api.get_job(self.context, job_4['id']) self.controller.delete(job_4['id']) filters_job_4 = [{ 'key': 'type', 'comparator': 'eq', 'value': job_4_succ['type'] }, { 'key': 'status', 'comparator': 'eq', 'value': job_4_succ['status'] }, { 'key': 'resource_id', 'comparator': 'eq', 'value': job_4_succ['resource_id'] }, { 'key': 'extra_id', 'comparator': 'eq', 'value': job_4_succ['extra_id'] }] self.assertEqual( 0, len(db_api.list_jobs(self.context, filters_job_4))) self.assertEqual(count, len(db_api.list_jobs_from_log(self.context))) count = count + 1 # successful case, delete a new job job_5 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.controller.delete(job_5['id']) filters_job_5 = [{ 'key': 'type', 'comparator': 'eq', 'value': job_5['type'] }, { 'key': 'status', 'comparator': 'eq', 'value': job_5['status'] }, { 'key': 'resource_id', 'comparator': 'eq', 'value': job_5['resource_id'] }, { 'key': 'extra_id', 'comparator': 'eq', 'value': job_5['extra_id'] }] self.assertEqual( 0, len(db_api.list_jobs(self.context, filters_job_5))) # successful case, delete a failed job job_6 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) db_api.finish_job(self.context, job_6['id'], False, timeutils.utcnow()) job_6_failed = db_api.get_job(self.context, job_6['id']) self.controller.delete(job_6['id']) filters_job_6 = [{ 'key': 'type', 'comparator': 'eq', 'value': job_6_failed['type'] }, { 'key': 'status', 'comparator': 'eq', 'value': job_6_failed['status'] }, { 'key': 'resource_id', 'comparator': 'eq', 'value': job_6_failed['resource_id'] }, { 'key': 'extra_id', 'comparator': 'eq', 'value': job_6_failed['extra_id'] }] self.assertEqual( 0, len(db_api.list_jobs(self.context, filters_job_6)))
def test_delete(self, mock_context): mock_context.return_value = self.context # cover all job types. # each 'for' loop adds one item in job log table, we set count variable # to record dynamic total job entries in job log table. count = 1 for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) resource_id = '#'.join([job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type]]) # failure case, only admin can delete the job job_1 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.context.is_admin = False res = self.controller.delete(job_1['id']) self._validate_error_code(res, 403) self.context.is_admin = True db_api.delete_job(self.context, job_1['id']) # failure case, job not found res = self.controller.delete(-123) self._validate_error_code(res, 404) # failure case, delete a running job job_2 = db_api.register_job(self.context, job['project_id'], job_type, resource_id) job = db_api.get_job(self.context, job_2['id']) res = self.controller.delete(job_2['id']) self._validate_error_code(res, 400) # finish the job and delete it db_api.finish_job(self.context, job_2['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_2['id']) # successful case, delete a successful job. successful job from # job log can't be deleted, here this successful job is from # job table. job_3 = self._prepare_job_element(job_type) resource_id_3 = '#'.join([job_3['resource'][resource_id_3] for resource_type_3, resource_id_3 in self.job_resource_map[job_type]]) job_4 = db_api.new_job(self.context, job_3['project_id'], job_type, resource_id_3) with self.context.session.begin(): job_dict = {'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid()} core.update_resource(self.context, models.AsyncJob, job_4['id'], job_dict) job_4_succ = db_api.get_job(self.context, job_4['id']) self.controller.delete(job_4['id']) filters_job_4 = [ {'key': 'type', 'comparator': 'eq', 'value': job_4_succ['type']}, {'key': 'status', 'comparator': 'eq', 'value': job_4_succ['status']}, {'key': 'resource_id', 'comparator': 'eq', 'value': job_4_succ['resource_id']}, {'key': 'extra_id', 'comparator': 'eq', 'value': job_4_succ['extra_id']}] self.assertEqual(0, len(db_api.list_jobs(self.context, filters_job_4))) self.assertEqual(count, len(db_api.list_jobs_from_log(self.context))) count = count + 1 # successful case, delete a new job job_5 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.controller.delete(job_5['id']) filters_job_5 = [ {'key': 'type', 'comparator': 'eq', 'value': job_5['type']}, {'key': 'status', 'comparator': 'eq', 'value': job_5['status']}, {'key': 'resource_id', 'comparator': 'eq', 'value': job_5['resource_id']}, {'key': 'extra_id', 'comparator': 'eq', 'value': job_5['extra_id']}] self.assertEqual(0, len(db_api.list_jobs(self.context, filters_job_5))) # successful case, delete a failed job job_6 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) db_api.finish_job(self.context, job_6['id'], False, timeutils.utcnow()) job_6_failed = db_api.get_job(self.context, job_6['id']) self.controller.delete(job_6['id']) filters_job_6 = [ {'key': 'type', 'comparator': 'eq', 'value': job_6_failed['type']}, {'key': 'status', 'comparator': 'eq', 'value': job_6_failed['status']}, {'key': 'resource_id', 'comparator': 'eq', 'value': job_6_failed['resource_id']}, {'key': 'extra_id', 'comparator': 'eq', 'value': job_6_failed['extra_id']}] self.assertEqual(0, len(db_api.list_jobs(self.context, filters_job_6)))