def delete(self, job_id): # delete a job from the database. If the job is running, the delete # operation will fail. In other cases, job will be deleted directly. context = t_context.extract_context_from_environ() if not policy.enforce(context, policy.ADMIN_API_JOB_DELETE): return utils.format_api_error(403, _('Unauthorized to delete a job')) try: db_api.get_job_from_log(context, job_id) return utils.format_api_error( 400, _('Job %(job_id)s is from job log') % {'job_id': job_id}) except Exception: try: job = db_api.get_job(context, job_id) except t_exc.ResourceNotFound: return utils.format_api_error( 404, _('Job %(job_id)s not found') % {'job_id': job_id}) try: # if job status = RUNNING, notify user this new one, delete # operation fails. if job['status'] == constants.JS_Running: return utils.format_api_error( 400, (_('Failed to delete the running job %(job_id)s') % { "job_id": job_id })) # if job status = SUCCESS, move the job entry to job log table, # then delete it from job table. elif job['status'] == constants.JS_Success: db_api.finish_job(context, job_id, True, timeutils.utcnow()) pecan.response.status = 200 return {} db_api.delete_job(context, job_id) pecan.response.status = 200 return {} except Exception as e: LOG.exception('Failed to delete the job: ' '%(exception)s ', {'exception': e}) return utils.format_api_error(500, _('Failed to delete the job'))
def delete(self, job_id): # delete a job from the database. If the job is running, the delete # operation will fail. In other cases, job will be deleted directly. context = t_context.extract_context_from_environ() if not policy.enforce(context, policy.ADMIN_API_JOB_DELETE): return utils.format_api_error( 403, _('Unauthorized to delete a job')) try: db_api.get_job_from_log(context, job_id) return utils.format_api_error( 400, _('Job %(job_id)s is from job log') % {'job_id': job_id}) except Exception: try: job = db_api.get_job(context, job_id) except t_exc.ResourceNotFound: return utils.format_api_error( 404, _('Job %(job_id)s not found') % {'job_id': job_id}) try: # if job status = RUNNING, notify user this new one, delete # operation fails. if job['status'] == constants.JS_Running: return utils.format_api_error( 400, (_('Failed to delete the running job %(job_id)s') % {"job_id": job_id})) # if job status = SUCCESS, move the job entry to job log table, # then delete it from job table. elif job['status'] == constants.JS_Success: db_api.finish_job(context, job_id, True, timeutils.utcnow()) pecan.response.status = 200 return {} db_api.delete_job(context, job_id) pecan.response.status = 200 return {} except Exception as e: LOG.exception('Failed to delete the job: ' '%(exception)s ', {'exception': e}) return utils.format_api_error( 500, _('Failed to delete the job'))
def test_redo_job(self): for job_type in self.all_job_types: job = self._prepare_job_element(job_type) jobs = [ # create an entirely new job { "job": job, "expected_error": 200 }, ] self._test_and_check(jobs) response = self.app.get('/v1.0/jobs') return_job = response.json jobs = return_job['jobs'] # redo a new job for job in jobs: response_1 = self.app.put('/v1.0/jobs/%(id)s' % {'id': job['id']}, expect_errors=True) self.assertEqual(response_1.status_int, 200) response_2 = self.app.put('/v1.0/jobs/123', expect_errors=True) self.assertEqual(response_2.status_int, 404) # redo a running job job_type_3 = constants.JT_NETWORK_UPDATE job_3 = self._prepare_job_element(job_type_3) resource_id_3 = '#'.join([job_3['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type_3]]) job_running_3 = db_api.register_job(self.context, job_3['project_id'], job_type_3, resource_id_3) self.assertEqual(constants.JS_Running, job_running_3['status']) response_3 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': job_running_3['id']}, expect_errors=True) self.assertEqual(response_3.status_int, 400) # redo a failed job job_type_4 = constants.JT_NETWORK_UPDATE job_4 = self._prepare_job_element(job_type_4) job_dict_4 = { "job": job_4, "expected_error": 200 } response_4 = self.app.post_json('/v1.0/jobs', dict(job=job_dict_4['job']), expect_errors=True) return_job_4 = response_4.json self.assertEqual(response_4.status_int, 200) db_api.finish_job(self.context, return_job_4['job']['id'], False, timeutils.utcnow()) job_fail_4 = db_api.get_job(self.context, return_job_4['job']['id']) self.assertEqual(constants.JS_Fail, job_fail_4['status']) response_5 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': return_job_4['job']['id']}, expect_errors=True) self.assertEqual(response_5.status_int, 200) # redo a successful job job_type_6 = constants.JT_NETWORK_UPDATE job_6 = self._prepare_job_element(job_type_6) job_dict_6 = { "job": job_6, "expected_error": 200 } response_6 = self.app.post_json('/v1.0/jobs', dict(job=job_dict_6['job']), expect_errors=True) return_job_6 = response_6.json with self.context.session.begin(): job_dict = {'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid()} core.update_resource(self.context, models.AsyncJob, return_job_6['job']['id'], job_dict) job_succ_6 = db_api.get_job(self.context, return_job_6['job']['id']) self.assertEqual(constants.JS_Success, job_succ_6['status']) response_7 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': return_job_6['job']['id']}, expect_errors=True) self.assertEqual(response_7.status_int, 400)
def test_get_one_and_get_all(self): all_job_ids = {} all_job_project_ids = {} index = 0 for job_type in self.all_job_types: job = self._prepare_job_element(job_type) jobs = [ { "job": job, "expected_error": 200 }, ] self._test_and_check(jobs) response = self.app.get('/v1.0/jobs') return_job = response.json all_job_ids[index] = return_job['jobs'][index]['id'] all_job_project_ids[job_type] = ( return_job['jobs'][index]['project_id']) index = index + 1 service_uris = ['jobs', 'jobs/detail'] amount_of_all_jobs = len(self.all_job_types) # with no filters all jobs are returned for service_uri in service_uris: response_1 = self.app.get('/v1.0/%(service_uri)s' % { 'service_uri': service_uri}) return_jobs_1 = response_1.json self.assertEqual(amount_of_all_jobs, len(return_jobs_1['jobs'])) self.assertIn('status', response_1) self.assertIn('resource', response_1) self.assertIn('project_id', response_1) self.assertIn('id', response_1) self.assertIn('timestamp', response_1) self.assertIn('type', response_1) self.assertNotIn('extra_id', response_1) self.assertNotIn('resource_id', response_1) # use job status filter response_2 = self.app.get('/v1.0/jobs?status=new') return_jobs_2 = response_2.json self.assertEqual(amount_of_all_jobs, len(return_jobs_2['jobs'])) response = self.app.get('/v1.0/jobs?status=fail') return_jobs_3 = response.json self.assertEqual(0, len(return_jobs_3['jobs'])) amount_of_fail_jobs = int(amount_of_all_jobs / 3) for i in xrange(amount_of_fail_jobs): db_api.finish_job(self.context, all_job_ids[i], False, timeutils.utcnow()) amount_of_succ_jobs = int(amount_of_all_jobs / 3) for i in xrange(amount_of_succ_jobs): db_api.finish_job(self.context, all_job_ids[amount_of_fail_jobs + i], True, timeutils.utcnow()) for service_uri in service_uris: response = self.app.get('/v1.0/%(service_uri)s?status=fail' % { 'service_uri': service_uri}) return_jobs = response.json self.assertEqual(amount_of_fail_jobs, len(return_jobs['jobs'])) response = self.app.get('/v1.0/%(service_uri)s?status=success' '' % {'service_uri': service_uri}) return_jobs = response.json self.assertEqual(amount_of_succ_jobs, len(return_jobs['jobs'])) # use job type filter or project id filter for job_type in self.all_job_types: response = self.app.get('/v1.0/%(service_uri)s?type=%(type)s' '' % {'service_uri': service_uri, 'type': job_type}) return_job = response.json self.assertEqual(1, len(return_job['jobs'])) response = self.app.get( '/v1.0/%(service_uri)s?project_id=%(project_id)s' % { 'service_uri': service_uri, 'project_id': all_job_project_ids[job_type]}) return_job = response.json self.assertEqual(1, len(return_job['jobs'])) # combine job type filter and project id filter response = self.app.get( '/v1.0/%(service_uri)s?project_id=%(project_id)s&' 'type=%(type)s' % { 'service_uri': service_uri, 'project_id': all_job_project_ids[job_type], 'type': job_type}) return_job = response.json self.assertEqual(1, len(return_job['jobs'])) # combine job type filter, project id filter and job status filter for i in xrange(amount_of_all_jobs): if i < amount_of_fail_jobs: # this aims to test service "/v1.0/jobs/{id}" response_1 = self.app.get('/v1.0/jobs/%(id)s' % { 'id': all_job_ids[i]}) return_job_1 = response_1.json response_2 = self.app.get( '/v1.0/%(service_uri)s?' 'project_id=%(project_id)s&' 'type=%(type)s&' 'status=%(status)s' % { 'service_uri': service_uri, 'project_id': return_job_1['job']['project_id'], 'type': return_job_1['job']['type'], 'status': 'fail'}) return_job_2 = response_2.json self.assertEqual(1, len(return_job_2['jobs'])) elif ((i >= amount_of_fail_jobs ) and (i < amount_of_fail_jobs + amount_of_succ_jobs)): # those jobs are set to 'success' and they are moved to # job log. their job ids are not stored in all_job_ids job_type = self.all_job_types[i] response = self.app.get( '/v1.0/%(service_uri)s?project_id=%(project_id)s&' 'type=%(type)s&status=%(status)s' % { 'service_uri': service_uri, 'project_id': all_job_project_ids[job_type], 'type': job_type, 'status': 'success'}) return_job = response.json self.assertEqual(1, len(return_job['jobs'])) response_2 = self.app.get( '/v1.0/%(service_uri)s?status=%(status)s' '&type=%(type)s' % { 'service_uri': service_uri, 'status': "success-x", 'type': job_type}) return_job_2 = response_2.json self.assertEqual(0, len(return_job_2['jobs'])) else: response_1 = self.app.get('/v1.0/jobs/%(id)s' % { 'id': all_job_ids[i]}) return_job_1 = response_1.json response_2 = self.app.get( '/v1.0/%(service_uri)s?project_id=%(project_id)s&' 'type=%(type)s&status=%(status)s' % { 'service_uri': service_uri, 'project_id': return_job_1['job']['project_id'], 'type': return_job_1['job']['type'], 'status': 'new'}) return_job_2 = response_2.json self.assertEqual(1, len(return_job_2['jobs'])) response_3 = self.app.get( '/v1.0/%(service_uri)s?status=%(status)s' '&type=%(type)s' % { 'service_uri': service_uri, 'status': "new-x", 'type': return_job_1['job']['type']}) return_job_3 = response_3.json self.assertEqual(0, len(return_job_3['jobs'])) # use unsupported filter, it will raise 400 error response = self.app.get('/v1.0/%(service_uri)s?' 'fake_filter=%(fake_filter)s' '' % {'service_uri': service_uri, 'fake_filter': "fake_filter"}, expect_errors=True) self.assertEqual(response.status_int, 400) # use invalid filter, it will return empty set response = self.app.get('/v1.0/%(service_uri)s?status=%(status)s' '' % {'service_uri': service_uri, 'status': "new-x"}) return_job = response.json self.assertEqual(0, len(return_job['jobs'])) @patch.object(context, 'extract_context_from_environ', new=fake_admin_context) def test_get_job_schemas(self): response = self.app.get('/v1.0/jobs/schemas') return_job_schemas = response.json job_schemas = [] for job_type in self.all_job_types: job = {} resource = [] for resource_type, resource_id in ( self.job_resource_map[job_type]): resource.append(resource_id) job['resource'] = resource job['type'] = job_type job_schemas.append(job) self.assertEqual(job_schemas, return_job_schemas['schemas'])
def test_post(self, mock_context): mock_context.return_value = self.context # cover all job types for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) kw_job = {'job': job} # failure case, only admin can create the job self.context.is_admin = False res = self.controller.post(**kw_job) self._validate_error_code(res, 403) self.context.is_admin = True # failure case, request body not found kw_job_1 = {'job_1': job} res = self.controller.post(**kw_job_1) self._validate_error_code(res, 400) # failure case, wrong job type parameter job_type_backup = job.pop('type') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = '' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = job_type_backup + '_1' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = job_type_backup # failure case, wrong resource parameter job_resource_backup = job.pop('resource') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['resource'] = copy.deepcopy(job_resource_backup) job['resource'].popitem() res = self.controller.post(**kw_job) self._validate_error_code(res, 400) fake_resource = 'fake_resource' job['resource'][fake_resource] = fake_resource res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['resource'] = job_resource_backup # failure case, wrong project id parameter project_id_backup = job.pop('project_id') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = '' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = uuidutils.generate_uuid() res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = project_id_backup # successful case, create an entirely new job. Because the job # status returned from controller has been formatted, so we not # only validate the database records, but also validate the return # value of the controller. job_1 = self.controller.post(**kw_job)['job'] job_in_db_1 = db_api.get_job(self.context, job_1['id']) self.assertEqual(job_type, job_in_db_1['type']) self.assertEqual(job['project_id'], job_in_db_1['project_id']) self.assertEqual(constants.JS_New, job_in_db_1['status']) self.assertEqual('NEW', job_1['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_1['resource'])) self.assertFalse('resource_id' in job_1) self.assertFalse('extra_id' in job_1) db_api.delete_job(self.context, job_1['id']) # successful case, target job already exists in the job table # and its status is NEW, then this newer job will be picked by # job handler. job_2 = self.controller.post(**kw_job)['job'] job_in_db_2 = db_api.get_job(self.context, job_2['id']) job_3 = self.controller.post(**kw_job)['job'] job_in_db_3 = db_api.get_job(self.context, job_3['id']) self.assertEqual(job_type, job_in_db_2['type']) self.assertEqual(job['project_id'], job_in_db_2['project_id']) self.assertEqual(constants.JS_New, job_in_db_2['status']) self.assertEqual('NEW', job_2['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_2['resource'])) self.assertFalse('resource_id' in job_2) self.assertFalse('extra_id' in job_2) self.assertEqual(job_type, job_in_db_3['type']) self.assertEqual(job['project_id'], job_in_db_3['project_id']) self.assertEqual(constants.JS_New, job_in_db_3['status']) self.assertEqual('NEW', job_3['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_3['resource'])) self.assertFalse('resource_id' in job_3) self.assertFalse('extra_id' in job_3) db_api.finish_job(self.context, job_3['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_3['id'])
def test_put(self, mock_context): mock_context.return_value = self.context # cover all job types for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) resource_id = '#'.join([ job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type] ]) # failure case, only admin can redo the job job_1 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.context.is_admin = False res = self.controller.put(job_1['id']) self._validate_error_code(res, 403) self.context.is_admin = True db_api.delete_job(self.context, job_1['id']) # failure case, job not found res = self.controller.put(-123) self._validate_error_code(res, 404) # failure case, redo a running job job_2 = db_api.register_job(self.context, job['project_id'], job_type, resource_id) res = self.controller.put(job_2['id']) self._validate_error_code(res, 400) db_api.finish_job(self.context, job_2['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_2['id']) # failure case, redo a successful job job_3 = self._prepare_job_element(job_type) resource_id_3 = '#'.join([ job_3['resource'][resource_id_3] for resource_type_3, resource_id_3 in self.job_resource_map[job_type] ]) job_4 = db_api.new_job(self.context, job_3['project_id'], job_type, resource_id_3) with self.context.session.begin(): job_dict = { 'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid() } core.update_resource(self.context, models.AsyncJob, job_4['id'], job_dict) res = self.controller.put(job_4['id']) self._validate_error_code(res, 400) db_api.finish_job(self.context, job_4['id'], True, timeutils.utcnow()) # successful case, redo a failed job job_5 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) db_api.finish_job(self.context, job_5['id'], False, timeutils.utcnow()) self.controller.put(job_5['id']) db_api.delete_job(self.context, job_5['id']) # successful case, redo a new job job_6 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.controller.put(job_6['id']) db_api.delete_job(self.context, job_6['id'])
def test_delete(self, mock_context): mock_context.return_value = self.context # cover all job types. # each 'for' loop adds one item in job log table, we set count variable # to record dynamic total job entries in job log table. count = 1 for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) resource_id = '#'.join([ job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type] ]) # failure case, only admin can delete the job job_1 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.context.is_admin = False res = self.controller.delete(job_1['id']) self._validate_error_code(res, 403) self.context.is_admin = True db_api.delete_job(self.context, job_1['id']) # failure case, job not found res = self.controller.delete(-123) self._validate_error_code(res, 404) # failure case, delete a running job job_2 = db_api.register_job(self.context, job['project_id'], job_type, resource_id) job = db_api.get_job(self.context, job_2['id']) res = self.controller.delete(job_2['id']) self._validate_error_code(res, 400) # finish the job and delete it db_api.finish_job(self.context, job_2['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_2['id']) # successful case, delete a successful job. successful job from # job log can't be deleted, here this successful job is from # job table. job_3 = self._prepare_job_element(job_type) resource_id_3 = '#'.join([ job_3['resource'][resource_id_3] for resource_type_3, resource_id_3 in self.job_resource_map[job_type] ]) job_4 = db_api.new_job(self.context, job_3['project_id'], job_type, resource_id_3) with self.context.session.begin(): job_dict = { 'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid() } core.update_resource(self.context, models.AsyncJob, job_4['id'], job_dict) job_4_succ = db_api.get_job(self.context, job_4['id']) self.controller.delete(job_4['id']) filters_job_4 = [{ 'key': 'type', 'comparator': 'eq', 'value': job_4_succ['type'] }, { 'key': 'status', 'comparator': 'eq', 'value': job_4_succ['status'] }, { 'key': 'resource_id', 'comparator': 'eq', 'value': job_4_succ['resource_id'] }, { 'key': 'extra_id', 'comparator': 'eq', 'value': job_4_succ['extra_id'] }] self.assertEqual( 0, len(db_api.list_jobs(self.context, filters_job_4))) self.assertEqual(count, len(db_api.list_jobs_from_log(self.context))) count = count + 1 # successful case, delete a new job job_5 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.controller.delete(job_5['id']) filters_job_5 = [{ 'key': 'type', 'comparator': 'eq', 'value': job_5['type'] }, { 'key': 'status', 'comparator': 'eq', 'value': job_5['status'] }, { 'key': 'resource_id', 'comparator': 'eq', 'value': job_5['resource_id'] }, { 'key': 'extra_id', 'comparator': 'eq', 'value': job_5['extra_id'] }] self.assertEqual( 0, len(db_api.list_jobs(self.context, filters_job_5))) # successful case, delete a failed job job_6 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) db_api.finish_job(self.context, job_6['id'], False, timeutils.utcnow()) job_6_failed = db_api.get_job(self.context, job_6['id']) self.controller.delete(job_6['id']) filters_job_6 = [{ 'key': 'type', 'comparator': 'eq', 'value': job_6_failed['type'] }, { 'key': 'status', 'comparator': 'eq', 'value': job_6_failed['status'] }, { 'key': 'resource_id', 'comparator': 'eq', 'value': job_6_failed['resource_id'] }, { 'key': 'extra_id', 'comparator': 'eq', 'value': job_6_failed['extra_id'] }] self.assertEqual( 0, len(db_api.list_jobs(self.context, filters_job_6)))
def test_get_one(self, mock_context): mock_context.return_value = self.context # failure case, only admin can list the job's info self.context.is_admin = False res = self.controller.get_one("schemas") self._validate_error_code(res, 403) res = self.controller.get_one("detail") self._validate_error_code(res, 403) res = self.controller.get_one(uuidutils.generate_uuid()) self._validate_error_code(res, 403) self.context.is_admin = True # failure case, parameter error res = self.controller.get_one("schemas_1") self._validate_error_code(res, 404) res = self.controller.get_one(uuidutils.generate_uuid()) self._validate_error_code(res, 404) # successful case, set id="schemas" to get job schemas job_schemas_2 = self.controller.get_one("schemas") job_schemas_3 = [] for job_type in self.job_resource_map.keys(): job = {} resource = [] for resource_type, resource_id in self.job_resource_map[job_type]: resource.append(resource_id) job['resource'] = resource job['type'] = job_type job_schemas_3.append(job) self.assertEqual(job_schemas_3, job_schemas_2['schemas']) # successful case, set id="detail" to get all jobs. # first, we need to create jobs in job table. amount_of_all_jobs = len(self.job_resource_map.keys()) all_job_ids = {} index = 0 for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) resource_id = '#'.join([ job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type] ]) job_1 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) all_job_ids[index] = job_1['id'] index = index + 1 # validate if the id=job_id, get_one(id=job_id) can take effective job_2 = self.controller.get_one(job_1['id'])['job'] self.assertEqual(job_1['type'], job_2['type']) self.assertEqual(job_1['project_id'], job_2['project_id']) self.assertEqual("NEW", job_2['status']) jobs_1 = self.controller.get_one("detail") self.assertEqual(amount_of_all_jobs, len(jobs_1['jobs'])) # create jobs in job log table, in order to validate # get_one(id=detail) can also get the jobs from job log amount_of_succ_jobs = int(len(all_job_ids) / 2) for i in xrange(amount_of_succ_jobs): db_api.finish_job(self.context, all_job_ids[i], True, timeutils.utcnow()) jobs_2 = self.controller.get_one("detail") self.assertEqual(amount_of_all_jobs, len(jobs_2['jobs'])) job_status_filter_1 = {'status': 'success'} jobs_3 = self.controller.get_one("detail", **job_status_filter_1) self.assertEqual(amount_of_succ_jobs, len(jobs_3['jobs'])) job_status_filter_2 = {'status': 'new'} jobs_4 = self.controller.get_one("detail", **job_status_filter_2) self.assertEqual(amount_of_all_jobs - amount_of_succ_jobs, len(jobs_4['jobs']))
def handle_args(*args, **kwargs): if IN_TEST: # NOTE(zhiyuan) job mechanism will cause some unpredictable # result in unit test so we would like to bypass it. However # we have problem mocking a decorator which decorates member # functions, that's why we use this label, not an elegant # way though. func(*args, **kwargs) return ctx = args[1] payload = kwargs['payload'] resource_id = payload[job_type] db_api.new_job(ctx, job_type, resource_id) start_time = datetime.datetime.now() while True: current_time = datetime.datetime.now() delta = current_time - start_time if delta.seconds >= CONF.worker_handle_timeout: # quit when this handle is running for a long time break time_new = db_api.get_latest_timestamp(ctx, constants.JS_New, job_type, resource_id) time_success = db_api.get_latest_timestamp( ctx, constants.JS_Success, job_type, resource_id) if time_success and time_success >= time_new: break job = db_api.register_job(ctx, job_type, resource_id) if not job: # fail to obtain the lock, let other worker handle the job running_job = db_api.get_running_job(ctx, job_type, resource_id) if not running_job: # there are two reasons that running_job is None. one # is that the running job has just been finished, the # other is that all workers fail to register the job # due to deadlock exception. so we sleep and try again eventlet.sleep(CONF.worker_sleep_time) continue job_time = running_job['timestamp'] current_time = datetime.datetime.now() delta = current_time - job_time if delta.seconds > CONF.job_run_expire: # previous running job expires, we set its status to # fail and try again to obtain the lock db_api.finish_job(ctx, running_job['id'], False, time_new) LOG.warning(_LW('Job %(job)s of type %(job_type)s for ' 'resource %(resource)s expires, set ' 'its state to Fail'), {'job': running_job['id'], 'job_type': job_type, 'resource': resource_id}) eventlet.sleep(CONF.worker_sleep_time) continue else: # previous running job is still valid, we just leave # the job to the worker who holds the lock break # successfully obtain the lock, start to execute handler try: func(*args, **kwargs) except Exception: db_api.finish_job(ctx, job['id'], False, time_new) LOG.error(_LE('Job %(job)s of type %(job_type)s for ' 'resource %(resource)s fails'), {'job': job['id'], 'job_type': job_type, 'resource': resource_id}) break db_api.finish_job(ctx, job['id'], True, time_new) eventlet.sleep(CONF.worker_sleep_time)
def test_put(self, mock_context): mock_context.return_value = self.context # cover all job types for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) resource_id = '#'.join([job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type]]) # failure case, only admin can redo the job job_1 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.context.is_admin = False res = self.controller.put(job_1['id']) self._validate_error_code(res, 403) self.context.is_admin = True db_api.delete_job(self.context, job_1['id']) # failure case, job not found res = self.controller.put(-123) self._validate_error_code(res, 404) # failure case, redo a running job job_2 = db_api.register_job(self.context, job['project_id'], job_type, resource_id) res = self.controller.put(job_2['id']) self._validate_error_code(res, 400) db_api.finish_job(self.context, job_2['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_2['id']) # failure case, redo a successful job job_3 = self._prepare_job_element(job_type) resource_id_3 = '#'.join([job_3['resource'][resource_id_3] for resource_type_3, resource_id_3 in self.job_resource_map[job_type]]) job_4 = db_api.new_job(self.context, job_3['project_id'], job_type, resource_id_3) with self.context.session.begin(): job_dict = {'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid()} core.update_resource(self.context, models.AsyncJob, job_4['id'], job_dict) res = self.controller.put(job_4['id']) self._validate_error_code(res, 400) db_api.finish_job(self.context, job_4['id'], True, timeutils.utcnow()) # successful case, redo a failed job job_5 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) db_api.finish_job(self.context, job_5['id'], False, timeutils.utcnow()) self.controller.put(job_5['id']) db_api.delete_job(self.context, job_5['id']) # successful case, redo a new job job_6 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.controller.put(job_6['id']) db_api.delete_job(self.context, job_6['id'])
def test_delete(self, mock_context): mock_context.return_value = self.context # cover all job types. # each 'for' loop adds one item in job log table, we set count variable # to record dynamic total job entries in job log table. count = 1 for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) resource_id = '#'.join([job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type]]) # failure case, only admin can delete the job job_1 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.context.is_admin = False res = self.controller.delete(job_1['id']) self._validate_error_code(res, 403) self.context.is_admin = True db_api.delete_job(self.context, job_1['id']) # failure case, job not found res = self.controller.delete(-123) self._validate_error_code(res, 404) # failure case, delete a running job job_2 = db_api.register_job(self.context, job['project_id'], job_type, resource_id) job = db_api.get_job(self.context, job_2['id']) res = self.controller.delete(job_2['id']) self._validate_error_code(res, 400) # finish the job and delete it db_api.finish_job(self.context, job_2['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_2['id']) # successful case, delete a successful job. successful job from # job log can't be deleted, here this successful job is from # job table. job_3 = self._prepare_job_element(job_type) resource_id_3 = '#'.join([job_3['resource'][resource_id_3] for resource_type_3, resource_id_3 in self.job_resource_map[job_type]]) job_4 = db_api.new_job(self.context, job_3['project_id'], job_type, resource_id_3) with self.context.session.begin(): job_dict = {'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid()} core.update_resource(self.context, models.AsyncJob, job_4['id'], job_dict) job_4_succ = db_api.get_job(self.context, job_4['id']) self.controller.delete(job_4['id']) filters_job_4 = [ {'key': 'type', 'comparator': 'eq', 'value': job_4_succ['type']}, {'key': 'status', 'comparator': 'eq', 'value': job_4_succ['status']}, {'key': 'resource_id', 'comparator': 'eq', 'value': job_4_succ['resource_id']}, {'key': 'extra_id', 'comparator': 'eq', 'value': job_4_succ['extra_id']}] self.assertEqual(0, len(db_api.list_jobs(self.context, filters_job_4))) self.assertEqual(count, len(db_api.list_jobs_from_log(self.context))) count = count + 1 # successful case, delete a new job job_5 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) self.controller.delete(job_5['id']) filters_job_5 = [ {'key': 'type', 'comparator': 'eq', 'value': job_5['type']}, {'key': 'status', 'comparator': 'eq', 'value': job_5['status']}, {'key': 'resource_id', 'comparator': 'eq', 'value': job_5['resource_id']}, {'key': 'extra_id', 'comparator': 'eq', 'value': job_5['extra_id']}] self.assertEqual(0, len(db_api.list_jobs(self.context, filters_job_5))) # successful case, delete a failed job job_6 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) db_api.finish_job(self.context, job_6['id'], False, timeutils.utcnow()) job_6_failed = db_api.get_job(self.context, job_6['id']) self.controller.delete(job_6['id']) filters_job_6 = [ {'key': 'type', 'comparator': 'eq', 'value': job_6_failed['type']}, {'key': 'status', 'comparator': 'eq', 'value': job_6_failed['status']}, {'key': 'resource_id', 'comparator': 'eq', 'value': job_6_failed['resource_id']}, {'key': 'extra_id', 'comparator': 'eq', 'value': job_6_failed['extra_id']}] self.assertEqual(0, len(db_api.list_jobs(self.context, filters_job_6)))
def test_get_all_jobs_with_pagination(self, mock_context): self.context.project_id = uuidutils.generate_uuid() mock_context.return_value = self.context # map job type to project id for later project id filter validation. job_project_id_map = {} amount_of_all_jobs = len(self.job_resource_map.keys()) amount_of_running_jobs = 3 count = 1 # cover all job types. for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) if count > 1: # for test convenience, the first job has a project ID # that is different from the context.project_id job['project_id'] = self.context.project_id job_project_id_map[job_type] = job['project_id'] resource_id = '#'.join([job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type]]) if count <= amount_of_running_jobs: db_api.register_job(self.context, job['project_id'], job_type, resource_id) # because jobs are sorted by timestamp, without time delay then # all jobs are created at the same time, paginate_query can't # identify them time.sleep(1) else: db_api.new_job(self.context, job['project_id'], job_type, resource_id) time.sleep(1) count = count + 1 # query the jobs with several kinds of filters. # supported filters: project id, job status, job type. job_status_filter_1 = {'status': 'new'} job_status_filter_2 = {'status': 'fail'} job_status_filter_3 = {'status': 'running'} invalid_filter = {'status': "new-x"} unsupported_filter = {'fake_filter': "fake_filter"} count = 1 for job_type in self.job_resource_map.keys(): job_type_filter_1 = {'type': job_type} job_type_filter_2 = {'type': job_type + '_1'} # failure case, only admin can list the jobs self.context.is_admin = False res = self.controller.get_all() self._validate_error_code(res, 403) self.context.is_admin = True # test when specify project ID filter from client, if this # project ID is different from the one from context, then # it will be ignored, project ID from context will be # used instead. filter1 = {'project_id': uuidutils.generate_uuid()} res1 = self.controller.get_all(**filter1) filter2 = {'project_id': self.context.project_id} res2 = self.controller.get_all(**filter2) self.assertEqual(len(res2['jobs']), len(res1['jobs'])) res3 = self.controller.get_all() # there is one job whose project ID is different from # context.project_id. As the list operation only retrieves the # jobs whose project ID equals to context.project_id, so this # special job entry won't be retrieved. self.assertEqual(len(res3['jobs']), len(res2['jobs'])) # successful case, filter by job type jobs_job_type_filter_1 = self.controller.get_all( **job_type_filter_1) if count == 1: self.assertEqual(0, len(jobs_job_type_filter_1['jobs'])) else: self.assertEqual(1, len(jobs_job_type_filter_1['jobs'])) jobs_job_type_filter_2 = self.controller.get_all( **job_type_filter_2) self.assertEqual(0, len(jobs_job_type_filter_2['jobs'])) # successful case, filter by job status and job type if count <= amount_of_running_jobs: all_filters = dict(list(job_status_filter_3.items()) + list(job_type_filter_1.items())) jobs_all_filters = self.controller.get_all(**all_filters) if count == 1: self.assertEqual(0, len(jobs_all_filters['jobs'])) else: self.assertEqual(1, len(jobs_all_filters['jobs'])) else: all_filters = dict(list(job_status_filter_1.items()) + list(job_type_filter_1.items())) jobs_all_filters = self.controller.get_all(**all_filters) self.assertEqual(1, len(jobs_all_filters['jobs'])) # successful case, contradictory filter contradict_filters = dict(list(job_status_filter_2.items()) + list((job_type_filter_2.items()))) jobs_contradict_filters = self.controller.get_all( **contradict_filters) self.assertEqual(0, len(jobs_contradict_filters['jobs'])) count = count + 1 # failure case, unsupported filter res = self.controller.get_all(**unsupported_filter) self._validate_error_code(res, 400) # successful case, invalid filter jobs_invalid_filter = self.controller.get_all(**invalid_filter) self.assertEqual(0, len(jobs_invalid_filter['jobs'])) # successful case, list jobs without filters jobs_empty_filters = self.controller.get_all() self.assertEqual(amount_of_all_jobs - 1, len(jobs_empty_filters['jobs'])) # successful case, filter by job status jobs_job_status_filter_1 = self.controller.get_all( **job_status_filter_1) self.assertEqual(amount_of_all_jobs - amount_of_running_jobs, len(jobs_job_status_filter_1['jobs'])) jobs_job_status_filter_2 = self.controller.get_all( **job_status_filter_2) self.assertEqual(0, len(jobs_job_status_filter_2['jobs'])) jobs_job_status_filter_3 = self.controller.get_all( **job_status_filter_3) self.assertEqual(amount_of_running_jobs - 1, len(jobs_job_status_filter_3['jobs'])) # test for paginate query job_paginate_no_filter_1 = self.controller.get_all() self.assertEqual(amount_of_all_jobs - 1, len(job_paginate_no_filter_1['jobs'])) # no limit no marker job_paginate_filter_1 = {'status': 'new'} jobs_paginate_filter_1 = self.controller.get_all( **job_paginate_filter_1) self.assertEqual(amount_of_all_jobs - amount_of_running_jobs, len(jobs_paginate_filter_1['jobs'])) # failed cases, unsupported limit type job_paginate_filter_2 = {'limit': '2test'} res = self.controller.get_all(**job_paginate_filter_2) self._validate_error_code(res, 400) # successful cases job_paginate_filter_4 = {'status': 'new', 'limit': '2'} res = self.controller.get_all(**job_paginate_filter_4) self.assertEqual(2, len(res['jobs'])) job_paginate_filter_5 = {'status': 'new', 'limit': 2} res = self.controller.get_all(**job_paginate_filter_5) self.assertEqual(2, len(res['jobs'])) job_paginate_filter_6 = {'status': 'running', 'limit': 1} res1 = self.controller.get_all(**job_paginate_filter_6) marker = res1['jobs'][0]['id'] job_paginate_filter_7 = {'status': 'running', 'marker': marker} res2 = self.controller.get_all(**job_paginate_filter_7) self.assertEqual(amount_of_running_jobs - 1, len(res2['jobs'])) job_paginate_filter_8 = {'status': 'new', 'limit': 3} res = self.controller.get_all(**job_paginate_filter_8) self.assertGreaterEqual(res['jobs'][0]['timestamp'], res['jobs'][1]['timestamp']) self.assertGreaterEqual(res['jobs'][1]['timestamp'], res['jobs'][2]['timestamp']) # unsupported marker type res = self.controller.get_all(marker=None) self.assertEqual(amount_of_all_jobs - 1, len(res['jobs'])) res = self.controller.get_all(marker='-123') self._validate_error_code(res, 400) # marker not in job table and job log table job_paginate_filter_9 = {'marker': uuidutils.generate_uuid()} res = self.controller.get_all(**job_paginate_filter_9) self._validate_error_code(res, 400) # test marker and limit limit = 2 pt = '/v1.0/jobs\?limit=\w+&marker=([\w-]+)' job_paginate_filter = {'status': 'new', 'limit': limit} res = self.controller.get_all(**job_paginate_filter) while 'jobs_links' in res: m = re.match(pt, res['jobs_links'][0]['href']) marker = m.group(1) self.assertEqual(limit, len(res['jobs'])) job_paginate_filter = {'status': 'new', 'limit': limit, 'marker': marker} res = self.controller.get_all(**job_paginate_filter) job_paginate_filter_10 = {'status': 'running'} res = self.controller.get_all(**job_paginate_filter_10) self.assertEqual(amount_of_running_jobs - 1, len(res['jobs'])) # add some rows to job log table for i in xrange(amount_of_running_jobs - 1): db_api.finish_job(self.context, res['jobs'][i]['id'], True, timeutils.utcnow()) time.sleep(1) res_success_log = db_api.list_jobs_from_log(self.context, None) self.assertEqual(amount_of_running_jobs - 1, len(res_success_log)) res_in_job = db_api.list_jobs(self.context, None) self.assertEqual(amount_of_all_jobs - (amount_of_running_jobs - 1), len(res_in_job)) job_paginate_filter_11 = {'limit': 2} res = self.controller.get_all(**job_paginate_filter_11) self.assertIsNotNone(res['jobs_links'][0]['href'])
def test_get_one_and_get_all(self, mock_context): self.context.project_id = uuidutils.generate_uuid() mock_context.return_value = self.context # failure case, only admin can list the job's info self.context.is_admin = False res = self.controller.get_one("schemas") self._validate_error_code(res, 403) res = self.controller.get_one("detail") self._validate_error_code(res, 403) res = self.controller.get_one(uuidutils.generate_uuid()) self._validate_error_code(res, 403) self.context.is_admin = True # failure case, parameter error res = self.controller.get_one("schemas_1") self._validate_error_code(res, 404) res = self.controller.get_one(uuidutils.generate_uuid()) self._validate_error_code(res, 404) # successful case, set id="schemas" to get job schemas job_schemas_2 = self.controller.get_one("schemas") job_schemas_3 = [] for job_type in self.job_resource_map.keys(): job = {} resource = [] for resource_type, resource_id in self.job_resource_map[job_type]: resource.append(resource_id) job['resource'] = resource job['type'] = job_type job_schemas_3.append(job) self.assertEqual(job_schemas_3, job_schemas_2['schemas']) # successful case, set id="detail" to get all jobs. # first, we need to create jobs in job table. amount_of_all_jobs = len(self.job_resource_map.keys()) all_job_ids = {} index = 0 for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) # for test convenience, all jobs have same project ID job['project_id'] = self.context.project_id resource_id = '#'.join([job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type]]) job_1 = db_api.new_job(self.context, job['project_id'], job_type, resource_id) all_job_ids[index] = job_1['id'] index = index + 1 time.sleep(1) # validate if the id=job_id, get_one(id=job_id) can take effective job_2 = self.controller.get_one(job_1['id'])['job'] self.assertEqual(job_1['type'], job_2['type']) self.assertEqual(job_1['project_id'], job_2['project_id']) self.assertEqual("NEW", job_2['status']) jobs_1 = self.controller.get_one("detail") self.assertEqual(amount_of_all_jobs, len(jobs_1['jobs'])) # create jobs in job log table, in order to validate # get_one(id=detail) can also get the jobs from job log amount_of_succ_jobs = int(len(all_job_ids) / 2) for i in xrange(amount_of_succ_jobs): db_api.finish_job(self.context, all_job_ids[i], True, timeutils.utcnow()) time.sleep(1) jobs_2 = self.controller.get_one("detail") self.assertEqual(amount_of_all_jobs, len(jobs_2['jobs'])) job_status_filter_1 = {'status': 'success'} jobs_3 = self.controller.get_one("detail", **job_status_filter_1) self.assertEqual(amount_of_succ_jobs, len(jobs_3['jobs'])) # set marker in job log res = self.controller.get_all(marker=jobs_3['jobs'][0]['id'], limit=amount_of_succ_jobs) self.assertEqual(amount_of_succ_jobs - 1, len(res['jobs'])) job_status_filter_2 = {'status': 'new'} amount_of_new_jobs = amount_of_all_jobs - amount_of_succ_jobs jobs_4 = self.controller.get_one("detail", **job_status_filter_2) self.assertEqual(amount_of_new_jobs, len(jobs_4['jobs'])) # set marker in job res = self.controller.get_all(marker=jobs_4['jobs'][0]['id'], limit=amount_of_new_jobs) self.assertEqual(amount_of_new_jobs, len(res['jobs']))
def test_redo_job(self): for job_type in self.all_job_types: job = self._prepare_job_element(job_type) jobs = [ # create an entirely new job { "job": job, "expected_error": 200 }, ] self._test_and_check(jobs) response = self.app.get('/v1.0/jobs') return_job = response.json jobs = return_job['jobs'] # redo a new job for job in jobs: response_1 = self.app.put('/v1.0/jobs/%(id)s' % {'id': job['id']}, expect_errors=True) self.assertEqual(response_1.status_int, 200) response_2 = self.app.put('/v1.0/jobs/123', expect_errors=True) self.assertEqual(response_2.status_int, 404) # redo a running job job_type_3 = constants.JT_NETWORK_UPDATE job_3 = self._prepare_job_element(job_type_3) resource_id_3 = '#'.join([job_3['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type_3]]) job_running_3 = db_api.register_job(self.context, job_3['project_id'], job_type_3, resource_id_3) self.assertEqual(constants.JS_Running, job_running_3['status']) response_3 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': job_running_3['id']}, expect_errors=True) self.assertEqual(response_3.status_int, 400) # redo a failed job job_type_4 = constants.JT_NETWORK_UPDATE job_4 = self._prepare_job_element(job_type_4) job_dict_4 = { "job": job_4, "expected_error": 200 } response_4 = self.app.post_json('/v1.0/jobs', dict(job=job_dict_4['job']), expect_errors=True) return_job_4 = response_4.json self.assertEqual(response_4.status_int, 200) db_api.finish_job(self.context, return_job_4['job']['id'], False, timeutils.utcnow()) job_fail_4 = db_api.get_job(self.context, return_job_4['job']['id']) self.assertEqual(constants.JS_Fail, job_fail_4['status']) response_5 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': return_job_4['job']['id']}, expect_errors=True) self.assertEqual(response_5.status_int, 200) # redo a successful job job_type_6 = constants.JT_NETWORK_UPDATE job_6 = self._prepare_job_element(job_type_6) job_dict_6 = { "job": job_6, "expected_error": 200 } response_6 = self.app.post_json('/v1.0/jobs', dict(job=job_dict_6['job']), expect_errors=True) return_job_6 = response_6.json with self.context.session.begin(): job_dict = {'status': constants.JS_Success, 'timestamp': timeutils.utcnow(), 'extra_id': uuidutils.generate_uuid()} core.update_resource(self.context, models.AsyncJob, return_job_6['job']['id'], job_dict) job_succ_6 = db_api.get_job(self.context, return_job_6['job']['id']) self.assertEqual(constants.JS_Success, job_succ_6['status']) response_7 = self.app.put('/v1.0/jobs/%(id)s' % { 'id': return_job_6['job']['id']}, expect_errors=True) self.assertEqual(response_7.status_int, 400)
def test_get_one_and_get_all(self, mock_context): self.context.project_id = "fake_project_id" mock_context.return_value = self.context all_job_ids = {} all_job_project_ids = {} index = 0 for job_type in self.all_job_types: if index == 0: # the first job has a project ID that differs from # context.project_id job = self._prepare_job_element(job_type) else: job = self._prepare_job_element(job_type, self.context.project_id) job = {"job": job, "expected_error": 200} back_jobid = self._test_and_obtain_id(job) all_job_ids[index] = back_jobid all_job_project_ids[job_type] = job['job']['project_id'] index = index + 1 service_uris = ['jobs', 'jobs/detail'] amount_of_all_jobs = len(self.all_job_types) # with no filters all jobs are returned for service_uri in service_uris: response_1 = self.app.get('/v1.0/%(service_uri)s' % { 'service_uri': service_uri}) return_jobs_1 = response_1.json self.assertEqual(amount_of_all_jobs - 1, len(return_jobs_1['jobs'])) self.assertIn('status', response_1) self.assertIn('resource', response_1) self.assertIn('project_id', response_1) self.assertIn('id', response_1) self.assertIn('timestamp', response_1) self.assertIn('type', response_1) self.assertNotIn('extra_id', response_1) self.assertNotIn('resource_id', response_1) # use job status filter response_2 = self.app.get('/v1.0/jobs?status=new') return_jobs_2 = response_2.json self.assertEqual(amount_of_all_jobs - 1, len(return_jobs_2['jobs'])) response = self.app.get('/v1.0/jobs?status=fail') return_jobs_3 = response.json self.assertEqual(0, len(return_jobs_3['jobs'])) amount_of_fail_jobs = int(amount_of_all_jobs / 3) for i in xrange(amount_of_fail_jobs): db_api.finish_job(self.context, all_job_ids[i], False, timeutils.utcnow()) amount_of_succ_jobs = int(amount_of_all_jobs / 3) for i in xrange(amount_of_succ_jobs): db_api.finish_job(self.context, all_job_ids[amount_of_fail_jobs + i], True, timeutils.utcnow()) for service_uri in service_uris: response = self.app.get('/v1.0/%(service_uri)s?status=fail' % { 'service_uri': service_uri}) return_jobs = response.json self.assertEqual(amount_of_fail_jobs - 1, len(return_jobs['jobs'])) response = self.app.get('/v1.0/%(service_uri)s?status=success' '' % {'service_uri': service_uri}) return_jobs = response.json self.assertEqual(amount_of_succ_jobs, len(return_jobs['jobs'])) # project ID filter in URL query string will be ignored, and # only the project ID in which the user is authorized will # be used as filter. response = self.app.get( '/v1.0/%(service_uri)s' % {'service_uri': service_uri}) return_job = response.json response1 = self.app.get( '/v1.0/%(service_uri)s?project_id=%(project_id)s' % { 'service_uri': service_uri, 'project_id': uuidutils.generate_uuid()}) return_job1 = response1.json response2 = self.app.get( '/v1.0/%(service_uri)s?project_id=%(project_id)s' % { 'service_uri': service_uri, 'project_id': 'fake_project_id'}) return_job2 = response2.json self.assertEqual(len(return_job2['jobs']), len(return_job1['jobs'])) self.assertEqual(len(return_job['jobs']), len(return_job2['jobs'])) # use job type filter count = 1 for job_type in self.all_job_types: response = self.app.get('/v1.0/%(service_uri)s?type=%(type)s' '' % {'service_uri': service_uri, 'type': job_type}) return_job = response.json if count == 1: self.assertEqual(0, len(return_job['jobs'])) else: self.assertEqual(1, len(return_job['jobs'])) count += 1 # combine job type and job status filter for i in xrange(1, amount_of_all_jobs): if i < amount_of_fail_jobs: # this aims to test service "/v1.0/jobs/{id}" response_1 = self.app.get('/v1.0/jobs/%(id)s' % { 'id': all_job_ids[i]}) return_job_1 = response_1.json response_2 = self.app.get( '/v1.0/%(service_uri)s?' 'type=%(type)s&' 'status=%(status)s' % { 'service_uri': service_uri, 'type': return_job_1['job']['type'], 'status': 'fail'}) return_job_2 = response_2.json self.assertEqual(1, len(return_job_2['jobs'])) elif ((i >= amount_of_fail_jobs ) and (i < amount_of_fail_jobs + amount_of_succ_jobs)): # those jobs are set to 'success' and they are moved to # job log. their job ids are not stored in all_job_ids job_type = self.all_job_types[i] response = self.app.get( '/v1.0/%(service_uri)s?' 'type=%(type)s&status=%(status)s' % { 'service_uri': service_uri, 'type': job_type, 'status': 'success'}) return_job = response.json self.assertEqual(1, len(return_job['jobs'])) response_2 = self.app.get( '/v1.0/%(service_uri)s?status=%(status)s' '&type=%(type)s' % { 'service_uri': service_uri, 'status': "success-x", 'type': job_type}) return_job_2 = response_2.json self.assertEqual(0, len(return_job_2['jobs'])) else: response_1 = self.app.get('/v1.0/jobs/%(id)s' % { 'id': all_job_ids[i]}) return_job_1 = response_1.json response_2 = self.app.get( '/v1.0/%(service_uri)s?' 'type=%(type)s&status=%(status)s' % { 'service_uri': service_uri, 'type': return_job_1['job']['type'], 'status': 'new'}) return_job_2 = response_2.json self.assertEqual(1, len(return_job_2['jobs'])) response_3 = self.app.get( '/v1.0/%(service_uri)s?status=%(status)s' '&type=%(type)s' % { 'service_uri': service_uri, 'status': "new-x", 'type': return_job_1['job']['type']}) return_job_3 = response_3.json self.assertEqual(0, len(return_job_3['jobs'])) # use unsupported filter, it will raise 400 error response = self.app.get('/v1.0/%(service_uri)s?' 'fake_filter=%(fake_filter)s' '' % {'service_uri': service_uri, 'fake_filter': "fake_filter"}, expect_errors=True) self.assertEqual(response.status_int, 400) # use invalid filter, it will return empty set response = self.app.get('/v1.0/%(service_uri)s?status=%(status)s' '' % {'service_uri': service_uri, 'status': "new-x"}) return_job = response.json self.assertEqual(0, len(return_job['jobs'])) @patch.object(context, 'extract_context_from_environ', new=fake_admin_context) def test_get_job_schemas(self): response = self.app.get('/v1.0/jobs/schemas') return_job_schemas = response.json job_schemas = [] for job_type in self.all_job_types: job = {} resource = [] for resource_type, resource_id in ( self.job_resource_map[job_type]): resource.append(resource_id) job['resource'] = resource job['type'] = job_type job_schemas.append(job) self.assertEqual(job_schemas, return_job_schemas['schemas'])
def test_get_all_jobs_with_pagination(self, mock_context): self.context.project_id = uuidutils.generate_uuid() mock_context.return_value = self.context # map job type to project id for later project id filter validation. job_project_id_map = {} amount_of_all_jobs = len(self.job_resource_map.keys()) amount_of_running_jobs = 3 count = 1 # cover all job types. for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) if count > 1: # for test convenience, the first job has a project ID # that is different from the context.project_id job['project_id'] = self.context.project_id job_project_id_map[job_type] = job['project_id'] resource_id = '#'.join([ job['resource'][resource_id] for resource_type, resource_id in self.job_resource_map[job_type] ]) if count <= amount_of_running_jobs: db_api.register_job(self.context, job['project_id'], job_type, resource_id) # because jobs are sorted by timestamp, without time delay then # all jobs are created at the same time, paginate_query can't # identify them time.sleep(1) else: db_api.new_job(self.context, job['project_id'], job_type, resource_id) time.sleep(1) count = count + 1 # query the jobs with several kinds of filters. # supported filters: project id, job status, job type. job_status_filter_1 = {'status': 'new'} job_status_filter_2 = {'status': 'fail'} job_status_filter_3 = {'status': 'running'} invalid_filter = {'status': "new-x"} unsupported_filter = {'fake_filter': "fake_filter"} count = 1 for job_type in self.job_resource_map.keys(): job_type_filter_1 = {'type': job_type} job_type_filter_2 = {'type': job_type + '_1'} # failure case, only admin can list the jobs self.context.is_admin = False res = self.controller.get_all() self._validate_error_code(res, 403) self.context.is_admin = True # test when specify project ID filter from client, if this # project ID is different from the one from context, then # it will be ignored, project ID from context will be # used instead. filter1 = {'project_id': uuidutils.generate_uuid()} res1 = self.controller.get_all(**filter1) filter2 = {'project_id': self.context.project_id} res2 = self.controller.get_all(**filter2) self.assertEqual(len(res2['jobs']), len(res1['jobs'])) res3 = self.controller.get_all() # there is one job whose project ID is different from # context.project_id. As the list operation only retrieves the # jobs whose project ID equals to context.project_id, so this # special job entry won't be retrieved. self.assertEqual(len(res3['jobs']), len(res2['jobs'])) # successful case, filter by job type jobs_job_type_filter_1 = self.controller.get_all( **job_type_filter_1) if count == 1: self.assertEqual(0, len(jobs_job_type_filter_1['jobs'])) else: self.assertEqual(1, len(jobs_job_type_filter_1['jobs'])) jobs_job_type_filter_2 = self.controller.get_all( **job_type_filter_2) self.assertEqual(0, len(jobs_job_type_filter_2['jobs'])) # successful case, filter by job status and job type if count <= amount_of_running_jobs: all_filters = dict( list(job_status_filter_3.items()) + list(job_type_filter_1.items())) jobs_all_filters = self.controller.get_all(**all_filters) if count == 1: self.assertEqual(0, len(jobs_all_filters['jobs'])) else: self.assertEqual(1, len(jobs_all_filters['jobs'])) else: all_filters = dict( list(job_status_filter_1.items()) + list(job_type_filter_1.items())) jobs_all_filters = self.controller.get_all(**all_filters) self.assertEqual(1, len(jobs_all_filters['jobs'])) # successful case, contradictory filter contradict_filters = dict( list(job_status_filter_2.items()) + list((job_type_filter_2.items()))) jobs_contradict_filters = self.controller.get_all( **contradict_filters) self.assertEqual(0, len(jobs_contradict_filters['jobs'])) count = count + 1 # failure case, unsupported filter res = self.controller.get_all(**unsupported_filter) self._validate_error_code(res, 400) # successful case, invalid filter jobs_invalid_filter = self.controller.get_all(**invalid_filter) self.assertEqual(0, len(jobs_invalid_filter['jobs'])) # successful case, list jobs without filters jobs_empty_filters = self.controller.get_all() self.assertEqual(amount_of_all_jobs - 1, len(jobs_empty_filters['jobs'])) # successful case, filter by job status jobs_job_status_filter_1 = self.controller.get_all( **job_status_filter_1) self.assertEqual(amount_of_all_jobs - amount_of_running_jobs, len(jobs_job_status_filter_1['jobs'])) jobs_job_status_filter_2 = self.controller.get_all( **job_status_filter_2) self.assertEqual(0, len(jobs_job_status_filter_2['jobs'])) jobs_job_status_filter_3 = self.controller.get_all( **job_status_filter_3) self.assertEqual(amount_of_running_jobs - 1, len(jobs_job_status_filter_3['jobs'])) # test for paginate query job_paginate_no_filter_1 = self.controller.get_all() self.assertEqual(amount_of_all_jobs - 1, len(job_paginate_no_filter_1['jobs'])) # no limit no marker job_paginate_filter_1 = {'status': 'new'} jobs_paginate_filter_1 = self.controller.get_all( **job_paginate_filter_1) self.assertEqual(amount_of_all_jobs - amount_of_running_jobs, len(jobs_paginate_filter_1['jobs'])) # failed cases, unsupported limit type job_paginate_filter_2 = {'limit': '2test'} res = self.controller.get_all(**job_paginate_filter_2) self._validate_error_code(res, 400) # successful cases job_paginate_filter_4 = {'status': 'new', 'limit': '2'} res = self.controller.get_all(**job_paginate_filter_4) self.assertEqual(2, len(res['jobs'])) job_paginate_filter_5 = {'status': 'new', 'limit': 2} res = self.controller.get_all(**job_paginate_filter_5) self.assertEqual(2, len(res['jobs'])) job_paginate_filter_6 = {'status': 'running', 'limit': 1} res1 = self.controller.get_all(**job_paginate_filter_6) marker = res1['jobs'][0]['id'] job_paginate_filter_7 = {'status': 'running', 'marker': marker} res2 = self.controller.get_all(**job_paginate_filter_7) self.assertEqual(amount_of_running_jobs - 1, len(res2['jobs'])) job_paginate_filter_8 = {'status': 'new', 'limit': 3} res = self.controller.get_all(**job_paginate_filter_8) self.assertGreaterEqual(res['jobs'][0]['timestamp'], res['jobs'][1]['timestamp']) self.assertGreaterEqual(res['jobs'][1]['timestamp'], res['jobs'][2]['timestamp']) # unsupported marker type res = self.controller.get_all(marker=None) self.assertEqual(amount_of_all_jobs - 1, len(res['jobs'])) res = self.controller.get_all(marker='-123') self._validate_error_code(res, 400) # marker not in job table and job log table job_paginate_filter_9 = {'marker': uuidutils.generate_uuid()} res = self.controller.get_all(**job_paginate_filter_9) self._validate_error_code(res, 400) # test marker and limit limit = 2 pt = '/v1.0/jobs\?limit=\w+&marker=([\w-]+)' job_paginate_filter = {'status': 'new', 'limit': limit} res = self.controller.get_all(**job_paginate_filter) while 'jobs_links' in res: m = re.match(pt, res['jobs_links'][0]['href']) marker = m.group(1) self.assertEqual(limit, len(res['jobs'])) job_paginate_filter = { 'status': 'new', 'limit': limit, 'marker': marker } res = self.controller.get_all(**job_paginate_filter) job_paginate_filter_10 = {'status': 'running'} res = self.controller.get_all(**job_paginate_filter_10) self.assertEqual(amount_of_running_jobs - 1, len(res['jobs'])) # add some rows to job log table for i in xrange(amount_of_running_jobs - 1): db_api.finish_job(self.context, res['jobs'][i]['id'], True, timeutils.utcnow()) time.sleep(1) res_success_log = db_api.list_jobs_from_log(self.context, None) self.assertEqual(amount_of_running_jobs - 1, len(res_success_log)) res_in_job = db_api.list_jobs(self.context, None) self.assertEqual(amount_of_all_jobs - (amount_of_running_jobs - 1), len(res_in_job)) job_paginate_filter_11 = {'limit': 2} res = self.controller.get_all(**job_paginate_filter_11) self.assertIsNotNone(res['jobs_links'][0]['href'])
def handle_args(*args, **kwargs): if IN_TEST: # NOTE(zhiyuan) job mechanism will cause some unpredictable # result in unit test so we would like to bypass it. However # we have problem mocking a decorator which decorates member # functions, that's why we use this label, not an elegant # way though. func(*args, **kwargs) return ctx = args[1] payload = kwargs['payload'] resource_id = payload[job_type] db_api.new_job(ctx, job_type, resource_id) start_time = datetime.datetime.now() while True: current_time = datetime.datetime.now() delta = current_time - start_time if delta.seconds >= CONF.worker_handle_timeout: # quit when this handle is running for a long time break time_new = db_api.get_latest_timestamp(ctx, constants.JS_New, job_type, resource_id) time_success = db_api.get_latest_timestamp( ctx, constants.JS_Success, job_type, resource_id) if time_success and time_success >= time_new: break job = db_api.register_job(ctx, job_type, resource_id) if not job: # fail to obtain the lock, let other worker handle the job running_job = db_api.get_running_job(ctx, job_type, resource_id) if not running_job: # there are two reasons that running_job is None. one # is that the running job has just been finished, the # other is that all workers fail to register the job # due to deadlock exception. so we sleep and try again eventlet.sleep(CONF.worker_sleep_time) continue job_time = running_job['timestamp'] current_time = datetime.datetime.now() delta = current_time - job_time if delta.seconds > CONF.job_run_expire: # previous running job expires, we set its status to # fail and try again to obtain the lock db_api.finish_job(ctx, running_job['id'], False, time_new) LOG.warning(_LW('Job %(job)s of type %(job_type)s for ' 'resource %(resource)s expires, set ' 'its state to Fail'), {'job': running_job['id'], 'job_type': job_type, 'resource': resource_id}) eventlet.sleep(CONF.worker_sleep_time) continue else: # previous running job is still valid, we just leave # the job to the worker who holds the lock break # successfully obtain the lock, start to execute handler try: func(*args, **kwargs) except Exception: db_api.finish_job(ctx, job['id'], False, time_new) LOG.error(_LE('Job %(job)s of type %(job_type)s for ' 'resource %(resource)s fails'), {'job': job['id'], 'job_type': job_type, 'resource': resource_id}) break db_api.finish_job(ctx, job['id'], True, time_new) eventlet.sleep(CONF.worker_sleep_time)
def test_post(self, mock_context): mock_context.return_value = self.context # cover all job types for job_type in self.job_resource_map.keys(): job = self._prepare_job_element(job_type) kw_job = {'job': job} # failure case, only admin can create the job self.context.is_admin = False res = self.controller.post(**kw_job) self._validate_error_code(res, 403) self.context.is_admin = True # failure case, request body not found kw_job_1 = {'job_1': job} res = self.controller.post(**kw_job_1) self._validate_error_code(res, 400) # failure case, wrong job type parameter job_type_backup = job.pop('type') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = '' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = job_type_backup + '_1' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['type'] = job_type_backup # failure case, wrong resource parameter job_resource_backup = job.pop('resource') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['resource'] = copy.deepcopy(job_resource_backup) job['resource'].popitem() res = self.controller.post(**kw_job) self._validate_error_code(res, 400) fake_resource = 'fake_resource' job['resource'][fake_resource] = fake_resource res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['resource'] = job_resource_backup # failure case, wrong project id parameter project_id_backup = job.pop('project_id') res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = '' res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = uuidutils.generate_uuid() res = self.controller.post(**kw_job) self._validate_error_code(res, 400) job['project_id'] = project_id_backup # successful case, create an entirely new job. Because the job # status returned from controller has been formatted, so we not # only validate the database records, but also validate the return # value of the controller. job_1 = self.controller.post(**kw_job)['job'] job_in_db_1 = db_api.get_job(self.context, job_1['id']) self.assertEqual(job_type, job_in_db_1['type']) self.assertEqual(job['project_id'], job_in_db_1['project_id']) self.assertEqual(constants.JS_New, job_in_db_1['status']) self.assertEqual('NEW', job_1['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_1['resource'])) self.assertFalse('resource_id' in job_1) self.assertFalse('extra_id' in job_1) db_api.delete_job(self.context, job_1['id']) # successful case, target job already exists in the job table # and its status is NEW, then this newer job will be picked by # job handler. job_2 = self.controller.post(**kw_job)['job'] job_in_db_2 = db_api.get_job(self.context, job_2['id']) job_3 = self.controller.post(**kw_job)['job'] job_in_db_3 = db_api.get_job(self.context, job_3['id']) self.assertEqual(job_type, job_in_db_2['type']) self.assertEqual(job['project_id'], job_in_db_2['project_id']) self.assertEqual(constants.JS_New, job_in_db_2['status']) self.assertEqual('NEW', job_2['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_2['resource'])) self.assertFalse('resource_id' in job_2) self.assertFalse('extra_id' in job_2) self.assertEqual(job_type, job_in_db_3['type']) self.assertEqual(job['project_id'], job_in_db_3['project_id']) self.assertEqual(constants.JS_New, job_in_db_3['status']) self.assertEqual('NEW', job_3['status']) self.assertEqual(len(constants.job_resource_map[job['type']]), len(job_3['resource'])) self.assertFalse('resource_id' in job_3) self.assertFalse('extra_id' in job_3) db_api.finish_job(self.context, job_3['id'], False, timeutils.utcnow()) db_api.delete_job(self.context, job_3['id'])