def test_redo_job(self):
for job_type in self.all_job_types:
job = self._prepare_job_element(job_type)
jobs = [
# create an entirely new job
{
"job": job,
"expected_error": 200
},
]
self._test_and_check(jobs)
response = self.app.get('/v1.0/jobs')
return_job = response.json
jobs = return_job['jobs']
# redo a new job
for job in jobs:
response_1 = self.app.put('/v1.0/jobs/%(id)s' % {'id': job['id']},
expect_errors=True)
self.assertEqual(response_1.status_int, 200)
response_2 = self.app.put('/v1.0/jobs/123', expect_errors=True)
self.assertEqual(response_2.status_int, 404)
# redo a running job
job_type_3 = constants.JT_NETWORK_UPDATE
job_3 = self._prepare_job_element(job_type_3)
resource_id_3 = '#'.join([job_3['resource'][resource_id]
for resource_type, resource_id
in self.job_resource_map[job_type_3]])
job_running_3 = db_api.register_job(self.context,
job_3['project_id'],
job_type_3,
resource_id_3)
self.assertEqual(constants.JS_Running, job_running_3['status'])
response_3 = self.app.put('/v1.0/jobs/%(id)s' % {
'id': job_running_3['id']}, expect_errors=True)
self.assertEqual(response_3.status_int, 400)
# redo a failed job
job_type_4 = constants.JT_NETWORK_UPDATE
job_4 = self._prepare_job_element(job_type_4)
job_dict_4 = {
"job": job_4,
"expected_error": 200
}
response_4 = self.app.post_json('/v1.0/jobs',
dict(job=job_dict_4['job']),
expect_errors=True)
return_job_4 = response_4.json
self.assertEqual(response_4.status_int, 200)
db_api.finish_job(self.context,
return_job_4['job']['id'],
False, timeutils.utcnow())
job_fail_4 = db_api.get_job(self.context, return_job_4['job']['id'])
self.assertEqual(constants.JS_Fail, job_fail_4['status'])
response_5 = self.app.put('/v1.0/jobs/%(id)s' % {
'id': return_job_4['job']['id']}, expect_errors=True)
self.assertEqual(response_5.status_int, 200)
# redo a successful job
job_type_6 = constants.JT_NETWORK_UPDATE
job_6 = self._prepare_job_element(job_type_6)
job_dict_6 = {
"job": job_6,
"expected_error": 200
}
response_6 = self.app.post_json('/v1.0/jobs',
dict(job=job_dict_6['job']),
expect_errors=True)
return_job_6 = response_6.json
with self.context.session.begin():
job_dict = {'status': constants.JS_Success,
'timestamp': timeutils.utcnow(),
'extra_id': uuidutils.generate_uuid()}
core.update_resource(self.context, models.AsyncJob,
return_job_6['job']['id'], job_dict)
job_succ_6 = db_api.get_job(self.context, return_job_6['job']['id'])
self.assertEqual(constants.JS_Success, job_succ_6['status'])
response_7 = self.app.put('/v1.0/jobs/%(id)s' % {
'id': return_job_6['job']['id']}, expect_errors=True)
self.assertEqual(response_7.status_int, 400)
def handle_args(*args, **kwargs):
if IN_TEST:
# NOTE(zhiyuan) job mechanism will cause some unpredictable
# result in unit test so we would like to bypass it. However
# we have problem mocking a decorator which decorates member
# functions, that's why we use this label, not an elegant
# way though.
func(*args, **kwargs)
return
ctx = args[1]
payload = kwargs['payload']
resource_id = payload[job_type]
db_api.new_job(ctx, job_type, resource_id)
start_time = datetime.datetime.now()
while True:
current_time = datetime.datetime.now()
delta = current_time - start_time
if delta.seconds >= CONF.worker_handle_timeout:
# quit when this handle is running for a long time
break
time_new = db_api.get_latest_timestamp(ctx, constants.JS_New,
job_type, resource_id)
time_success = db_api.get_latest_timestamp(
ctx, constants.JS_Success, job_type, resource_id)
if time_success and time_success >= time_new:
break
job = db_api.register_job(ctx, job_type, resource_id)
if not job:
# fail to obtain the lock, let other worker handle the job
running_job = db_api.get_running_job(ctx, job_type,
resource_id)
if not running_job:
# there are two reasons that running_job is None. one
# is that the running job has just been finished, the
# other is that all workers fail to register the job
# due to deadlock exception. so we sleep and try again
eventlet.sleep(CONF.worker_sleep_time)
continue
job_time = running_job['timestamp']
current_time = datetime.datetime.now()
delta = current_time - job_time
if delta.seconds > CONF.job_run_expire:
# previous running job expires, we set its status to
# fail and try again to obtain the lock
db_api.finish_job(ctx, running_job['id'], False,
time_new)
LOG.warning(_LW('Job %(job)s of type %(job_type)s for '
'resource %(resource)s expires, set '
'its state to Fail'),
{'job': running_job['id'],
'job_type': job_type,
'resource': resource_id})
eventlet.sleep(CONF.worker_sleep_time)
continue
else:
# previous running job is still valid, we just leave
# the job to the worker who holds the lock
break
# successfully obtain the lock, start to execute handler
try:
func(*args, **kwargs)
except Exception:
db_api.finish_job(ctx, job['id'], False, time_new)
LOG.error(_LE('Job %(job)s of type %(job_type)s for '
'resource %(resource)s fails'),
{'job': job['id'],
'job_type': job_type,
'resource': resource_id})
break
db_api.finish_job(ctx, job['id'], True, time_new)
eventlet.sleep(CONF.worker_sleep_time)
def test_delete(self, mock_context):
mock_context.return_value = self.context
# cover all job types.
# each 'for' loop adds one item in job log table, we set count variable
# to record dynamic total job entries in job log table.
count = 1
for job_type in self.job_resource_map.keys():
job = self._prepare_job_element(job_type)
resource_id = '#'.join([
job['resource'][resource_id] for resource_type, resource_id in
self.job_resource_map[job_type]
])
# failure case, only admin can delete the job
job_1 = db_api.new_job(self.context, job['project_id'], job_type,
resource_id)
self.context.is_admin = False
res = self.controller.delete(job_1['id'])
self._validate_error_code(res, 403)
self.context.is_admin = True
db_api.delete_job(self.context, job_1['id'])
# failure case, job not found
res = self.controller.delete(-123)
self._validate_error_code(res, 404)
# failure case, delete a running job
job_2 = db_api.register_job(self.context, job['project_id'],
job_type, resource_id)
job = db_api.get_job(self.context, job_2['id'])
res = self.controller.delete(job_2['id'])
self._validate_error_code(res, 400)
# finish the job and delete it
db_api.finish_job(self.context, job_2['id'], False,
timeutils.utcnow())
db_api.delete_job(self.context, job_2['id'])
# successful case, delete a successful job. successful job from
# job log can't be deleted, here this successful job is from
# job table.
job_3 = self._prepare_job_element(job_type)
resource_id_3 = '#'.join([
job_3['resource'][resource_id_3] for resource_type_3,
resource_id_3 in self.job_resource_map[job_type]
])
job_4 = db_api.new_job(self.context, job_3['project_id'], job_type,
resource_id_3)
with self.context.session.begin():
job_dict = {
'status': constants.JS_Success,
'timestamp': timeutils.utcnow(),
'extra_id': uuidutils.generate_uuid()
}
core.update_resource(self.context, models.AsyncJob,
job_4['id'], job_dict)
job_4_succ = db_api.get_job(self.context, job_4['id'])
self.controller.delete(job_4['id'])
filters_job_4 = [{
'key': 'type',
'comparator': 'eq',
'value': job_4_succ['type']
}, {
'key': 'status',
'comparator': 'eq',
'value': job_4_succ['status']
}, {
'key': 'resource_id',
'comparator': 'eq',
'value': job_4_succ['resource_id']
}, {
'key': 'extra_id',
'comparator': 'eq',
'value': job_4_succ['extra_id']
}]
self.assertEqual(
0, len(db_api.list_jobs(self.context, filters_job_4)))
self.assertEqual(count,
len(db_api.list_jobs_from_log(self.context)))
count = count + 1
# successful case, delete a new job
job_5 = db_api.new_job(self.context, job['project_id'], job_type,
resource_id)
self.controller.delete(job_5['id'])
filters_job_5 = [{
'key': 'type',
'comparator': 'eq',
'value': job_5['type']
}, {
'key': 'status',
'comparator': 'eq',
'value': job_5['status']
}, {
'key': 'resource_id',
'comparator': 'eq',
'value': job_5['resource_id']
}, {
'key': 'extra_id',
'comparator': 'eq',
'value': job_5['extra_id']
}]
self.assertEqual(
0, len(db_api.list_jobs(self.context, filters_job_5)))
# successful case, delete a failed job
job_6 = db_api.new_job(self.context, job['project_id'], job_type,
resource_id)
db_api.finish_job(self.context, job_6['id'], False,
timeutils.utcnow())
job_6_failed = db_api.get_job(self.context, job_6['id'])
self.controller.delete(job_6['id'])
filters_job_6 = [{
'key': 'type',
'comparator': 'eq',
'value': job_6_failed['type']
}, {
'key': 'status',
'comparator': 'eq',
'value': job_6_failed['status']
}, {
'key': 'resource_id',
'comparator': 'eq',
'value': job_6_failed['resource_id']
}, {
'key': 'extra_id',
'comparator': 'eq',
'value': job_6_failed['extra_id']
}]
self.assertEqual(
0, len(db_api.list_jobs(self.context, filters_job_6)))
def test_put(self, mock_context):
mock_context.return_value = self.context
# cover all job types
for job_type in self.job_resource_map.keys():
job = self._prepare_job_element(job_type)
resource_id = '#'.join([
job['resource'][resource_id] for resource_type, resource_id in
self.job_resource_map[job_type]
])
# failure case, only admin can redo the job
job_1 = db_api.new_job(self.context, job['project_id'], job_type,
resource_id)
self.context.is_admin = False
res = self.controller.put(job_1['id'])
self._validate_error_code(res, 403)
self.context.is_admin = True
db_api.delete_job(self.context, job_1['id'])
# failure case, job not found
res = self.controller.put(-123)
self._validate_error_code(res, 404)
# failure case, redo a running job
job_2 = db_api.register_job(self.context, job['project_id'],
job_type, resource_id)
res = self.controller.put(job_2['id'])
self._validate_error_code(res, 400)
db_api.finish_job(self.context, job_2['id'], False,
timeutils.utcnow())
db_api.delete_job(self.context, job_2['id'])
# failure case, redo a successful job
job_3 = self._prepare_job_element(job_type)
resource_id_3 = '#'.join([
job_3['resource'][resource_id_3] for resource_type_3,
resource_id_3 in self.job_resource_map[job_type]
])
job_4 = db_api.new_job(self.context, job_3['project_id'], job_type,
resource_id_3)
with self.context.session.begin():
job_dict = {
'status': constants.JS_Success,
'timestamp': timeutils.utcnow(),
'extra_id': uuidutils.generate_uuid()
}
core.update_resource(self.context, models.AsyncJob,
job_4['id'], job_dict)
res = self.controller.put(job_4['id'])
self._validate_error_code(res, 400)
db_api.finish_job(self.context, job_4['id'], True,
timeutils.utcnow())
# successful case, redo a failed job
job_5 = db_api.new_job(self.context, job['project_id'], job_type,
resource_id)
db_api.finish_job(self.context, job_5['id'], False,
timeutils.utcnow())
self.controller.put(job_5['id'])
db_api.delete_job(self.context, job_5['id'])
# successful case, redo a new job
job_6 = db_api.new_job(self.context, job['project_id'], job_type,
resource_id)
self.controller.put(job_6['id'])
db_api.delete_job(self.context, job_6['id'])
def test_put(self, mock_context):
mock_context.return_value = self.context
# cover all job types
for job_type in self.job_resource_map.keys():
job = self._prepare_job_element(job_type)
resource_id = '#'.join([job['resource'][resource_id]
for resource_type, resource_id
in self.job_resource_map[job_type]])
# failure case, only admin can redo the job
job_1 = db_api.new_job(self.context,
job['project_id'],
job_type, resource_id)
self.context.is_admin = False
res = self.controller.put(job_1['id'])
self._validate_error_code(res, 403)
self.context.is_admin = True
db_api.delete_job(self.context, job_1['id'])
# failure case, job not found
res = self.controller.put(-123)
self._validate_error_code(res, 404)
# failure case, redo a running job
job_2 = db_api.register_job(self.context,
job['project_id'],
job_type, resource_id)
res = self.controller.put(job_2['id'])
self._validate_error_code(res, 400)
db_api.finish_job(self.context, job_2['id'], False,
timeutils.utcnow())
db_api.delete_job(self.context, job_2['id'])
# failure case, redo a successful job
job_3 = self._prepare_job_element(job_type)
resource_id_3 = '#'.join([job_3['resource'][resource_id_3]
for resource_type_3, resource_id_3
in self.job_resource_map[job_type]])
job_4 = db_api.new_job(self.context,
job_3['project_id'],
job_type, resource_id_3)
with self.context.session.begin():
job_dict = {'status': constants.JS_Success,
'timestamp': timeutils.utcnow(),
'extra_id': uuidutils.generate_uuid()}
core.update_resource(self.context, models.AsyncJob,
job_4['id'], job_dict)
res = self.controller.put(job_4['id'])
self._validate_error_code(res, 400)
db_api.finish_job(self.context, job_4['id'], True,
timeutils.utcnow())
# successful case, redo a failed job
job_5 = db_api.new_job(self.context,
job['project_id'],
job_type, resource_id)
db_api.finish_job(self.context, job_5['id'], False,
timeutils.utcnow())
self.controller.put(job_5['id'])
db_api.delete_job(self.context, job_5['id'])
# successful case, redo a new job
job_6 = db_api.new_job(self.context,
job['project_id'],
job_type, resource_id)
self.controller.put(job_6['id'])
db_api.delete_job(self.context, job_6['id'])
def test_get_all_jobs(self, mock_context):
mock_context.return_value = self.context
# map job type to project id for later project id filter validation.
job_project_id_map = {}
amount_of_all_jobs = len(self.job_resource_map.keys())
amount_of_running_jobs = 3
count = 1
# cover all job types.
for job_type in self.job_resource_map.keys():
job = self._prepare_job_element(job_type)
job_project_id_map[job_type] = job['project_id']
resource_id = '#'.join([
job['resource'][resource_id] for resource_type, resource_id in
self.job_resource_map[job_type]
])
if count <= amount_of_running_jobs:
db_api.register_job(self.context, job['project_id'], job_type,
resource_id)
else:
db_api.new_job(self.context, job['project_id'], job_type,
resource_id)
count = count + 1
# query the jobs with several kinds of filters.
# supported filters: project id, job status, job type.
job_status_filter_1 = {'status': 'new'}
job_status_filter_2 = {'status': 'fail'}
job_status_filter_3 = {'status': 'running'}
invalid_filter = {'status': "new-x"}
unsupported_filter = {'fake_filter': "fake_filter"}
count = 1
for job_type in self.job_resource_map.keys():
project_id_filter_1 = {'project_id': job_project_id_map[job_type]}
project_id_filter_2 = {'project_id': uuidutils.generate_uuid()}
job_type_filter_1 = {'type': job_type}
job_type_filter_2 = {'type': job_type + '_1'}
# failure case, only admin can list the jobs
self.context.is_admin = False
res = self.controller.get_all()
self._validate_error_code(res, 403)
self.context.is_admin = True
# successful case, filter by project id
jobs_project_id_filter_1 = self.controller.get_all(
**project_id_filter_1)
self.assertEqual(1, len(jobs_project_id_filter_1['jobs']))
jobs_project_id_filter_2 = self.controller.get_all(
**project_id_filter_2)
self.assertEqual(0, len(jobs_project_id_filter_2['jobs']))
# successful case, filter by job type
jobs_job_type_filter_1 = self.controller.get_all(
**job_type_filter_1)
self.assertEqual(1, len(jobs_job_type_filter_1['jobs']))
jobs_job_type_filter_2 = self.controller.get_all(
**job_type_filter_2)
self.assertEqual(0, len(jobs_job_type_filter_2['jobs']))
# successful case, filter by project id, job status and job type
if count <= amount_of_running_jobs:
all_filters = dict(
list(project_id_filter_1.items()) +
list(job_status_filter_3.items()) +
list(job_type_filter_1.items()))
jobs_all_filters = self.controller.get_all(**all_filters)
self.assertEqual(1, len(jobs_all_filters['jobs']))
else:
all_filters = dict(
list(project_id_filter_1.items()) +
list(job_status_filter_1.items()) +
list(job_type_filter_1.items()))
jobs_all_filters = self.controller.get_all(**all_filters)
self.assertEqual(1, len(jobs_all_filters['jobs']))
# successful case, contradictory filter
contradict_filters = dict(
list(project_id_filter_1.items()) +
list(job_status_filter_2.items()) +
list((job_type_filter_2.items())))
jobs_contradict_filters = self.controller.get_all(
**contradict_filters)
self.assertEqual(0, len(jobs_contradict_filters['jobs']))
count = count + 1
# failure case, unsupported filter
res = self.controller.get_all(**unsupported_filter)
self._validate_error_code(res, 400)
# successful case, invalid filter
jobs_invalid_filter = self.controller.get_all(**invalid_filter)
self.assertEqual(0, len(jobs_invalid_filter['jobs']))
# successful case, list jobs without filters
jobs_empty_filters = self.controller.get_all()
self.assertEqual(amount_of_all_jobs, len(jobs_empty_filters['jobs']))
# successful case, filter by job status
jobs_job_status_filter_1 = self.controller.get_all(
**job_status_filter_1)
self.assertEqual(amount_of_all_jobs - amount_of_running_jobs,
len(jobs_job_status_filter_1['jobs']))
jobs_job_status_filter_2 = self.controller.get_all(
**job_status_filter_2)
self.assertEqual(0, len(jobs_job_status_filter_2['jobs']))
jobs_job_status_filter_3 = self.controller.get_all(
**job_status_filter_3)
self.assertEqual(amount_of_running_jobs,
len(jobs_job_status_filter_3['jobs']))
def test_delete(self, mock_context):
mock_context.return_value = self.context
# cover all job types.
# each 'for' loop adds one item in job log table, we set count variable
# to record dynamic total job entries in job log table.
count = 1
for job_type in self.job_resource_map.keys():
job = self._prepare_job_element(job_type)
resource_id = '#'.join([job['resource'][resource_id]
for resource_type, resource_id
in self.job_resource_map[job_type]])
# failure case, only admin can delete the job
job_1 = db_api.new_job(self.context, job['project_id'],
job_type,
resource_id)
self.context.is_admin = False
res = self.controller.delete(job_1['id'])
self._validate_error_code(res, 403)
self.context.is_admin = True
db_api.delete_job(self.context, job_1['id'])
# failure case, job not found
res = self.controller.delete(-123)
self._validate_error_code(res, 404)
# failure case, delete a running job
job_2 = db_api.register_job(self.context,
job['project_id'],
job_type, resource_id)
job = db_api.get_job(self.context, job_2['id'])
res = self.controller.delete(job_2['id'])
self._validate_error_code(res, 400)
# finish the job and delete it
db_api.finish_job(self.context, job_2['id'], False,
timeutils.utcnow())
db_api.delete_job(self.context, job_2['id'])
# successful case, delete a successful job. successful job from
# job log can't be deleted, here this successful job is from
# job table.
job_3 = self._prepare_job_element(job_type)
resource_id_3 = '#'.join([job_3['resource'][resource_id_3]
for resource_type_3, resource_id_3
in self.job_resource_map[job_type]])
job_4 = db_api.new_job(self.context,
job_3['project_id'],
job_type, resource_id_3)
with self.context.session.begin():
job_dict = {'status': constants.JS_Success,
'timestamp': timeutils.utcnow(),
'extra_id': uuidutils.generate_uuid()}
core.update_resource(self.context, models.AsyncJob,
job_4['id'], job_dict)
job_4_succ = db_api.get_job(self.context, job_4['id'])
self.controller.delete(job_4['id'])
filters_job_4 = [
{'key': 'type', 'comparator': 'eq',
'value': job_4_succ['type']},
{'key': 'status', 'comparator': 'eq',
'value': job_4_succ['status']},
{'key': 'resource_id', 'comparator': 'eq',
'value': job_4_succ['resource_id']},
{'key': 'extra_id', 'comparator': 'eq',
'value': job_4_succ['extra_id']}]
self.assertEqual(0, len(db_api.list_jobs(self.context,
filters_job_4)))
self.assertEqual(count,
len(db_api.list_jobs_from_log(self.context)))
count = count + 1
# successful case, delete a new job
job_5 = db_api.new_job(self.context,
job['project_id'], job_type,
resource_id)
self.controller.delete(job_5['id'])
filters_job_5 = [
{'key': 'type', 'comparator': 'eq', 'value': job_5['type']},
{'key': 'status', 'comparator': 'eq',
'value': job_5['status']},
{'key': 'resource_id', 'comparator': 'eq',
'value': job_5['resource_id']},
{'key': 'extra_id', 'comparator': 'eq',
'value': job_5['extra_id']}]
self.assertEqual(0, len(db_api.list_jobs(self.context,
filters_job_5)))
# successful case, delete a failed job
job_6 = db_api.new_job(self.context,
job['project_id'], job_type,
resource_id)
db_api.finish_job(self.context, job_6['id'], False,
timeutils.utcnow())
job_6_failed = db_api.get_job(self.context, job_6['id'])
self.controller.delete(job_6['id'])
filters_job_6 = [
{'key': 'type', 'comparator': 'eq',
'value': job_6_failed['type']},
{'key': 'status', 'comparator': 'eq',
'value': job_6_failed['status']},
{'key': 'resource_id', 'comparator': 'eq',
'value': job_6_failed['resource_id']},
{'key': 'extra_id', 'comparator': 'eq',
'value': job_6_failed['extra_id']}]
self.assertEqual(0, len(db_api.list_jobs(self.context,
filters_job_6)))
def test_get_all_jobs_with_pagination(self, mock_context):
self.context.project_id = uuidutils.generate_uuid()
mock_context.return_value = self.context
# map job type to project id for later project id filter validation.
job_project_id_map = {}
amount_of_all_jobs = len(self.job_resource_map.keys())
amount_of_running_jobs = 3
count = 1
# cover all job types.
for job_type in self.job_resource_map.keys():
job = self._prepare_job_element(job_type)
if count > 1:
# for test convenience, the first job has a project ID
# that is different from the context.project_id
job['project_id'] = self.context.project_id
job_project_id_map[job_type] = job['project_id']
resource_id = '#'.join([job['resource'][resource_id]
for resource_type, resource_id
in self.job_resource_map[job_type]])
if count <= amount_of_running_jobs:
db_api.register_job(self.context,
job['project_id'], job_type,
resource_id)
# because jobs are sorted by timestamp, without time delay then
# all jobs are created at the same time, paginate_query can't
# identify them
time.sleep(1)
else:
db_api.new_job(self.context,
job['project_id'], job_type,
resource_id)
time.sleep(1)
count = count + 1
# query the jobs with several kinds of filters.
# supported filters: project id, job status, job type.
job_status_filter_1 = {'status': 'new'}
job_status_filter_2 = {'status': 'fail'}
job_status_filter_3 = {'status': 'running'}
invalid_filter = {'status': "new-x"}
unsupported_filter = {'fake_filter': "fake_filter"}
count = 1
for job_type in self.job_resource_map.keys():
job_type_filter_1 = {'type': job_type}
job_type_filter_2 = {'type': job_type + '_1'}
# failure case, only admin can list the jobs
self.context.is_admin = False
res = self.controller.get_all()
self._validate_error_code(res, 403)
self.context.is_admin = True
# test when specify project ID filter from client, if this
# project ID is different from the one from context, then
# it will be ignored, project ID from context will be
# used instead.
filter1 = {'project_id': uuidutils.generate_uuid()}
res1 = self.controller.get_all(**filter1)
filter2 = {'project_id': self.context.project_id}
res2 = self.controller.get_all(**filter2)
self.assertEqual(len(res2['jobs']), len(res1['jobs']))
res3 = self.controller.get_all()
# there is one job whose project ID is different from
# context.project_id. As the list operation only retrieves the
# jobs whose project ID equals to context.project_id, so this
# special job entry won't be retrieved.
self.assertEqual(len(res3['jobs']), len(res2['jobs']))
# successful case, filter by job type
jobs_job_type_filter_1 = self.controller.get_all(
**job_type_filter_1)
if count == 1:
self.assertEqual(0, len(jobs_job_type_filter_1['jobs']))
else:
self.assertEqual(1, len(jobs_job_type_filter_1['jobs']))
jobs_job_type_filter_2 = self.controller.get_all(
**job_type_filter_2)
self.assertEqual(0, len(jobs_job_type_filter_2['jobs']))
# successful case, filter by job status and job type
if count <= amount_of_running_jobs:
all_filters = dict(list(job_status_filter_3.items()) +
list(job_type_filter_1.items()))
jobs_all_filters = self.controller.get_all(**all_filters)
if count == 1:
self.assertEqual(0, len(jobs_all_filters['jobs']))
else:
self.assertEqual(1, len(jobs_all_filters['jobs']))
else:
all_filters = dict(list(job_status_filter_1.items()) +
list(job_type_filter_1.items()))
jobs_all_filters = self.controller.get_all(**all_filters)
self.assertEqual(1, len(jobs_all_filters['jobs']))
# successful case, contradictory filter
contradict_filters = dict(list(job_status_filter_2.items()) +
list((job_type_filter_2.items())))
jobs_contradict_filters = self.controller.get_all(
**contradict_filters)
self.assertEqual(0, len(jobs_contradict_filters['jobs']))
count = count + 1
# failure case, unsupported filter
res = self.controller.get_all(**unsupported_filter)
self._validate_error_code(res, 400)
# successful case, invalid filter
jobs_invalid_filter = self.controller.get_all(**invalid_filter)
self.assertEqual(0, len(jobs_invalid_filter['jobs']))
# successful case, list jobs without filters
jobs_empty_filters = self.controller.get_all()
self.assertEqual(amount_of_all_jobs - 1,
len(jobs_empty_filters['jobs']))
# successful case, filter by job status
jobs_job_status_filter_1 = self.controller.get_all(
**job_status_filter_1)
self.assertEqual(amount_of_all_jobs - amount_of_running_jobs,
len(jobs_job_status_filter_1['jobs']))
jobs_job_status_filter_2 = self.controller.get_all(
**job_status_filter_2)
self.assertEqual(0, len(jobs_job_status_filter_2['jobs']))
jobs_job_status_filter_3 = self.controller.get_all(
**job_status_filter_3)
self.assertEqual(amount_of_running_jobs - 1,
len(jobs_job_status_filter_3['jobs']))
# test for paginate query
job_paginate_no_filter_1 = self.controller.get_all()
self.assertEqual(amount_of_all_jobs - 1,
len(job_paginate_no_filter_1['jobs']))
# no limit no marker
job_paginate_filter_1 = {'status': 'new'}
jobs_paginate_filter_1 = self.controller.get_all(
**job_paginate_filter_1)
self.assertEqual(amount_of_all_jobs - amount_of_running_jobs,
len(jobs_paginate_filter_1['jobs']))
# failed cases, unsupported limit type
job_paginate_filter_2 = {'limit': '2test'}
res = self.controller.get_all(**job_paginate_filter_2)
self._validate_error_code(res, 400)
# successful cases
job_paginate_filter_4 = {'status': 'new', 'limit': '2'}
res = self.controller.get_all(**job_paginate_filter_4)
self.assertEqual(2, len(res['jobs']))
job_paginate_filter_5 = {'status': 'new', 'limit': 2}
res = self.controller.get_all(**job_paginate_filter_5)
self.assertEqual(2, len(res['jobs']))
job_paginate_filter_6 = {'status': 'running', 'limit': 1}
res1 = self.controller.get_all(**job_paginate_filter_6)
marker = res1['jobs'][0]['id']
job_paginate_filter_7 = {'status': 'running', 'marker': marker}
res2 = self.controller.get_all(**job_paginate_filter_7)
self.assertEqual(amount_of_running_jobs - 1, len(res2['jobs']))
job_paginate_filter_8 = {'status': 'new', 'limit': 3}
res = self.controller.get_all(**job_paginate_filter_8)
self.assertGreaterEqual(res['jobs'][0]['timestamp'],
res['jobs'][1]['timestamp'])
self.assertGreaterEqual(res['jobs'][1]['timestamp'],
res['jobs'][2]['timestamp'])
# unsupported marker type
res = self.controller.get_all(marker=None)
self.assertEqual(amount_of_all_jobs - 1, len(res['jobs']))
res = self.controller.get_all(marker='-123')
self._validate_error_code(res, 400)
# marker not in job table and job log table
job_paginate_filter_9 = {'marker': uuidutils.generate_uuid()}
res = self.controller.get_all(**job_paginate_filter_9)
self._validate_error_code(res, 400)
# test marker and limit
limit = 2
pt = '/v1.0/jobs\?limit=\w+&marker=([\w-]+)'
job_paginate_filter = {'status': 'new', 'limit': limit}
res = self.controller.get_all(**job_paginate_filter)
while 'jobs_links' in res:
m = re.match(pt, res['jobs_links'][0]['href'])
marker = m.group(1)
self.assertEqual(limit, len(res['jobs']))
job_paginate_filter = {'status': 'new', 'limit': limit,
'marker': marker}
res = self.controller.get_all(**job_paginate_filter)
job_paginate_filter_10 = {'status': 'running'}
res = self.controller.get_all(**job_paginate_filter_10)
self.assertEqual(amount_of_running_jobs - 1, len(res['jobs']))
# add some rows to job log table
for i in xrange(amount_of_running_jobs - 1):
db_api.finish_job(self.context, res['jobs'][i]['id'], True,
timeutils.utcnow())
time.sleep(1)
res_success_log = db_api.list_jobs_from_log(self.context, None)
self.assertEqual(amount_of_running_jobs - 1, len(res_success_log))
res_in_job = db_api.list_jobs(self.context, None)
self.assertEqual(amount_of_all_jobs - (amount_of_running_jobs - 1),
len(res_in_job))
job_paginate_filter_11 = {'limit': 2}
res = self.controller.get_all(**job_paginate_filter_11)
self.assertIsNotNone(res['jobs_links'][0]['href'])
def test_redo_job(self):
for job_type in self.all_job_types:
job = self._prepare_job_element(job_type)
jobs = [
# create an entirely new job
{
"job": job,
"expected_error": 200
},
]
self._test_and_check(jobs)
response = self.app.get('/v1.0/jobs')
return_job = response.json
jobs = return_job['jobs']
# redo a new job
for job in jobs:
response_1 = self.app.put('/v1.0/jobs/%(id)s' % {'id': job['id']},
expect_errors=True)
self.assertEqual(response_1.status_int, 200)
response_2 = self.app.put('/v1.0/jobs/123', expect_errors=True)
self.assertEqual(response_2.status_int, 404)
# redo a running job
job_type_3 = constants.JT_NETWORK_UPDATE
job_3 = self._prepare_job_element(job_type_3)
resource_id_3 = '#'.join([job_3['resource'][resource_id]
for resource_type, resource_id
in self.job_resource_map[job_type_3]])
job_running_3 = db_api.register_job(self.context,
job_3['project_id'],
job_type_3,
resource_id_3)
self.assertEqual(constants.JS_Running, job_running_3['status'])
response_3 = self.app.put('/v1.0/jobs/%(id)s' % {
'id': job_running_3['id']}, expect_errors=True)
self.assertEqual(response_3.status_int, 400)
# redo a failed job
job_type_4 = constants.JT_NETWORK_UPDATE
job_4 = self._prepare_job_element(job_type_4)
job_dict_4 = {
"job": job_4,
"expected_error": 200
}
response_4 = self.app.post_json('/v1.0/jobs',
dict(job=job_dict_4['job']),
expect_errors=True)
return_job_4 = response_4.json
self.assertEqual(response_4.status_int, 200)
db_api.finish_job(self.context,
return_job_4['job']['id'],
False, timeutils.utcnow())
job_fail_4 = db_api.get_job(self.context, return_job_4['job']['id'])
self.assertEqual(constants.JS_Fail, job_fail_4['status'])
response_5 = self.app.put('/v1.0/jobs/%(id)s' % {
'id': return_job_4['job']['id']}, expect_errors=True)
self.assertEqual(response_5.status_int, 200)
# redo a successful job
job_type_6 = constants.JT_NETWORK_UPDATE
job_6 = self._prepare_job_element(job_type_6)
job_dict_6 = {
"job": job_6,
"expected_error": 200
}
response_6 = self.app.post_json('/v1.0/jobs',
dict(job=job_dict_6['job']),
expect_errors=True)
return_job_6 = response_6.json
with self.context.session.begin():
job_dict = {'status': constants.JS_Success,
'timestamp': timeutils.utcnow(),
'extra_id': uuidutils.generate_uuid()}
core.update_resource(self.context, models.AsyncJob,
return_job_6['job']['id'], job_dict)
job_succ_6 = db_api.get_job(self.context, return_job_6['job']['id'])
self.assertEqual(constants.JS_Success, job_succ_6['status'])
response_7 = self.app.put('/v1.0/jobs/%(id)s' % {
'id': return_job_6['job']['id']}, expect_errors=True)
self.assertEqual(response_7.status_int, 400)
def test_get_all_jobs_with_pagination(self, mock_context):
self.context.project_id = uuidutils.generate_uuid()
mock_context.return_value = self.context
# map job type to project id for later project id filter validation.
job_project_id_map = {}
amount_of_all_jobs = len(self.job_resource_map.keys())
amount_of_running_jobs = 3
count = 1
# cover all job types.
for job_type in self.job_resource_map.keys():
job = self._prepare_job_element(job_type)
if count > 1:
# for test convenience, the first job has a project ID
# that is different from the context.project_id
job['project_id'] = self.context.project_id
job_project_id_map[job_type] = job['project_id']
resource_id = '#'.join([
job['resource'][resource_id] for resource_type, resource_id in
self.job_resource_map[job_type]
])
if count <= amount_of_running_jobs:
db_api.register_job(self.context, job['project_id'], job_type,
resource_id)
# because jobs are sorted by timestamp, without time delay then
# all jobs are created at the same time, paginate_query can't
# identify them
time.sleep(1)
else:
db_api.new_job(self.context, job['project_id'], job_type,
resource_id)
time.sleep(1)
count = count + 1
# query the jobs with several kinds of filters.
# supported filters: project id, job status, job type.
job_status_filter_1 = {'status': 'new'}
job_status_filter_2 = {'status': 'fail'}
job_status_filter_3 = {'status': 'running'}
invalid_filter = {'status': "new-x"}
unsupported_filter = {'fake_filter': "fake_filter"}
count = 1
for job_type in self.job_resource_map.keys():
job_type_filter_1 = {'type': job_type}
job_type_filter_2 = {'type': job_type + '_1'}
# failure case, only admin can list the jobs
self.context.is_admin = False
res = self.controller.get_all()
self._validate_error_code(res, 403)
self.context.is_admin = True
# test when specify project ID filter from client, if this
# project ID is different from the one from context, then
# it will be ignored, project ID from context will be
# used instead.
filter1 = {'project_id': uuidutils.generate_uuid()}
res1 = self.controller.get_all(**filter1)
filter2 = {'project_id': self.context.project_id}
res2 = self.controller.get_all(**filter2)
self.assertEqual(len(res2['jobs']), len(res1['jobs']))
res3 = self.controller.get_all()
# there is one job whose project ID is different from
# context.project_id. As the list operation only retrieves the
# jobs whose project ID equals to context.project_id, so this
# special job entry won't be retrieved.
self.assertEqual(len(res3['jobs']), len(res2['jobs']))
# successful case, filter by job type
jobs_job_type_filter_1 = self.controller.get_all(
**job_type_filter_1)
if count == 1:
self.assertEqual(0, len(jobs_job_type_filter_1['jobs']))
else:
self.assertEqual(1, len(jobs_job_type_filter_1['jobs']))
jobs_job_type_filter_2 = self.controller.get_all(
**job_type_filter_2)
self.assertEqual(0, len(jobs_job_type_filter_2['jobs']))
# successful case, filter by job status and job type
if count <= amount_of_running_jobs:
all_filters = dict(
list(job_status_filter_3.items()) +
list(job_type_filter_1.items()))
jobs_all_filters = self.controller.get_all(**all_filters)
if count == 1:
self.assertEqual(0, len(jobs_all_filters['jobs']))
else:
self.assertEqual(1, len(jobs_all_filters['jobs']))
else:
all_filters = dict(
list(job_status_filter_1.items()) +
list(job_type_filter_1.items()))
jobs_all_filters = self.controller.get_all(**all_filters)
self.assertEqual(1, len(jobs_all_filters['jobs']))
# successful case, contradictory filter
contradict_filters = dict(
list(job_status_filter_2.items()) +
list((job_type_filter_2.items())))
jobs_contradict_filters = self.controller.get_all(
**contradict_filters)
self.assertEqual(0, len(jobs_contradict_filters['jobs']))
count = count + 1
# failure case, unsupported filter
res = self.controller.get_all(**unsupported_filter)
self._validate_error_code(res, 400)
# successful case, invalid filter
jobs_invalid_filter = self.controller.get_all(**invalid_filter)
self.assertEqual(0, len(jobs_invalid_filter['jobs']))
# successful case, list jobs without filters
jobs_empty_filters = self.controller.get_all()
self.assertEqual(amount_of_all_jobs - 1,
len(jobs_empty_filters['jobs']))
# successful case, filter by job status
jobs_job_status_filter_1 = self.controller.get_all(
**job_status_filter_1)
self.assertEqual(amount_of_all_jobs - amount_of_running_jobs,
len(jobs_job_status_filter_1['jobs']))
jobs_job_status_filter_2 = self.controller.get_all(
**job_status_filter_2)
self.assertEqual(0, len(jobs_job_status_filter_2['jobs']))
jobs_job_status_filter_3 = self.controller.get_all(
**job_status_filter_3)
self.assertEqual(amount_of_running_jobs - 1,
len(jobs_job_status_filter_3['jobs']))
# test for paginate query
job_paginate_no_filter_1 = self.controller.get_all()
self.assertEqual(amount_of_all_jobs - 1,
len(job_paginate_no_filter_1['jobs']))
# no limit no marker
job_paginate_filter_1 = {'status': 'new'}
jobs_paginate_filter_1 = self.controller.get_all(
**job_paginate_filter_1)
self.assertEqual(amount_of_all_jobs - amount_of_running_jobs,
len(jobs_paginate_filter_1['jobs']))
# failed cases, unsupported limit type
job_paginate_filter_2 = {'limit': '2test'}
res = self.controller.get_all(**job_paginate_filter_2)
self._validate_error_code(res, 400)
# successful cases
job_paginate_filter_4 = {'status': 'new', 'limit': '2'}
res = self.controller.get_all(**job_paginate_filter_4)
self.assertEqual(2, len(res['jobs']))
job_paginate_filter_5 = {'status': 'new', 'limit': 2}
res = self.controller.get_all(**job_paginate_filter_5)
self.assertEqual(2, len(res['jobs']))
job_paginate_filter_6 = {'status': 'running', 'limit': 1}
res1 = self.controller.get_all(**job_paginate_filter_6)
marker = res1['jobs'][0]['id']
job_paginate_filter_7 = {'status': 'running', 'marker': marker}
res2 = self.controller.get_all(**job_paginate_filter_7)
self.assertEqual(amount_of_running_jobs - 1, len(res2['jobs']))
job_paginate_filter_8 = {'status': 'new', 'limit': 3}
res = self.controller.get_all(**job_paginate_filter_8)
self.assertGreaterEqual(res['jobs'][0]['timestamp'],
res['jobs'][1]['timestamp'])
self.assertGreaterEqual(res['jobs'][1]['timestamp'],
res['jobs'][2]['timestamp'])
# unsupported marker type
res = self.controller.get_all(marker=None)
self.assertEqual(amount_of_all_jobs - 1, len(res['jobs']))
res = self.controller.get_all(marker='-123')
self._validate_error_code(res, 400)
# marker not in job table and job log table
job_paginate_filter_9 = {'marker': uuidutils.generate_uuid()}
res = self.controller.get_all(**job_paginate_filter_9)
self._validate_error_code(res, 400)
# test marker and limit
limit = 2
pt = '/v1.0/jobs\?limit=\w+&marker=([\w-]+)'
job_paginate_filter = {'status': 'new', 'limit': limit}
res = self.controller.get_all(**job_paginate_filter)
while 'jobs_links' in res:
m = re.match(pt, res['jobs_links'][0]['href'])
marker = m.group(1)
self.assertEqual(limit, len(res['jobs']))
job_paginate_filter = {
'status': 'new',
'limit': limit,
'marker': marker
}
res = self.controller.get_all(**job_paginate_filter)
job_paginate_filter_10 = {'status': 'running'}
res = self.controller.get_all(**job_paginate_filter_10)
self.assertEqual(amount_of_running_jobs - 1, len(res['jobs']))
# add some rows to job log table
for i in xrange(amount_of_running_jobs - 1):
db_api.finish_job(self.context, res['jobs'][i]['id'], True,
timeutils.utcnow())
time.sleep(1)
res_success_log = db_api.list_jobs_from_log(self.context, None)
self.assertEqual(amount_of_running_jobs - 1, len(res_success_log))
res_in_job = db_api.list_jobs(self.context, None)
self.assertEqual(amount_of_all_jobs - (amount_of_running_jobs - 1),
len(res_in_job))
job_paginate_filter_11 = {'limit': 2}
res = self.controller.get_all(**job_paginate_filter_11)
self.assertIsNotNone(res['jobs_links'][0]['href'])
def handle_args(*args, **kwargs):
if IN_TEST:
# NOTE(zhiyuan) job mechanism will cause some unpredictable
# result in unit test so we would like to bypass it. However
# we have problem mocking a decorator which decorates member
# functions, that's why we use this label, not an elegant
# way though.
func(*args, **kwargs)
return
ctx = args[1]
payload = kwargs['payload']
resource_id = payload[job_type]
db_api.new_job(ctx, job_type, resource_id)
start_time = datetime.datetime.now()
while True:
current_time = datetime.datetime.now()
delta = current_time - start_time
if delta.seconds >= CONF.worker_handle_timeout:
# quit when this handle is running for a long time
break
time_new = db_api.get_latest_timestamp(ctx, constants.JS_New,
job_type, resource_id)
time_success = db_api.get_latest_timestamp(
ctx, constants.JS_Success, job_type, resource_id)
if time_success and time_success >= time_new:
break
job = db_api.register_job(ctx, job_type, resource_id)
if not job:
# fail to obtain the lock, let other worker handle the job
running_job = db_api.get_running_job(ctx, job_type,
resource_id)
if not running_job:
# there are two reasons that running_job is None. one
# is that the running job has just been finished, the
# other is that all workers fail to register the job
# due to deadlock exception. so we sleep and try again
eventlet.sleep(CONF.worker_sleep_time)
continue
job_time = running_job['timestamp']
current_time = datetime.datetime.now()
delta = current_time - job_time
if delta.seconds > CONF.job_run_expire:
# previous running job expires, we set its status to
# fail and try again to obtain the lock
db_api.finish_job(ctx, running_job['id'], False,
time_new)
LOG.warning(_LW('Job %(job)s of type %(job_type)s for '
'resource %(resource)s expires, set '
'its state to Fail'),
{'job': running_job['id'],
'job_type': job_type,
'resource': resource_id})
eventlet.sleep(CONF.worker_sleep_time)
continue
else:
# previous running job is still valid, we just leave
# the job to the worker who holds the lock
break
# successfully obtain the lock, start to execute handler
try:
func(*args, **kwargs)
except Exception:
db_api.finish_job(ctx, job['id'], False, time_new)
LOG.error(_LE('Job %(job)s of type %(job_type)s for '
'resource %(resource)s fails'),
{'job': job['id'],
'job_type': job_type,
'resource': resource_id})
break
db_api.finish_job(ctx, job['id'], True, time_new)
eventlet.sleep(CONF.worker_sleep_time)