def _get_request_params(self, request):
filter_args = {}
params = request.params
if params.get('next_run_after') is not None:
next_run_after = params['next_run_after']
next_run_after = timeutils.parse_isotime(next_run_after)
next_run_after = timeutils.normalize_time(next_run_after)
filter_args['next_run_after'] = next_run_after
if params.get('next_run_before') is not None:
next_run_before = params['next_run_before']
next_run_before = timeutils.parse_isotime(next_run_before)
next_run_before = timeutils.normalize_time(next_run_before)
filter_args['next_run_before'] = next_run_before
if request.params.get('tenant') is not None:
filter_args['tenant'] = request.params['tenant']
filter_args['limit'] = params.get('limit')
filter_args['marker'] = params.get('marker')
for filter_key in params.keys():
if filter_key not in filter_args:
filter_args[filter_key] = params[filter_key]
return filter_args
def _get_request_params(self, request):
filter_args = {}
params = request.params
if params.get('next_run_after') is not None:
next_run_after = params['next_run_after']
next_run_after = timeutils.parse_isotime(next_run_after)
next_run_after = timeutils.normalize_time(next_run_after)
filter_args['next_run_after'] = next_run_after
if params.get('next_run_before') is not None:
next_run_before = params['next_run_before']
next_run_before = timeutils.parse_isotime(next_run_before)
next_run_before = timeutils.normalize_time(next_run_before)
filter_args['next_run_before'] = next_run_before
if request.params.get('tenant') is not None:
filter_args['tenant'] = request.params['tenant']
filter_args['limit'] = params.get('limit')
filter_args['marker'] = params.get('marker')
for filter_key in params.keys():
if filter_key not in filter_args:
filter_args[filter_key] = params[filter_key]
return filter_args
def update(self, request, schedule_id, body):
if not body:
msg = _('The request body must not be empty')
raise webob.exc.HTTPBadRequest(explanation=msg)
elif 'schedule' not in body:
msg = _('The request body must contain a "schedule" entity')
raise webob.exc.HTTPBadRequest(explanation=msg)
# NOTE(jculp): only raise if a blank tenant is passed
# passing no tenant at all is perfectly fine.
elif('tenant' in body['schedule'] and not
body['schedule']['tenant'].strip()):
msg = _('The request body has not specified a "tenant" entity')
raise webob.exc.HTTPBadRequest(explanation=msg)
api_utils.deserialize_schedule_metadata(body['schedule'])
values = {}
values.update(body['schedule'])
try:
values = api_utils.check_read_only_properties(values)
except exception.Forbidden as e:
raise webob.exc.HTTPForbidden(explanation=unicode(e))
request_next_run = body['schedule'].get('next_run')
times = {
'minute': None,
'hour': None,
'month': None,
'day_of_week': None,
'day_of_month': None,
}
update_schedule_times = False
for key in times:
if key in values:
times[key] = values[key]
update_schedule_times = True
if update_schedule_times:
# NOTE(ameade): We must recalculate the schedules next_run time
# since the schedule has changed
values.update(times)
values['next_run'] = api_utils.schedule_to_next_run(times)
elif request_next_run:
try:
timeutils.parse_isotime(request_next_run)
except ValueError as e:
msg = _('Invalid "next_run" value. Must be ISO 8601 format')
raise webob.exc.HTTPBadRequest(explanation=msg)
try:
schedule = self.db_api.schedule_update(schedule_id, values)
except exception.NotFound:
msg = _('Schedule %s could not be found.') % schedule_id
raise webob.exc.HTTPNotFound(explanation=msg)
utils.serialize_datetimes(schedule)
api_utils.serialize_schedule_metadata(schedule)
return {'schedule': schedule}
def update(self, request, schedule_id, body):
if not body:
msg = _('The request body must not be empty')
raise webob.exc.HTTPBadRequest(explanation=msg)
elif not 'schedule' in body:
msg = _('The request body must contain a "schedule" entity')
raise webob.exc.HTTPBadRequest(explanation=msg)
# NOTE(jculp): only raise if a blank tenant is passed
# passing no tenant at all is perfectly fine.
elif ('tenant' in body['schedule'] and not
body['schedule']['tenant'].strip()):
msg = _('The request body has not specified a "tenant" entity')
raise webob.exc.HTTPBadRequest(explanation=msg)
api_utils.deserialize_schedule_metadata(body['schedule'])
values = {}
values.update(body['schedule'])
try:
values = api_utils.check_read_only_properties(values)
except exception.Forbidden as e:
raise webob.exc.HTTPForbidden(explanation=unicode(e))
request_next_run = body['schedule'].get('next_run')
times = {
'minute': None,
'hour': None,
'month': None,
'day_of_week': None,
'day_of_month': None,
}
update_schedule_times = False
for key in times:
if key in values:
times[key] = values[key]
update_schedule_times = True
if update_schedule_times:
# NOTE(ameade): We must recalculate the schedules next_run time
# since the schedule has changed
values.update(times)
values['next_run'] = api_utils.schedule_to_next_run(times)
elif request_next_run:
try:
timeutils.parse_isotime(request_next_run)
except ValueError as e:
msg = _('Invalid "next_run" value. Must be ISO 8601 format')
raise webob.exc.HTTPBadRequest(explanation=msg)
try:
schedule = self.db_api.schedule_update(schedule_id, values)
except exception.NotFound:
msg = _('Schedule %s could not be found.') % schedule_id
raise webob.exc.HTTPNotFound(explanation=msg)
utils.serialize_datetimes(schedule)
api_utils.serialize_schedule_metadata(schedule)
return {'schedule': schedule}
def test_create(self):
expected_next_run = timeutils.parse_isotime(
'1989-01-19T12:00:00Z').replace(tzinfo=None)
self._stub_notifications(None, 'qonos.job.create', 'fake-payload',
'INFO')
def fake_schedule_to_next_run(_schedule, start_time=None):
self.assertEqual(timeutils.utcnow(), start_time)
return expected_next_run
self.stubs.Set(api_utils, 'schedule_to_next_run',
fake_schedule_to_next_run)
request = unit_utils.get_fake_request(method='POST')
fixture = {'job': {'schedule_id': self.schedule_1['id']}}
job = self.controller.create(request, fixture).get('job')
self.assertNotEqual(job, None)
self.assertNotEqual(job.get('id'), None)
self.assertEqual(job['schedule_id'], self.schedule_1['id'])
self.assertEqual(job['tenant'], self.schedule_1['tenant'])
self.assertEqual(job['action'], self.schedule_1['action'])
self.assertEqual(job['status'], 'QUEUED')
self.assertEqual(len(job['metadata']), 0)
schedule = db_api.schedule_get_by_id(self.schedule_1['id'])
self.assertNotEqual(schedule['next_run'], self.schedule_1['next_run'])
self.assertEqual(schedule['next_run'], expected_next_run)
self.assertNotEqual(schedule['last_scheduled'],
self.schedule_1.get('last_scheduled'))
self.assertTrue(schedule.get('last_scheduled'))
def create(self, request, body):
if (body is None or body.get('job') is None or
body['job'].get('schedule_id') is None):
raise webob.exc.HTTPBadRequest()
job = body['job']
try:
schedule = self.db_api.schedule_get_by_id(job['schedule_id'])
except exception.NotFound:
raise webob.exc.HTTPNotFound()
# Check integrity of schedule and update next run
expected_next_run = job.get('next_run')
if expected_next_run:
expected_next_run = timeutils.parse_isotime(job.get('next_run'))
next_run = api_utils.schedule_to_next_run(schedule, timeutils.utcnow())
try:
self.db_api.schedule_test_and_set_next_run(schedule['id'],
expected_next_run, next_run)
except exception.NotFound:
msg = _("Specified next run does not match the current next run"
" value. This could mean schedule has either changed"
"or has already been scheduled since you last expected.")
raise webob.exc.HTTPConflict(explanation=msg)
# Update schedule last_scheduled
values = {}
values['last_scheduled'] = timeutils.utcnow()
self.db_api.schedule_update(schedule['id'], values)
# Create job
values = {}
values.update(job)
values['tenant'] = schedule['tenant']
values['action'] = schedule['action']
values['status'] = 'QUEUED'
job_metadata = []
for metadata in schedule['schedule_metadata']:
job_metadata.append({
'key': metadata['key'],
'value': metadata['value']
})
values['job_metadata'] = job_metadata
job_action = values['action']
if not 'timeout' in values:
values['timeout'] = api_utils.get_new_timeout_by_action(job_action)
values['hard_timeout'] = \
api_utils.get_new_timeout_by_action(job_action)
job = self.db_api.job_create(values)
utils.serialize_datetimes(job)
api_utils.serialize_job_metadata(job)
job = {'job': job}
utils.generate_notification(None, 'qonos.job.create', job, 'INFO')
return job
def test_create_with_next_run(self):
expected_next_run = timeutils.parse_isotime('1989-01-19T12:00:00Z')
def fake_schedule_to_next_run(_schedule, start_time=None):
self.assertEqual(timeutils.utcnow(), start_time)
return expected_next_run
self.stubs.Set(api_utils, 'schedule_to_next_run',
fake_schedule_to_next_run)
self._stub_notifications(None, 'qonos.job.create', 'fake-payload',
'INFO')
request = unit_utils.get_fake_request(method='POST')
fixture = {'job': {'schedule_id': self.schedule_1['id'],
'next_run':
timeutils.isotime(self.schedule_1['next_run'])}}
job = self.controller.create(request, fixture).get('job')
self.assertNotEqual(job, None)
self.assertNotEqual(job.get('id'), None)
self.assertEqual(job['schedule_id'], self.schedule_1['id'])
self.assertEqual(job['tenant'], self.schedule_1['tenant'])
self.assertEqual(job['action'], self.schedule_1['action'])
self.assertEqual(job['status'], 'QUEUED')
self.assertEqual(len(job['metadata']), 0)
schedule = db_api.schedule_get_by_id(self.schedule_1['id'])
self.assertNotEqual(schedule['next_run'], self.schedule_1['next_run'])
self.assertEqual(schedule['next_run'], expected_next_run)
self.assertNotEqual(schedule['last_scheduled'],
self.schedule_1.get('last_scheduled'))
self.assertTrue(schedule.get('last_scheduled'))
def list(self, request):
params = request.params.copy()
try:
params = utils.get_pagination_limit(params)
except exception.Invalid as e:
raise webob.exc.HTTPBadRequest(explanation=str(e))
if 'status' in params:
params['status'] = str(params['status']).upper()
if 'timeout' in params:
timeout = timeutils.parse_isotime(params['timeout'])
params['timeout'] = timeutils.normalize_time(timeout)
if 'hard_timeout' in params:
hard_timeout = timeutils.parse_isotime(params['hard_timeout'])
params['hard_timeout'] = timeutils.normalize_time(hard_timeout)
try:
jobs = self.db_api.job_get_all(params)
except exception.NotFound:
raise webob.exc.HTTPNotFound()
limit = params.get('limit')
if len(jobs) != 0 and len(jobs) == limit:
next_page = '/v1/jobs?marker=%s' % jobs[-1].get('id')
else:
next_page = None
for job in jobs:
utils.serialize_datetimes(job)
api_utils.serialize_job_metadata(job)
links = [{'rel': 'next', 'href': next_page}]
return {'jobs': jobs, 'jobs_links': links}
def list(self, request):
params = request.params.copy()
try:
params = utils.get_pagination_limit(params)
except exception.Invalid as e:
raise webob.exc.HTTPBadRequest(explanation=str(e))
if 'status' in params:
params['status'] = str(params['status']).upper()
if 'timeout' in params:
timeout = timeutils.parse_isotime(params['timeout'])
params['timeout'] = timeutils.normalize_time(timeout)
if 'hard_timeout' in params:
hard_timeout = timeutils.parse_isotime(params['hard_timeout'])
params['hard_timeout'] = timeutils.normalize_time(hard_timeout)
try:
jobs = self.db_api.job_get_all(params)
except exception.NotFound:
raise webob.exc.HTTPNotFound()
limit = params.get('limit')
if len(jobs) != 0 and len(jobs) == limit:
next_page = '/v1/jobs?marker=%s' % jobs[-1].get('id')
else:
next_page = None
for job in jobs:
utils.serialize_datetimes(job)
api_utils.serialize_job_metadata(job)
links = [{'rel': 'next', 'href': next_page}]
return {'jobs': jobs, 'jobs_links': links}
def update_status(self, request, job_id, body):
status = body.get('status')
if not status:
raise webob.exc.HTTPBadRequest()
values = {'status': status['status'].upper()}
if 'timeout' in status:
timeout = timeutils.parse_isotime(status['timeout'])
values['timeout'] = timeutils.normalize_time(timeout)
job = None
try:
job = self.db_api.job_update(job_id, values)
except exception.NotFound:
msg = _('Job %s could not be found.') % job_id
raise webob.exc.HTTPNotFound(explanation=msg)
if status['status'].upper() in ['ERROR', 'CANCELLED']:
values = self._get_error_values(status, job)
self.db_api.job_fault_create(values)
return {'status': {'status': job['status'], 'timeout': job['timeout']}}
def update_status(self, request, job_id, body):
status = body.get('status')
if not status:
raise webob.exc.HTTPBadRequest()
values = {'status': status['status'].upper()}
if 'timeout' in status:
timeout = timeutils.parse_isotime(status['timeout'])
values['timeout'] = timeutils.normalize_time(timeout)
job = None
try:
job = self.db_api.job_update(job_id, values)
except exception.NotFound:
msg = _('Job %s could not be found.') % job_id
raise webob.exc.HTTPNotFound(explanation=msg)
if status['status'].upper() in ['ERROR', 'CANCELLED']:
values = self._get_error_values(status, job)
self.db_api.job_fault_create(values)
return {'status': {'status': job['status'],
'timeout': job['timeout']}}
def test_job_workflow(self):
# (setup) create schedule
meta1 = {'key': 'key1', 'value': 'value1'}
meta2 = {'key': 'key2', 'value': 'value2'}
request = {
'schedule': {
'tenant': TENANT1,
'action': 'snapshot',
'minute': '30',
'hour': '12',
'metadata': {
meta1['key']: meta1['value'],
meta2['key']: meta2['value'],
}
}
}
schedule = self.client.create_schedule(request)
meta_fixture1 = {meta1['key']: meta1['value']}
meta_fixture2 = {meta2['key']: meta2['value']}
# create job
new_job = self.client.create_job(schedule['id'], schedule['next_run'])
self.assertNotEqual(new_job.get('id'), None)
self.assertEqual(new_job['schedule_id'], schedule['id'])
self.assertEqual(new_job['tenant'], schedule['tenant'])
self.assertEqual(new_job['action'], schedule['action'])
self.assertEqual(new_job['status'], 'QUEUED')
self.assertEqual(new_job['worker_id'], None)
self.assertNotEqual(new_job.get('timeout'), None)
self.assertNotEqual(new_job.get('hard_timeout'), None)
self.assertMetadataInList(new_job['metadata'], meta_fixture1)
self.assertMetadataInList(new_job['metadata'], meta_fixture2)
# ensure schedule was updated
updated_schedule = self.client.get_schedule(schedule['id'])
self.assertTrue(updated_schedule.get('last_scheduled'))
# change the schedule times
request = {
'schedule': {
'minute': '33',
'hour': '1',
}
}
updated_schedule = self.client.update_schedule(schedule['id'], request)
self.assertNotEqual(updated_schedule['next_run'], schedule['next_run'])
# attempt to recreate the job using old next_run
self.assertRaises(client_exc.Duplicate, self.client.create_job,
schedule['id'], schedule['next_run'])
# list jobs
jobs = self.client.list_jobs()
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0]['id'], new_job['id'])
self.assertEqual(jobs[0]['schedule_id'], new_job['schedule_id'])
self.assertEqual(jobs[0]['status'], new_job['status'])
self.assertEqual(jobs[0]['retry_count'], new_job['retry_count'])
# get job
job = self.client.get_job(new_job['id'])
self.assertEqual(job['id'], new_job['id'])
self.assertEqual(job['schedule_id'], new_job['schedule_id'])
self.assertEqual(job['status'], new_job['status'])
self.assertEqual(job['retry_count'], new_job['retry_count'])
# list job metadata
metadata = self.client.list_job_metadata(new_job['id'])
self.assertMetadataInList(metadata, meta_fixture1)
self.assertMetadataInList(metadata, meta_fixture2)
# update job metadata
new_meta = {'foo': 'bar'}
new_meta.update(meta_fixture1)
metadata = self.client.update_job_metadata(new_job['id'], new_meta)
self.assertMetadataInList(metadata, meta_fixture1)
self.assertMetadataInList(metadata, new_meta)
# list job metadata
metadata = self.client.list_job_metadata(new_job['id'])
self.assertMetadataInList(metadata, meta_fixture1)
self.assertMetadataInList(metadata, new_meta)
# update status without timeout
self.client.update_job_status(job['id'], 'processing')
status = self.client.get_job(job['id'])['status']
self.assertNotEqual(status, new_job['status'])
self.assertEqual(status, 'PROCESSING')
# update status with timeout
timeout = '2010-11-30T17:00:00Z'
self.client.update_job_status(job['id'], 'done', timeout)
updated_job = self.client.get_job(new_job['id'])
self.assertNotEqual(updated_job['status'], new_job['status'])
self.assertEqual(updated_job['status'], 'DONE')
self.assertNotEqual(updated_job['timeout'], new_job['timeout'])
self.assertEqual(updated_job['timeout'], timeout)
# update status with timeout as a datetime
timeout_str = '2010-11-30T18:00:00Z'
timeout = timeutils.parse_isotime(timeout_str)
self.client.update_job_status(job['id'], 'done', timeout)
updated_job = self.client.get_job(new_job['id'])
self.assertNotEqual(updated_job['status'], new_job['status'])
self.assertEqual(updated_job['status'], 'DONE')
self.assertNotEqual(updated_job['timeout'], new_job['timeout'])
self.assertEqual(updated_job['timeout'], timeout_str)
# update status with error
error_message = 'ermagerd! errer!'
self.client.update_job_status(job['id'],
'error',
error_message=error_message)
status = self.client.get_job(job['id'])['status']
self.assertNotEqual(status, new_job['status'])
self.assertEqual(status, 'ERROR')
job_fault = self.db_api.job_fault_latest_for_job_id(job['id'])
self.assertNotEqual(job_fault, None)
self.assertEqual(job_fault['job_id'], job['id'])
self.assertEqual(job_fault['tenant'], job['tenant'])
self.assertEqual(job_fault['schedule_id'], job['schedule_id'])
self.assertEqual(job_fault['worker_id'], job['worker_id']
or 'UNASSIGNED')
self.assertEqual(job_fault['action'], job['action'])
self.assertNotEqual(job_fault['job_metadata'], None)
self.assertEqual(job_fault['message'], error_message)
self.assertNotEqual(job_fault['created_at'], None)
self.assertNotEqual(job_fault['updated_at'], None)
self.assertNotEqual(job_fault['id'], None)
# cancel the job
error_message = 'ermagerd! cancelled!'
self.client.update_job_status(job['id'],
'CANCELLED',
error_message=error_message)
status = self.client.get_job(job['id'])['status']
self.assertNotEqual(status, new_job['status'])
self.assertEqual(status, 'CANCELLED')
job_fault = self.db_api.job_fault_latest_for_job_id(job['id'])
self.assertNotEqual(job_fault, None)
self.assertEqual(job_fault['job_id'], job['id'])
self.assertEqual(job_fault['tenant'], job['tenant'])
self.assertEqual(job_fault['schedule_id'], job['schedule_id'])
self.assertEqual(job_fault['worker_id'], job['worker_id']
or 'UNASSIGNED')
self.assertEqual(job_fault['action'], job['action'])
self.assertNotEqual(job_fault['job_metadata'], None)
self.assertEqual(job_fault['message'], error_message)
self.assertNotEqual(job_fault['created_at'], None)
self.assertNotEqual(job_fault['updated_at'], None)
self.assertNotEqual(job_fault['id'], None)
# delete job
self.client.delete_job(job['id'])
# make sure job no longer exists
self.assertRaises(client_exc.NotFound, self.client.get_job, job['id'])
def _create_jobs(self):
next_run = timeutils.parse_isotime('2012-11-27T02:30:00Z').replace(
tzinfo=None)
fixture = {
'id': unit_utils.SCHEDULE_UUID1,
'tenant': unit_utils.TENANT1,
'action': 'snapshot',
'minute': '30',
'hour': '2',
'next_run': next_run,
}
self.schedule_1 = db_api.schedule_create(fixture)
fixture = {
'id':
unit_utils.SCHEDULE_UUID2,
'tenant':
unit_utils.TENANT2,
'action':
'snapshot',
'minute':
'30',
'hour':
'2',
'next_run':
next_run,
'schedule_metadata': [{
'key': 'instance_id',
'value': 'my_instance',
}],
}
self.schedule_2 = db_api.schedule_create(fixture)
now = timeutils.utcnow()
timeout = now + datetime.timedelta(hours=1)
hard_timeout = now + datetime.timedelta(hours=8)
fixture = {
'id': unit_utils.JOB_UUID1,
'schedule_id': self.schedule_1['id'],
'tenant': unit_utils.TENANT1,
'worker_id': unit_utils.WORKER_UUID1,
'action': 'snapshot',
'status': 'QUEUED',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 0,
}
self.job_1 = db_api.job_create(fixture)
timeout = now + datetime.timedelta(hours=2)
hard_timeout = now + datetime.timedelta(hours=4)
fixture = {
'id': unit_utils.JOB_UUID2,
'schedule_id': self.schedule_2['id'],
'tenant': unit_utils.TENANT2,
'worker_id': unit_utils.WORKER_UUID2,
'action': 'snapshot',
'status': 'ERROR',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 1,
'job_metadata': [
{
'key': 'instance_id',
'value': 'my_instance',
},
]
}
self.job_2 = db_api.job_create(fixture)
fixture = {
'id': unit_utils.JOB_UUID3,
'schedule_id': self.schedule_1['id'],
'tenant': unit_utils.TENANT3,
'worker_id': unit_utils.WORKER_UUID1,
'action': 'snapshot',
'status': 'QUEUED',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 0,
}
self.job_3 = db_api.job_create(fixture)
fixture = {
'id': unit_utils.JOB_UUID4,
'schedule_id': self.schedule_1['id'],
'tenant': unit_utils.TENANT1,
'worker_id': unit_utils.WORKER_UUID1,
'action': 'test_action',
'status': 'QUEUED',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 0,
}
self.job_4 = db_api.job_create(fixture)
def fake_schedule_to_next_run(*args, **kwargs):
return timeutils.parse_isotime(expected_next_run)
def test_schedule_workflow(self):
schedules = self.client.list_schedules()
self.assertEqual(len(schedules), 0)
# create invalid schedule
request = {'not a schedule': 'yes'}
self.assertRaises(client_exc.BadRequest, self.client.create_schedule,
request)
# create malformed schedule
request = 'not a schedule'
self.assertRaises(client_exc.BadRequest, self.client.create_schedule,
request)
# create schedule with no body
self.assertRaises(client_exc.BadRequest, self.client.create_schedule,
None)
# create schedule
request = {
'schedule': {
'tenant': TENANT1,
'action': 'snapshot',
'minute': 30,
'hour': 12,
'metadata': {
'instance_id': 'my_instance_1'
},
}
}
schedule = self.client.create_schedule(request)
self.assertTrue(schedule['id'])
self.assertEqual(schedule['tenant'], TENANT1)
self.assertEqual(schedule['action'], 'snapshot')
self.assertEqual(schedule['minute'], 30)
self.assertEqual(schedule['hour'], 12)
self.assertTrue('metadata' in schedule)
metadata = schedule['metadata']
self.assertEqual(1, len(metadata))
self.assertEqual(metadata['instance_id'], 'my_instance_1')
# Get schedule
schedule = self.client.get_schedule(schedule['id'])
self.assertTrue(schedule['id'])
self.assertEqual(schedule['tenant'], TENANT1)
self.assertEqual(schedule['action'], 'snapshot')
self.assertEqual(schedule['minute'], 30)
self.assertEqual(schedule['hour'], 12)
# list schedules
schedules = self.client.list_schedules()
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
# list schedules, next_run filters
filters = {}
filters['next_run_after'] = schedule['next_run']
filters['next_run_before'] = schedule['next_run']
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
filters['next_run_after'] = schedule['next_run']
after_datetime = timeutils.parse_isotime(schedule['next_run'])
before_datetime = (after_datetime - datetime.timedelta(seconds=1))
filters['next_run_before'] = timeutils.isotime(before_datetime)
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 0)
# list schedules, next_run_before filters
filters = {}
filters['next_run_before'] = schedule['next_run']
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
# list schedules, next_run_after filters
filters = {}
filters['next_run_after'] = schedule['next_run']
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
# list schedules, tenant filters
filters = {}
filters['tenant'] = TENANT1
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
filters['tenant'] = 'aaaa-bbbb-cccc-dddd'
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 0)
# list schedules, metadata filter
filter = {}
filter['instance_id'] = 'my_instance_1'
schedules = self.client.list_schedules(filter_args=filter)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
filters['instance_id'] = 'aaaa-bbbb-cccc-dddd'
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 0)
filter = {}
filter['instance_name'] = 'aaaa-bbbb-cccc-dddd'
schedules = self.client.list_schedules(filter_args=filter)
self.assertEqual(len(schedules), 0)
# test filter by action
filter = {}
filter['action'] = 'snapshot'
schedules = self.client.list_schedules(filter_args=filter)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
filter['action'] = 'test_action'
schedules = self.client.list_schedules(filter_args=filter)
self.assertEqual(len(schedules), 0)
# update schedule
request = {'schedule': {'hour': 14}}
updated_schedule = self.client.update_schedule(schedule['id'], request)
self.assertEqual(updated_schedule['id'], schedule['id'])
self.assertEqual(updated_schedule['tenant'], schedule['tenant'])
self.assertEqual(updated_schedule['action'], schedule['action'])
self.assertEqual(updated_schedule['minute'], None)
self.assertEqual(updated_schedule['month'], None)
self.assertEqual(updated_schedule['day_of_week'], None)
self.assertEqual(updated_schedule['day_of_month'], None)
self.assertEqual(updated_schedule['hour'], request['schedule']['hour'])
self.assertNotEqual(updated_schedule['hour'], schedule['hour'])
self.assertNotEqual(updated_schedule['next_run'], schedule['next_run'])
# update schedule metadata
request = {
'schedule': {
'metadata': {
'instance_id': 'my_instance_2',
'retention': '3',
}
}
}
schedule = self.client.get_schedule(schedule['id'])
updated_schedule = self.client.update_schedule(schedule['id'], request)
self.assertEqual(updated_schedule['id'], schedule['id'])
self.assertEqual(updated_schedule['tenant'], schedule['tenant'])
self.assertEqual(updated_schedule['action'], schedule['action'])
self.assertEqual(updated_schedule['hour'], schedule['hour'])
self.assertEqual(updated_schedule['minute'], schedule['minute'])
self.assertEqual(updated_schedule['month'], schedule['month'])
self.assertEqual(updated_schedule['day_of_week'],
schedule['day_of_week'])
self.assertEqual(updated_schedule['day_of_month'],
schedule['day_of_month'])
self.assertTrue('metadata' in updated_schedule)
metadata = updated_schedule['metadata']
self.assertEqual(2, len(metadata))
self.assertEqual(metadata['instance_id'], 'my_instance_2')
self.assertEqual(metadata['retention'], '3')
# delete schedule
self.client.delete_schedule(schedule['id'])
# make sure schedule no longer exists
self.assertRaises(client_exc.NotFound, self.client.get_schedule,
schedule['id'])
def _process_job(self, job):
payload = {'job': job}
if job['status'] == 'QUEUED':
self.send_notification_start(payload)
else:
self.send_notification_retry(payload)
job_id = job['id']
hard_timeout = timeutils.normalize_time(
timeutils.parse_isotime(job['hard_timeout']))
hard_timed_out = hard_timeout <= self._get_utcnow()
if hard_timed_out:
msg = ('Job %(job_id)s has reached/exceeded its'
' hard timeout: %(hard_timeout)s.' % {
'job_id': job_id,
'hard_timeout': job['hard_timeout']
})
self._job_hard_timed_out(job, msg)
LOG.info(
_('[%(worker_tag)s] Job hard timed out: %(msg)s') % {
'worker_tag': self.get_worker_tag(),
'msg': msg
})
return
max_retried = job['retry_count'] > self.max_retry
if max_retried:
msg = ('Job %(job_id)s has reached/exceeded its'
' max_retry count: %(retry_count)s.' % {
'job_id': job_id,
'retry_count': job['retry_count']
})
self._job_max_retried(job, msg)
LOG.info(
_('[%(worker_tag)s] Job max_retry reached: %(msg)s') % {
'worker_tag': self.get_worker_tag(),
'msg': msg
})
return
schedule = self._get_schedule(job)
if schedule is None:
msg = ('Schedule %(schedule_id)s not found for job %(job_id)s' % {
'schedule_id': job['schedule_id'],
'job_id': job_id
})
self._job_cancelled(job, msg)
LOG.info(
_('[%(worker_tag)s] Job cancelled: %(msg)s') % {
'worker_tag': self.get_worker_tag(),
'msg': msg
})
return
now = self._get_utcnow()
self.next_timeout = now + self.initial_timeout
self._job_processing(job, self.next_timeout)
self.next_update = self._get_utcnow() + self.update_interval
instance_id = self._get_instance_id(job)
if not instance_id:
msg = ('Job %s does not specify an instance_id in its metadata.' %
job_id)
self._job_cancelled(job, msg)
return
image_id = self._get_image_id(job)
if image_id is None:
image_id = self._create_image(job, instance_id, schedule)
if image_id is None:
return
else:
LOG.info(
_("[%(worker_tag)s] Resuming image: %(image_id)s") % {
'worker_tag': self.get_worker_tag(),
'image_id': image_id
})
active = False
retry = True
while retry and not active and not self.stopping:
image_status = self._poll_image_status(job, image_id)
active = image_status == 'ACTIVE'
if not active:
retry = True
try:
self._update_job(job_id, "PROCESSING")
except exc.OutOfTimeException:
retry = False
else:
time.sleep(self.image_poll_interval)
if active:
self._process_retention(instance_id,
self.current_job['schedule_id'])
self._job_succeeded(self.current_job)
elif not active and not retry:
self._job_timed_out(self.current_job)
elif self.stopping:
# Timeout job so it gets picked up again quickly rather than
# queuing up behind a bunch of new jobs, but not so soon that
# another worker will pick it up before everything is shut down
# and thus burn through the retries
timeout = self._get_utcnow() + self.timeout_worker_stop
self._job_processing(self.current_job, timeout=timeout)
LOG.debug("[%s] Snapshot complete" % self.get_worker_tag())
def fake_schedule_to_next_run(*args, **kwargs):
return timeutils.parse_isotime(expected_next_run)
def _process_job(self, job):
payload = {'job': job}
if job['status'] == 'QUEUED':
self.send_notification_start(payload)
else:
self.send_notification_retry(payload)
job_id = job['id']
hard_timeout = timeutils.normalize_time(
timeutils.parse_isotime(job['hard_timeout']))
hard_timed_out = hard_timeout <= self._get_utcnow()
if hard_timed_out:
msg = ('Job %(job_id)s has reached/exceeded its'
' hard timeout: %(hard_timeout)s.' %
{'job_id': job_id, 'hard_timeout': job['hard_timeout']})
self._job_hard_timed_out(job, msg)
LOG.info(_('[%(worker_tag)s] Job hard timed out: %(msg)s') %
{'worker_tag': self.get_worker_tag(), 'msg': msg})
return
max_retried = job['retry_count'] > self.max_retry
if max_retried:
msg = ('Job %(job_id)s has reached/exceeded its'
' max_retry count: %(retry_count)s.' %
{'job_id': job_id, 'retry_count': job['retry_count']})
self._job_max_retried(job, msg)
LOG.info(_('[%(worker_tag)s] Job max_retry reached: %(msg)s') %
{'worker_tag': self.get_worker_tag(), 'msg': msg})
return
schedule = self._get_schedule(job)
if schedule is None:
msg = ('Schedule %(schedule_id)s not found for job %(job_id)s' %
{'schedule_id': job['schedule_id'], 'job_id': job_id})
self._job_cancelled(job, msg)
LOG.info(_('[%(worker_tag)s] Job cancelled: %(msg)s') %
{'worker_tag': self.get_worker_tag(),
'msg': msg})
return
now = self._get_utcnow()
self.next_timeout = now + self.initial_timeout
self._job_processing(job, self.next_timeout)
self.next_update = self._get_utcnow() + self.update_interval
instance_id = self._get_instance_id(job)
if not instance_id:
msg = ('Job %s does not specify an instance_id in its metadata.'
% job_id)
self._job_cancelled(job, msg)
return
image_id = self._get_image_id(job)
if image_id is None:
image_id = self._create_image(job, instance_id,
schedule)
if image_id is None:
return
else:
LOG.info(_("[%(worker_tag)s] Resuming image: %(image_id)s")
% {'worker_tag': self.get_worker_tag(),
'image_id': image_id})
active = False
retry = True
while retry and not active and not self.stopping:
image_status = self._poll_image_status(job, image_id)
active = image_status == 'ACTIVE'
if not active:
retry = True
try:
self._update_job(job_id, "PROCESSING")
except exc.OutOfTimeException:
retry = False
else:
time.sleep(self.image_poll_interval)
if active:
self._process_retention(instance_id,
self.current_job['schedule_id'])
self._job_succeeded(self.current_job)
elif not active and not retry:
self._job_timed_out(self.current_job)
elif self.stopping:
# Timeout job so it gets picked up again quickly rather than
# queuing up behind a bunch of new jobs, but not so soon that
# another worker will pick it up before everything is shut down
# and thus burn through the retries
timeout = self._get_utcnow() + self.timeout_worker_stop
self._job_processing(self.current_job, timeout=timeout)
LOG.debug("[%s] Snapshot complete" % self.get_worker_tag())
def test_schedule_workflow(self):
schedules = self.client.list_schedules()
self.assertEqual(len(schedules), 0)
# create invalid schedule
request = {'not a schedule': 'yes'}
self.assertRaises(client_exc.BadRequest, self.client.create_schedule,
request)
# create malformed schedule
request = 'not a schedule'
self.assertRaises(client_exc.BadRequest, self.client.create_schedule,
request)
# create schedule with no body
self.assertRaises(client_exc.BadRequest, self.client.create_schedule,
None)
# create schedule
request = {
'schedule':
{
'tenant': TENANT1,
'action': 'snapshot',
'minute': 30,
'hour': 12,
'metadata': {'instance_id': 'my_instance_1'},
}
}
schedule = self.client.create_schedule(request)
self.assertTrue(schedule['id'])
self.assertEqual(schedule['tenant'], TENANT1)
self.assertEqual(schedule['action'], 'snapshot')
self.assertEqual(schedule['minute'], 30)
self.assertEqual(schedule['hour'], 12)
self.assertTrue('metadata' in schedule)
metadata = schedule['metadata']
self.assertEqual(1, len(metadata))
self.assertEqual(metadata['instance_id'], 'my_instance_1')
# Get schedule
schedule = self.client.get_schedule(schedule['id'])
self.assertTrue(schedule['id'])
self.assertEqual(schedule['tenant'], TENANT1)
self.assertEqual(schedule['action'], 'snapshot')
self.assertEqual(schedule['minute'], 30)
self.assertEqual(schedule['hour'], 12)
# list schedules
schedules = self.client.list_schedules()
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
# list schedules, next_run filters
filters = {}
filters['next_run_after'] = schedule['next_run']
filters['next_run_before'] = schedule['next_run']
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
filters['next_run_after'] = schedule['next_run']
after_datetime = timeutils.parse_isotime(schedule['next_run'])
before_datetime = (after_datetime - datetime.timedelta(seconds=1))
filters['next_run_before'] = timeutils.isotime(before_datetime)
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 0)
# list schedules, next_run_before filters
filters = {}
filters['next_run_before'] = schedule['next_run']
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
# list schedules, next_run_after filters
filters = {}
filters['next_run_after'] = schedule['next_run']
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
# list schedules, tenant filters
filters = {}
filters['tenant'] = TENANT1
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
filters['tenant'] = 'aaaa-bbbb-cccc-dddd'
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 0)
# list schedules, metadata filter
filter = {}
filter['instance_id'] = 'my_instance_1'
schedules = self.client.list_schedules(filter_args=filter)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
filters['instance_id'] = 'aaaa-bbbb-cccc-dddd'
schedules = self.client.list_schedules(filter_args=filters)
self.assertEqual(len(schedules), 0)
filter = {}
filter['instance_name'] = 'aaaa-bbbb-cccc-dddd'
schedules = self.client.list_schedules(filter_args=filter)
self.assertEqual(len(schedules), 0)
# test filter by action
filter = {}
filter['action'] = 'snapshot'
schedules = self.client.list_schedules(filter_args=filter)
self.assertEqual(len(schedules), 1)
self.assertEqual(schedules[0], schedule)
filter['action'] = 'test_action'
schedules = self.client.list_schedules(filter_args=filter)
self.assertEqual(len(schedules), 0)
# update schedule
request = {'schedule': {'hour': 14}}
updated_schedule = self.client.update_schedule(schedule['id'], request)
self.assertEqual(updated_schedule['id'], schedule['id'])
self.assertEqual(updated_schedule['tenant'], schedule['tenant'])
self.assertEqual(updated_schedule['action'], schedule['action'])
self.assertEqual(updated_schedule['minute'], None)
self.assertEqual(updated_schedule['month'], None)
self.assertEqual(updated_schedule['day_of_week'], None)
self.assertEqual(updated_schedule['day_of_month'], None)
self.assertEqual(updated_schedule['hour'], request['schedule']['hour'])
self.assertNotEqual(updated_schedule['hour'], schedule['hour'])
self.assertNotEqual(updated_schedule['next_run'], schedule['next_run'])
# update schedule metadata
request = {'schedule': {'metadata': {'instance_id': 'my_instance_2',
'retention': '3',
}}}
schedule = self.client.get_schedule(schedule['id'])
updated_schedule = self.client.update_schedule(schedule['id'], request)
self.assertEqual(updated_schedule['id'], schedule['id'])
self.assertEqual(updated_schedule['tenant'], schedule['tenant'])
self.assertEqual(updated_schedule['action'], schedule['action'])
self.assertEqual(updated_schedule['hour'], schedule['hour'])
self.assertEqual(updated_schedule['minute'], schedule['minute'])
self.assertEqual(updated_schedule['month'], schedule['month'])
self.assertEqual(updated_schedule['day_of_week'],
schedule['day_of_week'])
self.assertEqual(updated_schedule['day_of_month'],
schedule['day_of_month'])
self.assertTrue('metadata' in updated_schedule)
metadata = updated_schedule['metadata']
self.assertEqual(2, len(metadata))
self.assertEqual(metadata['instance_id'], 'my_instance_2')
self.assertEqual(metadata['retention'], '3')
# delete schedule
self.client.delete_schedule(schedule['id'])
# make sure schedule no longer exists
self.assertRaises(client_exc.NotFound, self.client.get_schedule,
schedule['id'])
def create(self, request, body):
if (body is None or body.get('job') is None
or body['job'].get('schedule_id') is None):
raise webob.exc.HTTPBadRequest()
job = body['job']
try:
schedule = self.db_api.schedule_get_by_id(job['schedule_id'])
except exception.NotFound:
raise webob.exc.HTTPNotFound()
# Check integrity of schedule and update next run
expected_next_run = job.get('next_run')
if expected_next_run:
try:
expected_next_run = timeutils.parse_isotime(expected_next_run)
expected_next_run = expected_next_run.replace(tzinfo=None)
except ValueError as e:
msg = _('Invalid "next_run" value. Must be ISO 8601 format')
raise webob.exc.HTTPBadRequest(explanation=msg)
next_run = api_utils.schedule_to_next_run(schedule, timeutils.utcnow())
next_run = next_run.replace(tzinfo=None)
try:
self.db_api.schedule_test_and_set_next_run(schedule['id'],
expected_next_run,
next_run)
except exception.NotFound:
msg = _("Specified next run does not match the current next run"
" value. This could mean schedule has either changed"
"or has already been scheduled since you last expected.")
raise webob.exc.HTTPConflict(explanation=msg)
# Update schedule last_scheduled
values = {}
values['last_scheduled'] = timeutils.utcnow()
self.db_api.schedule_update(schedule['id'], values)
# Create job
values = {}
values.update(job)
values['tenant'] = schedule['tenant']
values['action'] = schedule['action']
values['status'] = 'QUEUED'
job_metadata = []
for metadata in schedule['schedule_metadata']:
job_metadata.append({
'key': metadata['key'],
'value': metadata['value']
})
values['job_metadata'] = job_metadata
job_action = values['action']
if not 'timeout' in values:
values['timeout'] = api_utils.get_new_timeout_by_action(job_action)
values['hard_timeout'] = \
api_utils.get_new_timeout_by_action(job_action)
job = self.db_api.job_create(values)
utils.serialize_datetimes(job)
api_utils.serialize_job_metadata(job)
job = {'job': job}
utils.generate_notification(None, 'qonos.job.create', job, 'INFO')
return job
def test_job_workflow(self):
# (setup) create schedule
meta1 = {'key': 'key1', 'value': 'value1'}
meta2 = {'key': 'key2', 'value': 'value2'}
request = {
'schedule':
{
'tenant': TENANT1,
'action': 'snapshot',
'minute': '30',
'hour': '12',
'metadata': {
meta1['key']: meta1['value'],
meta2['key']: meta2['value'],
}
}
}
schedule = self.client.create_schedule(request)
meta_fixture1 = {meta1['key']: meta1['value']}
meta_fixture2 = {meta2['key']: meta2['value']}
# create job
new_job = self.client.create_job(schedule['id'], schedule['next_run'])
self.assertNotEqual(new_job.get('id'), None)
self.assertEqual(new_job['schedule_id'], schedule['id'])
self.assertEqual(new_job['tenant'], schedule['tenant'])
self.assertEqual(new_job['action'], schedule['action'])
self.assertEqual(new_job['status'], 'QUEUED')
self.assertEqual(new_job['worker_id'], None)
self.assertNotEqual(new_job.get('timeout'), None)
self.assertNotEqual(new_job.get('hard_timeout'), None)
self.assertMetadataInList(new_job['metadata'], meta_fixture1)
self.assertMetadataInList(new_job['metadata'], meta_fixture2)
# ensure schedule was updated
updated_schedule = self.client.get_schedule(schedule['id'])
self.assertTrue(updated_schedule.get('last_scheduled'))
# change the schedule times
request = {
'schedule':
{
'minute': '33',
'hour': '1',
}
}
updated_schedule = self.client.update_schedule(schedule['id'], request)
self.assertNotEqual(updated_schedule['next_run'], schedule['next_run'])
# attempt to recreate the job using old next_run
self.assertRaises(client_exc.Duplicate, self.client.create_job,
schedule['id'], schedule['next_run'])
# list jobs
jobs = self.client.list_jobs()
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0]['id'], new_job['id'])
self.assertEqual(jobs[0]['schedule_id'], new_job['schedule_id'])
self.assertEqual(jobs[0]['status'], new_job['status'])
self.assertEqual(jobs[0]['retry_count'], new_job['retry_count'])
# get job
job = self.client.get_job(new_job['id'])
self.assertEqual(job['id'], new_job['id'])
self.assertEqual(job['schedule_id'], new_job['schedule_id'])
self.assertEqual(job['status'], new_job['status'])
self.assertEqual(job['retry_count'], new_job['retry_count'])
# list job metadata
metadata = self.client.list_job_metadata(new_job['id'])
self.assertMetadataInList(metadata, meta_fixture1)
self.assertMetadataInList(metadata, meta_fixture2)
# update job metadata
new_meta = {'foo': 'bar'}
new_meta.update(meta_fixture1)
metadata = self.client.update_job_metadata(new_job['id'], new_meta)
self.assertMetadataInList(metadata, meta_fixture1)
self.assertMetadataInList(metadata, new_meta)
# list job metadata
metadata = self.client.list_job_metadata(new_job['id'])
self.assertMetadataInList(metadata, meta_fixture1)
self.assertMetadataInList(metadata, new_meta)
# update status without timeout
self.client.update_job_status(job['id'], 'processing')
status = self.client.get_job(job['id'])['status']
self.assertNotEqual(status, new_job['status'])
self.assertEqual(status, 'PROCESSING')
# update status with timeout
timeout = '2010-11-30T17:00:00Z'
self.client.update_job_status(job['id'], 'done', timeout)
updated_job = self.client.get_job(new_job['id'])
self.assertNotEqual(updated_job['status'], new_job['status'])
self.assertEqual(updated_job['status'], 'DONE')
self.assertNotEqual(updated_job['timeout'], new_job['timeout'])
self.assertEqual(updated_job['timeout'], timeout)
# update status with timeout as a datetime
timeout_str = '2010-11-30T18:00:00Z'
timeout = timeutils.parse_isotime(timeout_str)
self.client.update_job_status(job['id'], 'done', timeout)
updated_job = self.client.get_job(new_job['id'])
self.assertNotEqual(updated_job['status'], new_job['status'])
self.assertEqual(updated_job['status'], 'DONE')
self.assertNotEqual(updated_job['timeout'], new_job['timeout'])
self.assertEqual(updated_job['timeout'], timeout_str)
# update status with error
error_message = 'ermagerd! errer!'
self.client.update_job_status(job['id'], 'error',
error_message=error_message)
status = self.client.get_job(job['id'])['status']
self.assertNotEqual(status, new_job['status'])
self.assertEqual(status, 'ERROR')
job_fault = self.db_api.job_fault_latest_for_job_id(job['id'])
self.assertNotEqual(job_fault, None)
self.assertEqual(job_fault['job_id'], job['id'])
self.assertEqual(job_fault['tenant'], job['tenant'])
self.assertEqual(job_fault['schedule_id'], job['schedule_id'])
self.assertEqual(job_fault['worker_id'],
job['worker_id'] or 'UNASSIGNED')
self.assertEqual(job_fault['action'], job['action'])
self.assertNotEqual(job_fault['job_metadata'], None)
self.assertEqual(job_fault['message'], error_message)
self.assertNotEqual(job_fault['created_at'], None)
self.assertNotEqual(job_fault['updated_at'], None)
self.assertNotEqual(job_fault['id'], None)
# cancel the job
error_message = 'ermagerd! cancelled!'
self.client.update_job_status(job['id'], 'CANCELLED',
error_message=error_message)
status = self.client.get_job(job['id'])['status']
self.assertNotEqual(status, new_job['status'])
self.assertEqual(status, 'CANCELLED')
job_fault = self.db_api.job_fault_latest_for_job_id(job['id'])
self.assertNotEqual(job_fault, None)
self.assertEqual(job_fault['job_id'], job['id'])
self.assertEqual(job_fault['tenant'], job['tenant'])
self.assertEqual(job_fault['schedule_id'], job['schedule_id'])
self.assertEqual(job_fault['worker_id'],
job['worker_id'] or 'UNASSIGNED')
self.assertEqual(job_fault['action'], job['action'])
self.assertNotEqual(job_fault['job_metadata'], None)
self.assertEqual(job_fault['message'], error_message)
self.assertNotEqual(job_fault['created_at'], None)
self.assertNotEqual(job_fault['updated_at'], None)
self.assertNotEqual(job_fault['id'], None)
# delete job
self.client.delete_job(job['id'])
# make sure job no longer exists
self.assertRaises(client_exc.NotFound, self.client.get_job, job['id'])
def _create_jobs(self):
fixture = {
'id': unit_utils.SCHEDULE_UUID1,
'tenant': unit_utils.TENANT1,
'action': 'snapshot',
'minute': '30',
'hour': '2',
'next_run': timeutils.parse_isotime('2012-11-27T02:30:00Z')
}
self.schedule_1 = db_api.schedule_create(fixture)
fixture = {
'id': unit_utils.SCHEDULE_UUID2,
'tenant': unit_utils.TENANT2,
'action': 'snapshot',
'minute': '30',
'hour': '2',
'next_run': timeutils.parse_isotime('2012-11-27T02:30:00Z'),
'schedule_metadata': [
{
'key': 'instance_id',
'value': 'my_instance',
}
],
}
self.schedule_2 = db_api.schedule_create(fixture)
now = timeutils.utcnow()
timeout = now + datetime.timedelta(hours=1)
hard_timeout = now + datetime.timedelta(hours=4)
fixture = {
'id': unit_utils.JOB_UUID1,
'schedule_id': self.schedule_1['id'],
'tenant': unit_utils.TENANT1,
'worker_id': unit_utils.WORKER_UUID1,
'action': 'snapshot',
'status': 'QUEUED',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 0,
}
self.job_1 = db_api.job_create(fixture)
fixture = {
'id': unit_utils.JOB_UUID2,
'schedule_id': self.schedule_2['id'],
'tenant': unit_utils.TENANT2,
'worker_id': unit_utils.WORKER_UUID2,
'action': 'snapshot',
'status': 'ERROR',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 1,
'job_metadata': [
{
'key': 'instance_id',
'value': 'my_instance',
},
]
}
self.job_2 = db_api.job_create(fixture)
fixture = {
'id': unit_utils.JOB_UUID3,
'schedule_id': self.schedule_1['id'],
'tenant': unit_utils.TENANT1,
'worker_id': unit_utils.WORKER_UUID1,
'action': 'snapshot',
'status': 'QUEUED',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 0,
}
self.job_3 = db_api.job_create(fixture)
fixture = {
'id': unit_utils.JOB_UUID4,
'schedule_id': self.schedule_1['id'],
'tenant': unit_utils.TENANT1,
'worker_id': unit_utils.WORKER_UUID1,
'action': 'snapshot',
'status': 'QUEUED',
'timeout': timeout,
'hard_timeout': hard_timeout,
'retry_count': 0,
}
self.job_4 = db_api.job_create(fixture)