def _get_request_params(self, request): filter_args = {} params = request.params if params.get('next_run_after') is not None: next_run_after = params['next_run_after'] next_run_after = timeutils.parse_isotime(next_run_after) next_run_after = timeutils.normalize_time(next_run_after) filter_args['next_run_after'] = next_run_after if params.get('next_run_before') is not None: next_run_before = params['next_run_before'] next_run_before = timeutils.parse_isotime(next_run_before) next_run_before = timeutils.normalize_time(next_run_before) filter_args['next_run_before'] = next_run_before if request.params.get('tenant') is not None: filter_args['tenant'] = request.params['tenant'] filter_args['limit'] = params.get('limit') filter_args['marker'] = params.get('marker') for filter_key in params.keys(): if filter_key not in filter_args: filter_args[filter_key] = params[filter_key] return filter_args
def update(self, request, schedule_id, body): if not body: msg = _('The request body must not be empty') raise webob.exc.HTTPBadRequest(explanation=msg) elif 'schedule' not in body: msg = _('The request body must contain a "schedule" entity') raise webob.exc.HTTPBadRequest(explanation=msg) # NOTE(jculp): only raise if a blank tenant is passed # passing no tenant at all is perfectly fine. elif('tenant' in body['schedule'] and not body['schedule']['tenant'].strip()): msg = _('The request body has not specified a "tenant" entity') raise webob.exc.HTTPBadRequest(explanation=msg) api_utils.deserialize_schedule_metadata(body['schedule']) values = {} values.update(body['schedule']) try: values = api_utils.check_read_only_properties(values) except exception.Forbidden as e: raise webob.exc.HTTPForbidden(explanation=unicode(e)) request_next_run = body['schedule'].get('next_run') times = { 'minute': None, 'hour': None, 'month': None, 'day_of_week': None, 'day_of_month': None, } update_schedule_times = False for key in times: if key in values: times[key] = values[key] update_schedule_times = True if update_schedule_times: # NOTE(ameade): We must recalculate the schedules next_run time # since the schedule has changed values.update(times) values['next_run'] = api_utils.schedule_to_next_run(times) elif request_next_run: try: timeutils.parse_isotime(request_next_run) except ValueError as e: msg = _('Invalid "next_run" value. Must be ISO 8601 format') raise webob.exc.HTTPBadRequest(explanation=msg) try: schedule = self.db_api.schedule_update(schedule_id, values) except exception.NotFound: msg = _('Schedule %s could not be found.') % schedule_id raise webob.exc.HTTPNotFound(explanation=msg) utils.serialize_datetimes(schedule) api_utils.serialize_schedule_metadata(schedule) return {'schedule': schedule}
def update(self, request, schedule_id, body): if not body: msg = _('The request body must not be empty') raise webob.exc.HTTPBadRequest(explanation=msg) elif not 'schedule' in body: msg = _('The request body must contain a "schedule" entity') raise webob.exc.HTTPBadRequest(explanation=msg) # NOTE(jculp): only raise if a blank tenant is passed # passing no tenant at all is perfectly fine. elif ('tenant' in body['schedule'] and not body['schedule']['tenant'].strip()): msg = _('The request body has not specified a "tenant" entity') raise webob.exc.HTTPBadRequest(explanation=msg) api_utils.deserialize_schedule_metadata(body['schedule']) values = {} values.update(body['schedule']) try: values = api_utils.check_read_only_properties(values) except exception.Forbidden as e: raise webob.exc.HTTPForbidden(explanation=unicode(e)) request_next_run = body['schedule'].get('next_run') times = { 'minute': None, 'hour': None, 'month': None, 'day_of_week': None, 'day_of_month': None, } update_schedule_times = False for key in times: if key in values: times[key] = values[key] update_schedule_times = True if update_schedule_times: # NOTE(ameade): We must recalculate the schedules next_run time # since the schedule has changed values.update(times) values['next_run'] = api_utils.schedule_to_next_run(times) elif request_next_run: try: timeutils.parse_isotime(request_next_run) except ValueError as e: msg = _('Invalid "next_run" value. Must be ISO 8601 format') raise webob.exc.HTTPBadRequest(explanation=msg) try: schedule = self.db_api.schedule_update(schedule_id, values) except exception.NotFound: msg = _('Schedule %s could not be found.') % schedule_id raise webob.exc.HTTPNotFound(explanation=msg) utils.serialize_datetimes(schedule) api_utils.serialize_schedule_metadata(schedule) return {'schedule': schedule}
def test_create(self): expected_next_run = timeutils.parse_isotime( '1989-01-19T12:00:00Z').replace(tzinfo=None) self._stub_notifications(None, 'qonos.job.create', 'fake-payload', 'INFO') def fake_schedule_to_next_run(_schedule, start_time=None): self.assertEqual(timeutils.utcnow(), start_time) return expected_next_run self.stubs.Set(api_utils, 'schedule_to_next_run', fake_schedule_to_next_run) request = unit_utils.get_fake_request(method='POST') fixture = {'job': {'schedule_id': self.schedule_1['id']}} job = self.controller.create(request, fixture).get('job') self.assertNotEqual(job, None) self.assertNotEqual(job.get('id'), None) self.assertEqual(job['schedule_id'], self.schedule_1['id']) self.assertEqual(job['tenant'], self.schedule_1['tenant']) self.assertEqual(job['action'], self.schedule_1['action']) self.assertEqual(job['status'], 'QUEUED') self.assertEqual(len(job['metadata']), 0) schedule = db_api.schedule_get_by_id(self.schedule_1['id']) self.assertNotEqual(schedule['next_run'], self.schedule_1['next_run']) self.assertEqual(schedule['next_run'], expected_next_run) self.assertNotEqual(schedule['last_scheduled'], self.schedule_1.get('last_scheduled')) self.assertTrue(schedule.get('last_scheduled'))
def create(self, request, body): if (body is None or body.get('job') is None or body['job'].get('schedule_id') is None): raise webob.exc.HTTPBadRequest() job = body['job'] try: schedule = self.db_api.schedule_get_by_id(job['schedule_id']) except exception.NotFound: raise webob.exc.HTTPNotFound() # Check integrity of schedule and update next run expected_next_run = job.get('next_run') if expected_next_run: expected_next_run = timeutils.parse_isotime(job.get('next_run')) next_run = api_utils.schedule_to_next_run(schedule, timeutils.utcnow()) try: self.db_api.schedule_test_and_set_next_run(schedule['id'], expected_next_run, next_run) except exception.NotFound: msg = _("Specified next run does not match the current next run" " value. This could mean schedule has either changed" "or has already been scheduled since you last expected.") raise webob.exc.HTTPConflict(explanation=msg) # Update schedule last_scheduled values = {} values['last_scheduled'] = timeutils.utcnow() self.db_api.schedule_update(schedule['id'], values) # Create job values = {} values.update(job) values['tenant'] = schedule['tenant'] values['action'] = schedule['action'] values['status'] = 'QUEUED' job_metadata = [] for metadata in schedule['schedule_metadata']: job_metadata.append({ 'key': metadata['key'], 'value': metadata['value'] }) values['job_metadata'] = job_metadata job_action = values['action'] if not 'timeout' in values: values['timeout'] = api_utils.get_new_timeout_by_action(job_action) values['hard_timeout'] = \ api_utils.get_new_timeout_by_action(job_action) job = self.db_api.job_create(values) utils.serialize_datetimes(job) api_utils.serialize_job_metadata(job) job = {'job': job} utils.generate_notification(None, 'qonos.job.create', job, 'INFO') return job
def test_create_with_next_run(self): expected_next_run = timeutils.parse_isotime('1989-01-19T12:00:00Z') def fake_schedule_to_next_run(_schedule, start_time=None): self.assertEqual(timeutils.utcnow(), start_time) return expected_next_run self.stubs.Set(api_utils, 'schedule_to_next_run', fake_schedule_to_next_run) self._stub_notifications(None, 'qonos.job.create', 'fake-payload', 'INFO') request = unit_utils.get_fake_request(method='POST') fixture = {'job': {'schedule_id': self.schedule_1['id'], 'next_run': timeutils.isotime(self.schedule_1['next_run'])}} job = self.controller.create(request, fixture).get('job') self.assertNotEqual(job, None) self.assertNotEqual(job.get('id'), None) self.assertEqual(job['schedule_id'], self.schedule_1['id']) self.assertEqual(job['tenant'], self.schedule_1['tenant']) self.assertEqual(job['action'], self.schedule_1['action']) self.assertEqual(job['status'], 'QUEUED') self.assertEqual(len(job['metadata']), 0) schedule = db_api.schedule_get_by_id(self.schedule_1['id']) self.assertNotEqual(schedule['next_run'], self.schedule_1['next_run']) self.assertEqual(schedule['next_run'], expected_next_run) self.assertNotEqual(schedule['last_scheduled'], self.schedule_1.get('last_scheduled')) self.assertTrue(schedule.get('last_scheduled'))
def list(self, request): params = request.params.copy() try: params = utils.get_pagination_limit(params) except exception.Invalid as e: raise webob.exc.HTTPBadRequest(explanation=str(e)) if 'status' in params: params['status'] = str(params['status']).upper() if 'timeout' in params: timeout = timeutils.parse_isotime(params['timeout']) params['timeout'] = timeutils.normalize_time(timeout) if 'hard_timeout' in params: hard_timeout = timeutils.parse_isotime(params['hard_timeout']) params['hard_timeout'] = timeutils.normalize_time(hard_timeout) try: jobs = self.db_api.job_get_all(params) except exception.NotFound: raise webob.exc.HTTPNotFound() limit = params.get('limit') if len(jobs) != 0 and len(jobs) == limit: next_page = '/v1/jobs?marker=%s' % jobs[-1].get('id') else: next_page = None for job in jobs: utils.serialize_datetimes(job) api_utils.serialize_job_metadata(job) links = [{'rel': 'next', 'href': next_page}] return {'jobs': jobs, 'jobs_links': links}
def update_status(self, request, job_id, body): status = body.get('status') if not status: raise webob.exc.HTTPBadRequest() values = {'status': status['status'].upper()} if 'timeout' in status: timeout = timeutils.parse_isotime(status['timeout']) values['timeout'] = timeutils.normalize_time(timeout) job = None try: job = self.db_api.job_update(job_id, values) except exception.NotFound: msg = _('Job %s could not be found.') % job_id raise webob.exc.HTTPNotFound(explanation=msg) if status['status'].upper() in ['ERROR', 'CANCELLED']: values = self._get_error_values(status, job) self.db_api.job_fault_create(values) return {'status': {'status': job['status'], 'timeout': job['timeout']}}
def test_job_workflow(self): # (setup) create schedule meta1 = {'key': 'key1', 'value': 'value1'} meta2 = {'key': 'key2', 'value': 'value2'} request = { 'schedule': { 'tenant': TENANT1, 'action': 'snapshot', 'minute': '30', 'hour': '12', 'metadata': { meta1['key']: meta1['value'], meta2['key']: meta2['value'], } } } schedule = self.client.create_schedule(request) meta_fixture1 = {meta1['key']: meta1['value']} meta_fixture2 = {meta2['key']: meta2['value']} # create job new_job = self.client.create_job(schedule['id'], schedule['next_run']) self.assertNotEqual(new_job.get('id'), None) self.assertEqual(new_job['schedule_id'], schedule['id']) self.assertEqual(new_job['tenant'], schedule['tenant']) self.assertEqual(new_job['action'], schedule['action']) self.assertEqual(new_job['status'], 'QUEUED') self.assertEqual(new_job['worker_id'], None) self.assertNotEqual(new_job.get('timeout'), None) self.assertNotEqual(new_job.get('hard_timeout'), None) self.assertMetadataInList(new_job['metadata'], meta_fixture1) self.assertMetadataInList(new_job['metadata'], meta_fixture2) # ensure schedule was updated updated_schedule = self.client.get_schedule(schedule['id']) self.assertTrue(updated_schedule.get('last_scheduled')) # change the schedule times request = { 'schedule': { 'minute': '33', 'hour': '1', } } updated_schedule = self.client.update_schedule(schedule['id'], request) self.assertNotEqual(updated_schedule['next_run'], schedule['next_run']) # attempt to recreate the job using old next_run self.assertRaises(client_exc.Duplicate, self.client.create_job, schedule['id'], schedule['next_run']) # list jobs jobs = self.client.list_jobs() self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0]['id'], new_job['id']) self.assertEqual(jobs[0]['schedule_id'], new_job['schedule_id']) self.assertEqual(jobs[0]['status'], new_job['status']) self.assertEqual(jobs[0]['retry_count'], new_job['retry_count']) # get job job = self.client.get_job(new_job['id']) self.assertEqual(job['id'], new_job['id']) self.assertEqual(job['schedule_id'], new_job['schedule_id']) self.assertEqual(job['status'], new_job['status']) self.assertEqual(job['retry_count'], new_job['retry_count']) # list job metadata metadata = self.client.list_job_metadata(new_job['id']) self.assertMetadataInList(metadata, meta_fixture1) self.assertMetadataInList(metadata, meta_fixture2) # update job metadata new_meta = {'foo': 'bar'} new_meta.update(meta_fixture1) metadata = self.client.update_job_metadata(new_job['id'], new_meta) self.assertMetadataInList(metadata, meta_fixture1) self.assertMetadataInList(metadata, new_meta) # list job metadata metadata = self.client.list_job_metadata(new_job['id']) self.assertMetadataInList(metadata, meta_fixture1) self.assertMetadataInList(metadata, new_meta) # update status without timeout self.client.update_job_status(job['id'], 'processing') status = self.client.get_job(job['id'])['status'] self.assertNotEqual(status, new_job['status']) self.assertEqual(status, 'PROCESSING') # update status with timeout timeout = '2010-11-30T17:00:00Z' self.client.update_job_status(job['id'], 'done', timeout) updated_job = self.client.get_job(new_job['id']) self.assertNotEqual(updated_job['status'], new_job['status']) self.assertEqual(updated_job['status'], 'DONE') self.assertNotEqual(updated_job['timeout'], new_job['timeout']) self.assertEqual(updated_job['timeout'], timeout) # update status with timeout as a datetime timeout_str = '2010-11-30T18:00:00Z' timeout = timeutils.parse_isotime(timeout_str) self.client.update_job_status(job['id'], 'done', timeout) updated_job = self.client.get_job(new_job['id']) self.assertNotEqual(updated_job['status'], new_job['status']) self.assertEqual(updated_job['status'], 'DONE') self.assertNotEqual(updated_job['timeout'], new_job['timeout']) self.assertEqual(updated_job['timeout'], timeout_str) # update status with error error_message = 'ermagerd! errer!' self.client.update_job_status(job['id'], 'error', error_message=error_message) status = self.client.get_job(job['id'])['status'] self.assertNotEqual(status, new_job['status']) self.assertEqual(status, 'ERROR') job_fault = self.db_api.job_fault_latest_for_job_id(job['id']) self.assertNotEqual(job_fault, None) self.assertEqual(job_fault['job_id'], job['id']) self.assertEqual(job_fault['tenant'], job['tenant']) self.assertEqual(job_fault['schedule_id'], job['schedule_id']) self.assertEqual(job_fault['worker_id'], job['worker_id'] or 'UNASSIGNED') self.assertEqual(job_fault['action'], job['action']) self.assertNotEqual(job_fault['job_metadata'], None) self.assertEqual(job_fault['message'], error_message) self.assertNotEqual(job_fault['created_at'], None) self.assertNotEqual(job_fault['updated_at'], None) self.assertNotEqual(job_fault['id'], None) # cancel the job error_message = 'ermagerd! cancelled!' self.client.update_job_status(job['id'], 'CANCELLED', error_message=error_message) status = self.client.get_job(job['id'])['status'] self.assertNotEqual(status, new_job['status']) self.assertEqual(status, 'CANCELLED') job_fault = self.db_api.job_fault_latest_for_job_id(job['id']) self.assertNotEqual(job_fault, None) self.assertEqual(job_fault['job_id'], job['id']) self.assertEqual(job_fault['tenant'], job['tenant']) self.assertEqual(job_fault['schedule_id'], job['schedule_id']) self.assertEqual(job_fault['worker_id'], job['worker_id'] or 'UNASSIGNED') self.assertEqual(job_fault['action'], job['action']) self.assertNotEqual(job_fault['job_metadata'], None) self.assertEqual(job_fault['message'], error_message) self.assertNotEqual(job_fault['created_at'], None) self.assertNotEqual(job_fault['updated_at'], None) self.assertNotEqual(job_fault['id'], None) # delete job self.client.delete_job(job['id']) # make sure job no longer exists self.assertRaises(client_exc.NotFound, self.client.get_job, job['id'])
def _create_jobs(self): next_run = timeutils.parse_isotime('2012-11-27T02:30:00Z').replace( tzinfo=None) fixture = { 'id': unit_utils.SCHEDULE_UUID1, 'tenant': unit_utils.TENANT1, 'action': 'snapshot', 'minute': '30', 'hour': '2', 'next_run': next_run, } self.schedule_1 = db_api.schedule_create(fixture) fixture = { 'id': unit_utils.SCHEDULE_UUID2, 'tenant': unit_utils.TENANT2, 'action': 'snapshot', 'minute': '30', 'hour': '2', 'next_run': next_run, 'schedule_metadata': [{ 'key': 'instance_id', 'value': 'my_instance', }], } self.schedule_2 = db_api.schedule_create(fixture) now = timeutils.utcnow() timeout = now + datetime.timedelta(hours=1) hard_timeout = now + datetime.timedelta(hours=8) fixture = { 'id': unit_utils.JOB_UUID1, 'schedule_id': self.schedule_1['id'], 'tenant': unit_utils.TENANT1, 'worker_id': unit_utils.WORKER_UUID1, 'action': 'snapshot', 'status': 'QUEUED', 'timeout': timeout, 'hard_timeout': hard_timeout, 'retry_count': 0, } self.job_1 = db_api.job_create(fixture) timeout = now + datetime.timedelta(hours=2) hard_timeout = now + datetime.timedelta(hours=4) fixture = { 'id': unit_utils.JOB_UUID2, 'schedule_id': self.schedule_2['id'], 'tenant': unit_utils.TENANT2, 'worker_id': unit_utils.WORKER_UUID2, 'action': 'snapshot', 'status': 'ERROR', 'timeout': timeout, 'hard_timeout': hard_timeout, 'retry_count': 1, 'job_metadata': [ { 'key': 'instance_id', 'value': 'my_instance', }, ] } self.job_2 = db_api.job_create(fixture) fixture = { 'id': unit_utils.JOB_UUID3, 'schedule_id': self.schedule_1['id'], 'tenant': unit_utils.TENANT3, 'worker_id': unit_utils.WORKER_UUID1, 'action': 'snapshot', 'status': 'QUEUED', 'timeout': timeout, 'hard_timeout': hard_timeout, 'retry_count': 0, } self.job_3 = db_api.job_create(fixture) fixture = { 'id': unit_utils.JOB_UUID4, 'schedule_id': self.schedule_1['id'], 'tenant': unit_utils.TENANT1, 'worker_id': unit_utils.WORKER_UUID1, 'action': 'test_action', 'status': 'QUEUED', 'timeout': timeout, 'hard_timeout': hard_timeout, 'retry_count': 0, } self.job_4 = db_api.job_create(fixture)
def fake_schedule_to_next_run(*args, **kwargs): return timeutils.parse_isotime(expected_next_run)
def test_schedule_workflow(self): schedules = self.client.list_schedules() self.assertEqual(len(schedules), 0) # create invalid schedule request = {'not a schedule': 'yes'} self.assertRaises(client_exc.BadRequest, self.client.create_schedule, request) # create malformed schedule request = 'not a schedule' self.assertRaises(client_exc.BadRequest, self.client.create_schedule, request) # create schedule with no body self.assertRaises(client_exc.BadRequest, self.client.create_schedule, None) # create schedule request = { 'schedule': { 'tenant': TENANT1, 'action': 'snapshot', 'minute': 30, 'hour': 12, 'metadata': { 'instance_id': 'my_instance_1' }, } } schedule = self.client.create_schedule(request) self.assertTrue(schedule['id']) self.assertEqual(schedule['tenant'], TENANT1) self.assertEqual(schedule['action'], 'snapshot') self.assertEqual(schedule['minute'], 30) self.assertEqual(schedule['hour'], 12) self.assertTrue('metadata' in schedule) metadata = schedule['metadata'] self.assertEqual(1, len(metadata)) self.assertEqual(metadata['instance_id'], 'my_instance_1') # Get schedule schedule = self.client.get_schedule(schedule['id']) self.assertTrue(schedule['id']) self.assertEqual(schedule['tenant'], TENANT1) self.assertEqual(schedule['action'], 'snapshot') self.assertEqual(schedule['minute'], 30) self.assertEqual(schedule['hour'], 12) # list schedules schedules = self.client.list_schedules() self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) # list schedules, next_run filters filters = {} filters['next_run_after'] = schedule['next_run'] filters['next_run_before'] = schedule['next_run'] schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) filters['next_run_after'] = schedule['next_run'] after_datetime = timeutils.parse_isotime(schedule['next_run']) before_datetime = (after_datetime - datetime.timedelta(seconds=1)) filters['next_run_before'] = timeutils.isotime(before_datetime) schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 0) # list schedules, next_run_before filters filters = {} filters['next_run_before'] = schedule['next_run'] schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) # list schedules, next_run_after filters filters = {} filters['next_run_after'] = schedule['next_run'] schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) # list schedules, tenant filters filters = {} filters['tenant'] = TENANT1 schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) filters['tenant'] = 'aaaa-bbbb-cccc-dddd' schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 0) # list schedules, metadata filter filter = {} filter['instance_id'] = 'my_instance_1' schedules = self.client.list_schedules(filter_args=filter) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) filters['instance_id'] = 'aaaa-bbbb-cccc-dddd' schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 0) filter = {} filter['instance_name'] = 'aaaa-bbbb-cccc-dddd' schedules = self.client.list_schedules(filter_args=filter) self.assertEqual(len(schedules), 0) # test filter by action filter = {} filter['action'] = 'snapshot' schedules = self.client.list_schedules(filter_args=filter) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) filter['action'] = 'test_action' schedules = self.client.list_schedules(filter_args=filter) self.assertEqual(len(schedules), 0) # update schedule request = {'schedule': {'hour': 14}} updated_schedule = self.client.update_schedule(schedule['id'], request) self.assertEqual(updated_schedule['id'], schedule['id']) self.assertEqual(updated_schedule['tenant'], schedule['tenant']) self.assertEqual(updated_schedule['action'], schedule['action']) self.assertEqual(updated_schedule['minute'], None) self.assertEqual(updated_schedule['month'], None) self.assertEqual(updated_schedule['day_of_week'], None) self.assertEqual(updated_schedule['day_of_month'], None) self.assertEqual(updated_schedule['hour'], request['schedule']['hour']) self.assertNotEqual(updated_schedule['hour'], schedule['hour']) self.assertNotEqual(updated_schedule['next_run'], schedule['next_run']) # update schedule metadata request = { 'schedule': { 'metadata': { 'instance_id': 'my_instance_2', 'retention': '3', } } } schedule = self.client.get_schedule(schedule['id']) updated_schedule = self.client.update_schedule(schedule['id'], request) self.assertEqual(updated_schedule['id'], schedule['id']) self.assertEqual(updated_schedule['tenant'], schedule['tenant']) self.assertEqual(updated_schedule['action'], schedule['action']) self.assertEqual(updated_schedule['hour'], schedule['hour']) self.assertEqual(updated_schedule['minute'], schedule['minute']) self.assertEqual(updated_schedule['month'], schedule['month']) self.assertEqual(updated_schedule['day_of_week'], schedule['day_of_week']) self.assertEqual(updated_schedule['day_of_month'], schedule['day_of_month']) self.assertTrue('metadata' in updated_schedule) metadata = updated_schedule['metadata'] self.assertEqual(2, len(metadata)) self.assertEqual(metadata['instance_id'], 'my_instance_2') self.assertEqual(metadata['retention'], '3') # delete schedule self.client.delete_schedule(schedule['id']) # make sure schedule no longer exists self.assertRaises(client_exc.NotFound, self.client.get_schedule, schedule['id'])
def _process_job(self, job): payload = {'job': job} if job['status'] == 'QUEUED': self.send_notification_start(payload) else: self.send_notification_retry(payload) job_id = job['id'] hard_timeout = timeutils.normalize_time( timeutils.parse_isotime(job['hard_timeout'])) hard_timed_out = hard_timeout <= self._get_utcnow() if hard_timed_out: msg = ('Job %(job_id)s has reached/exceeded its' ' hard timeout: %(hard_timeout)s.' % { 'job_id': job_id, 'hard_timeout': job['hard_timeout'] }) self._job_hard_timed_out(job, msg) LOG.info( _('[%(worker_tag)s] Job hard timed out: %(msg)s') % { 'worker_tag': self.get_worker_tag(), 'msg': msg }) return max_retried = job['retry_count'] > self.max_retry if max_retried: msg = ('Job %(job_id)s has reached/exceeded its' ' max_retry count: %(retry_count)s.' % { 'job_id': job_id, 'retry_count': job['retry_count'] }) self._job_max_retried(job, msg) LOG.info( _('[%(worker_tag)s] Job max_retry reached: %(msg)s') % { 'worker_tag': self.get_worker_tag(), 'msg': msg }) return schedule = self._get_schedule(job) if schedule is None: msg = ('Schedule %(schedule_id)s not found for job %(job_id)s' % { 'schedule_id': job['schedule_id'], 'job_id': job_id }) self._job_cancelled(job, msg) LOG.info( _('[%(worker_tag)s] Job cancelled: %(msg)s') % { 'worker_tag': self.get_worker_tag(), 'msg': msg }) return now = self._get_utcnow() self.next_timeout = now + self.initial_timeout self._job_processing(job, self.next_timeout) self.next_update = self._get_utcnow() + self.update_interval instance_id = self._get_instance_id(job) if not instance_id: msg = ('Job %s does not specify an instance_id in its metadata.' % job_id) self._job_cancelled(job, msg) return image_id = self._get_image_id(job) if image_id is None: image_id = self._create_image(job, instance_id, schedule) if image_id is None: return else: LOG.info( _("[%(worker_tag)s] Resuming image: %(image_id)s") % { 'worker_tag': self.get_worker_tag(), 'image_id': image_id }) active = False retry = True while retry and not active and not self.stopping: image_status = self._poll_image_status(job, image_id) active = image_status == 'ACTIVE' if not active: retry = True try: self._update_job(job_id, "PROCESSING") except exc.OutOfTimeException: retry = False else: time.sleep(self.image_poll_interval) if active: self._process_retention(instance_id, self.current_job['schedule_id']) self._job_succeeded(self.current_job) elif not active and not retry: self._job_timed_out(self.current_job) elif self.stopping: # Timeout job so it gets picked up again quickly rather than # queuing up behind a bunch of new jobs, but not so soon that # another worker will pick it up before everything is shut down # and thus burn through the retries timeout = self._get_utcnow() + self.timeout_worker_stop self._job_processing(self.current_job, timeout=timeout) LOG.debug("[%s] Snapshot complete" % self.get_worker_tag())
def _process_job(self, job): payload = {'job': job} if job['status'] == 'QUEUED': self.send_notification_start(payload) else: self.send_notification_retry(payload) job_id = job['id'] hard_timeout = timeutils.normalize_time( timeutils.parse_isotime(job['hard_timeout'])) hard_timed_out = hard_timeout <= self._get_utcnow() if hard_timed_out: msg = ('Job %(job_id)s has reached/exceeded its' ' hard timeout: %(hard_timeout)s.' % {'job_id': job_id, 'hard_timeout': job['hard_timeout']}) self._job_hard_timed_out(job, msg) LOG.info(_('[%(worker_tag)s] Job hard timed out: %(msg)s') % {'worker_tag': self.get_worker_tag(), 'msg': msg}) return max_retried = job['retry_count'] > self.max_retry if max_retried: msg = ('Job %(job_id)s has reached/exceeded its' ' max_retry count: %(retry_count)s.' % {'job_id': job_id, 'retry_count': job['retry_count']}) self._job_max_retried(job, msg) LOG.info(_('[%(worker_tag)s] Job max_retry reached: %(msg)s') % {'worker_tag': self.get_worker_tag(), 'msg': msg}) return schedule = self._get_schedule(job) if schedule is None: msg = ('Schedule %(schedule_id)s not found for job %(job_id)s' % {'schedule_id': job['schedule_id'], 'job_id': job_id}) self._job_cancelled(job, msg) LOG.info(_('[%(worker_tag)s] Job cancelled: %(msg)s') % {'worker_tag': self.get_worker_tag(), 'msg': msg}) return now = self._get_utcnow() self.next_timeout = now + self.initial_timeout self._job_processing(job, self.next_timeout) self.next_update = self._get_utcnow() + self.update_interval instance_id = self._get_instance_id(job) if not instance_id: msg = ('Job %s does not specify an instance_id in its metadata.' % job_id) self._job_cancelled(job, msg) return image_id = self._get_image_id(job) if image_id is None: image_id = self._create_image(job, instance_id, schedule) if image_id is None: return else: LOG.info(_("[%(worker_tag)s] Resuming image: %(image_id)s") % {'worker_tag': self.get_worker_tag(), 'image_id': image_id}) active = False retry = True while retry and not active and not self.stopping: image_status = self._poll_image_status(job, image_id) active = image_status == 'ACTIVE' if not active: retry = True try: self._update_job(job_id, "PROCESSING") except exc.OutOfTimeException: retry = False else: time.sleep(self.image_poll_interval) if active: self._process_retention(instance_id, self.current_job['schedule_id']) self._job_succeeded(self.current_job) elif not active and not retry: self._job_timed_out(self.current_job) elif self.stopping: # Timeout job so it gets picked up again quickly rather than # queuing up behind a bunch of new jobs, but not so soon that # another worker will pick it up before everything is shut down # and thus burn through the retries timeout = self._get_utcnow() + self.timeout_worker_stop self._job_processing(self.current_job, timeout=timeout) LOG.debug("[%s] Snapshot complete" % self.get_worker_tag())
def test_schedule_workflow(self): schedules = self.client.list_schedules() self.assertEqual(len(schedules), 0) # create invalid schedule request = {'not a schedule': 'yes'} self.assertRaises(client_exc.BadRequest, self.client.create_schedule, request) # create malformed schedule request = 'not a schedule' self.assertRaises(client_exc.BadRequest, self.client.create_schedule, request) # create schedule with no body self.assertRaises(client_exc.BadRequest, self.client.create_schedule, None) # create schedule request = { 'schedule': { 'tenant': TENANT1, 'action': 'snapshot', 'minute': 30, 'hour': 12, 'metadata': {'instance_id': 'my_instance_1'}, } } schedule = self.client.create_schedule(request) self.assertTrue(schedule['id']) self.assertEqual(schedule['tenant'], TENANT1) self.assertEqual(schedule['action'], 'snapshot') self.assertEqual(schedule['minute'], 30) self.assertEqual(schedule['hour'], 12) self.assertTrue('metadata' in schedule) metadata = schedule['metadata'] self.assertEqual(1, len(metadata)) self.assertEqual(metadata['instance_id'], 'my_instance_1') # Get schedule schedule = self.client.get_schedule(schedule['id']) self.assertTrue(schedule['id']) self.assertEqual(schedule['tenant'], TENANT1) self.assertEqual(schedule['action'], 'snapshot') self.assertEqual(schedule['minute'], 30) self.assertEqual(schedule['hour'], 12) # list schedules schedules = self.client.list_schedules() self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) # list schedules, next_run filters filters = {} filters['next_run_after'] = schedule['next_run'] filters['next_run_before'] = schedule['next_run'] schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) filters['next_run_after'] = schedule['next_run'] after_datetime = timeutils.parse_isotime(schedule['next_run']) before_datetime = (after_datetime - datetime.timedelta(seconds=1)) filters['next_run_before'] = timeutils.isotime(before_datetime) schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 0) # list schedules, next_run_before filters filters = {} filters['next_run_before'] = schedule['next_run'] schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) # list schedules, next_run_after filters filters = {} filters['next_run_after'] = schedule['next_run'] schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) # list schedules, tenant filters filters = {} filters['tenant'] = TENANT1 schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) filters['tenant'] = 'aaaa-bbbb-cccc-dddd' schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 0) # list schedules, metadata filter filter = {} filter['instance_id'] = 'my_instance_1' schedules = self.client.list_schedules(filter_args=filter) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) filters['instance_id'] = 'aaaa-bbbb-cccc-dddd' schedules = self.client.list_schedules(filter_args=filters) self.assertEqual(len(schedules), 0) filter = {} filter['instance_name'] = 'aaaa-bbbb-cccc-dddd' schedules = self.client.list_schedules(filter_args=filter) self.assertEqual(len(schedules), 0) # test filter by action filter = {} filter['action'] = 'snapshot' schedules = self.client.list_schedules(filter_args=filter) self.assertEqual(len(schedules), 1) self.assertEqual(schedules[0], schedule) filter['action'] = 'test_action' schedules = self.client.list_schedules(filter_args=filter) self.assertEqual(len(schedules), 0) # update schedule request = {'schedule': {'hour': 14}} updated_schedule = self.client.update_schedule(schedule['id'], request) self.assertEqual(updated_schedule['id'], schedule['id']) self.assertEqual(updated_schedule['tenant'], schedule['tenant']) self.assertEqual(updated_schedule['action'], schedule['action']) self.assertEqual(updated_schedule['minute'], None) self.assertEqual(updated_schedule['month'], None) self.assertEqual(updated_schedule['day_of_week'], None) self.assertEqual(updated_schedule['day_of_month'], None) self.assertEqual(updated_schedule['hour'], request['schedule']['hour']) self.assertNotEqual(updated_schedule['hour'], schedule['hour']) self.assertNotEqual(updated_schedule['next_run'], schedule['next_run']) # update schedule metadata request = {'schedule': {'metadata': {'instance_id': 'my_instance_2', 'retention': '3', }}} schedule = self.client.get_schedule(schedule['id']) updated_schedule = self.client.update_schedule(schedule['id'], request) self.assertEqual(updated_schedule['id'], schedule['id']) self.assertEqual(updated_schedule['tenant'], schedule['tenant']) self.assertEqual(updated_schedule['action'], schedule['action']) self.assertEqual(updated_schedule['hour'], schedule['hour']) self.assertEqual(updated_schedule['minute'], schedule['minute']) self.assertEqual(updated_schedule['month'], schedule['month']) self.assertEqual(updated_schedule['day_of_week'], schedule['day_of_week']) self.assertEqual(updated_schedule['day_of_month'], schedule['day_of_month']) self.assertTrue('metadata' in updated_schedule) metadata = updated_schedule['metadata'] self.assertEqual(2, len(metadata)) self.assertEqual(metadata['instance_id'], 'my_instance_2') self.assertEqual(metadata['retention'], '3') # delete schedule self.client.delete_schedule(schedule['id']) # make sure schedule no longer exists self.assertRaises(client_exc.NotFound, self.client.get_schedule, schedule['id'])
def create(self, request, body): if (body is None or body.get('job') is None or body['job'].get('schedule_id') is None): raise webob.exc.HTTPBadRequest() job = body['job'] try: schedule = self.db_api.schedule_get_by_id(job['schedule_id']) except exception.NotFound: raise webob.exc.HTTPNotFound() # Check integrity of schedule and update next run expected_next_run = job.get('next_run') if expected_next_run: try: expected_next_run = timeutils.parse_isotime(expected_next_run) expected_next_run = expected_next_run.replace(tzinfo=None) except ValueError as e: msg = _('Invalid "next_run" value. Must be ISO 8601 format') raise webob.exc.HTTPBadRequest(explanation=msg) next_run = api_utils.schedule_to_next_run(schedule, timeutils.utcnow()) next_run = next_run.replace(tzinfo=None) try: self.db_api.schedule_test_and_set_next_run(schedule['id'], expected_next_run, next_run) except exception.NotFound: msg = _("Specified next run does not match the current next run" " value. This could mean schedule has either changed" "or has already been scheduled since you last expected.") raise webob.exc.HTTPConflict(explanation=msg) # Update schedule last_scheduled values = {} values['last_scheduled'] = timeutils.utcnow() self.db_api.schedule_update(schedule['id'], values) # Create job values = {} values.update(job) values['tenant'] = schedule['tenant'] values['action'] = schedule['action'] values['status'] = 'QUEUED' job_metadata = [] for metadata in schedule['schedule_metadata']: job_metadata.append({ 'key': metadata['key'], 'value': metadata['value'] }) values['job_metadata'] = job_metadata job_action = values['action'] if not 'timeout' in values: values['timeout'] = api_utils.get_new_timeout_by_action(job_action) values['hard_timeout'] = \ api_utils.get_new_timeout_by_action(job_action) job = self.db_api.job_create(values) utils.serialize_datetimes(job) api_utils.serialize_job_metadata(job) job = {'job': job} utils.generate_notification(None, 'qonos.job.create', job, 'INFO') return job
def _create_jobs(self): fixture = { 'id': unit_utils.SCHEDULE_UUID1, 'tenant': unit_utils.TENANT1, 'action': 'snapshot', 'minute': '30', 'hour': '2', 'next_run': timeutils.parse_isotime('2012-11-27T02:30:00Z') } self.schedule_1 = db_api.schedule_create(fixture) fixture = { 'id': unit_utils.SCHEDULE_UUID2, 'tenant': unit_utils.TENANT2, 'action': 'snapshot', 'minute': '30', 'hour': '2', 'next_run': timeutils.parse_isotime('2012-11-27T02:30:00Z'), 'schedule_metadata': [ { 'key': 'instance_id', 'value': 'my_instance', } ], } self.schedule_2 = db_api.schedule_create(fixture) now = timeutils.utcnow() timeout = now + datetime.timedelta(hours=1) hard_timeout = now + datetime.timedelta(hours=4) fixture = { 'id': unit_utils.JOB_UUID1, 'schedule_id': self.schedule_1['id'], 'tenant': unit_utils.TENANT1, 'worker_id': unit_utils.WORKER_UUID1, 'action': 'snapshot', 'status': 'QUEUED', 'timeout': timeout, 'hard_timeout': hard_timeout, 'retry_count': 0, } self.job_1 = db_api.job_create(fixture) fixture = { 'id': unit_utils.JOB_UUID2, 'schedule_id': self.schedule_2['id'], 'tenant': unit_utils.TENANT2, 'worker_id': unit_utils.WORKER_UUID2, 'action': 'snapshot', 'status': 'ERROR', 'timeout': timeout, 'hard_timeout': hard_timeout, 'retry_count': 1, 'job_metadata': [ { 'key': 'instance_id', 'value': 'my_instance', }, ] } self.job_2 = db_api.job_create(fixture) fixture = { 'id': unit_utils.JOB_UUID3, 'schedule_id': self.schedule_1['id'], 'tenant': unit_utils.TENANT1, 'worker_id': unit_utils.WORKER_UUID1, 'action': 'snapshot', 'status': 'QUEUED', 'timeout': timeout, 'hard_timeout': hard_timeout, 'retry_count': 0, } self.job_3 = db_api.job_create(fixture) fixture = { 'id': unit_utils.JOB_UUID4, 'schedule_id': self.schedule_1['id'], 'tenant': unit_utils.TENANT1, 'worker_id': unit_utils.WORKER_UUID1, 'action': 'snapshot', 'status': 'QUEUED', 'timeout': timeout, 'hard_timeout': hard_timeout, 'retry_count': 0, } self.job_4 = db_api.job_create(fixture)