def test_do_cleanup_not_cleaning_already_claimed(self): """Basic cleanup that doesn't touch already cleaning works.""" vol = utils.create_volume(self.context, status='creating') worker1 = db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) worker1 = db.worker_get(self.context, id=worker1.id) vol2 = utils.create_volume(self.context, status='deleting') worker2 = db.worker_create(self.context, status='deleting', resource_type='Volume', resource_id=vol2.id, service_id=self.service.id + 1) worker2 = db.worker_get(self.context, id=worker2.id) # Simulate that the change to vol2 worker happened between # worker_get_all and trying to claim a work for cleanup worker2.service_id = self.service.id clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) with mock.patch('cinder.db.worker_get_all') as get_all_mock: get_all_mock.return_value = [worker1, worker2] mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) self.assertEqual(worker2.id, workers[0].id) vol.refresh() self.assertEqual('creating_cleaned', vol.status) vol2.refresh() self.assertEqual('deleting', vol2.status)
def _prepare_params(self, ctxt, params, allowed): if not allowed.issuperset(params): invalid_keys = set(params).difference(allowed) msg = _('Invalid filter keys: %s') % ', '.join(invalid_keys) raise exception.InvalidInput(reason=msg) if params.get('binary') not in (None, 'cinder-volume', 'cinder-scheduler'): msg = _('binary must be empty or set to cinder-volume or ' 'cinder-scheduler') raise exception.InvalidInput(reason=msg) for boolean in ('disabled', 'is_up'): if params.get(boolean) is not None: params[boolean] = utils.get_bool_param(boolean, params) resource_type = params.get('resource_type') if resource_type: resource_type = resource_type.title() types = cleanable.CinderCleanableObject.cleanable_resource_types if resource_type not in types: msg = (_('Resource type %s not valid, must be ') % resource_type) msg = utils.build_or_str(types, msg + '%s.') raise exception.InvalidInput(reason=msg) params['resource_type'] = resource_type resource_id = params.get('resource_id') if resource_id: if not uuidutils.is_uuid_like(resource_id): msg = (_('Resource ID must be a UUID, and %s is not.') % resource_id) raise exception.InvalidInput(reason=msg) # If we have the resource type but we don't have where it is # located, we get it from the DB to limit the distribution of the # request by the scheduler, otherwise it will be distributed to all # the services. location_keys = {'service_id', 'cluster_name', 'host'} if not location_keys.intersection(params): workers = db.worker_get_all(ctxt, resource_id=resource_id, binary=params.get('binary'), resource_type=resource_type) if len(workers) == 0: msg = (_('There is no resource with UUID %s pending ' 'cleanup.'), resource_id) raise exception.InvalidInput(reason=msg) if len(workers) > 1: msg = (_('There are multiple resources with UUID %s ' 'pending cleanup. Please be more specific.'), resource_id) raise exception.InvalidInput(reason=msg) worker = workers[0] params.update(service_id=worker.service_id, resource_type=worker.resource_type) return params
def test_do_cleanup_not_cleaning_already_claimed(self): """Basic cleanup that doesn't touch already cleaning works.""" vol = utils.create_volume(self.context, status='creating') worker1 = db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) worker1 = db.worker_get(self.context, id=worker1.id) vol2 = utils.create_volume(self.context, status='deleting') worker2 = db.worker_create(self.context, status='deleting', resource_type='Volume', resource_id=vol2.id, service_id=self.service.id + 1) worker2 = db.worker_get(self.context, id=worker2.id) # Simulate that the change to vol2 worker happened between # worker_get_all and trying to claim a work for cleanup worker2.service_id = self.service.id clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) with mock.patch('cinder.db.worker_get_all') as get_all_mock: get_all_mock.return_value = [worker1, worker2] mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) self.assertEqual(worker2.id, workers[0].id) vol.refresh() self.assertEqual('creating_cleaned', vol.status) vol2.refresh() self.assertEqual('deleting', vol2.status)
def test_worker_destroy(self): """Test that worker destroy really deletes the DB entry.""" worker = self._create_workers(1)[0] res = db.worker_destroy(self.ctxt, id=worker.id) self.assertEqual(1, res) db_workers = db.worker_get_all(self.ctxt, read_deleted='yes') self.assertListEqual([], db_workers)
def test_worker_destroy(self): """Test that worker destroy really deletes the DB entry.""" worker = self._create_workers(1)[0] res = db.worker_destroy(self.ctxt, id=worker.id) self.assertEqual(1, res) db_workers = db.worker_get_all(self.ctxt, read_deleted='yes') self.assertListEqual([], db_workers)
def test_worker_get_all(self): """Test basic get_all method.""" self._create_workers(1) service = db.service_create(self.ctxt, {}) workers = self._create_workers(3, service_id=service.id) db_workers = db.worker_get_all(self.ctxt, service_id=service.id) self._assertEqualListsOfObjects(workers, db_workers)
def test_worker_get_all(self): """Test basic get_all method.""" self._create_workers(1) service = db.service_create(self.ctxt, {}) workers = self._create_workers(3, service_id=service.id) db_workers = db.worker_get_all(self.ctxt, service_id=service.id) self._assertEqualListsOfObjects(workers, db_workers)
def test_worker_get_all_until(self): """Test get_all until a specific time.""" workers = self._create_workers(3, read_back=True) timestamp = workers[-1].updated_at time.sleep(0.1) self._create_workers(3) db_workers = db.worker_get_all(self.ctxt, until=timestamp) self._assertEqualListsOfObjects(workers, db_workers)
def test_worker_get_all_until(self): """Test get_all until a specific time.""" workers = self._create_workers(3, read_back=True) timestamp = workers[-1].updated_at time.sleep(0.1) self._create_workers(3) db_workers = db.worker_get_all(self.ctxt, until=timestamp) self._assertEqualListsOfObjects(workers, db_workers)
def test_do_cleanup_resource_deleted(self): """Cleanup on a resource that's been already deleted.""" vol = utils.create_volume(self.context, status='creating') db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) vol.destroy() clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertListEqual([], workers)
def test_do_cleanup(self): """Basic successful cleanup.""" vol = utils.create_volume(self.context, status='creating') db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) self.assertListEqual([], db.worker_get_all(self.context)) vol.refresh() self.assertEqual('creating_cleaned', vol.status)
def test_do_cleanup_revive_on_cleanup_fail(self, mock_clean): """Cleanup will revive a worker if cleanup fails.""" vol = utils.create_volume(self.context, status='creating') db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) vol.refresh() self.assertEqual('creating', vol.status)
def test_do_cleanup_keep_worker(self): """Cleanup on a resource that will remove worker when cleaning up.""" vol = utils.create_volume(self.context, status='deleting') db.worker_create(self.context, status='deleting', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id, keep_after_clean=True) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) vol.refresh() self.assertEqual('deleting_cleaned', vol.status)
def test_do_cleanup(self): """Basic successful cleanup.""" vol = utils.create_volume(self.context, status='creating') db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) self.assertListEqual([], db.worker_get_all(self.context)) vol.refresh() self.assertEqual('creating_cleaned', vol.status)
def test_do_cleanup_resource_deleted(self): """Cleanup on a resource that's been already deleted.""" vol = utils.create_volume(self.context, status='creating') db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) vol.destroy() clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertListEqual([], workers)
def test_do_cleanup_resource_on_another_service(self): """Cleanup on a resource that's been claimed by other service.""" vol = utils.create_volume(self.context, status='deleting') db.worker_create(self.context, status='deleting', resource_type='Volume', resource_id=vol.id, service_id=self.service.id + 1) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) vol.refresh() self.assertEqual('deleting', vol.status)
def test_do_cleanup_resource_changed_status(self): """Cleanup on a resource that's changed status.""" vol = utils.create_volume(self.context, status='available') db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertListEqual([], workers) vol.refresh() self.assertEqual('available', vol.status)
def test_do_cleanup_revive_on_cleanup_fail(self, mock_clean): """Cleanup will revive a worker if cleanup fails.""" vol = utils.create_volume(self.context, status='creating') db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) vol.refresh() self.assertEqual('creating', vol.status)
def test_do_cleanup_not_cleaning_already_claimed_by_us(self): """Basic cleanup that doesn't touch other thread's claimed works.""" now = timeutils.utcnow() delta = timeutils.datetime.timedelta(seconds=1) original_time = now - delta # Creating the worker in the future, and then changing the in-memory # value of worker2.updated_at to an earlier time, we effectively # simulate that the worker entry was created in the past and that it # has been just updated between worker_get_all and trying # to claim a work for cleanup other_thread_claimed_time = now + delta vol = utils.create_volume(self.context, status='creating') worker1 = db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id, updated_at=original_time) worker1 = db.worker_get(self.context, id=worker1.id) vol2 = utils.create_volume(self.context, status='deleting') worker2 = db.worker_create(self.context, status='deleting', resource_type='Volume', resource_id=vol2.id, service_id=self.service.id, updated_at=other_thread_claimed_time) worker2 = db.worker_get(self.context, id=worker2.id) # This with the mock below simulates worker2 was created in the past # and updated right between worker_get_all and worker_claim_for_cleanup worker2.updated_at = original_time clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) with mock.patch('cinder.manager.timeutils.utcnow', return_value=now),\ mock.patch('cinder.db.worker_get_all') as get_all_mock: get_all_mock.return_value = [worker1, worker2] mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) self.assertEqual(worker2.id, workers[0].id) vol.refresh() self.assertEqual('creating_cleaned', vol.status) vol2.refresh() self.assertEqual('deleting', vol2.status)
def test_do_cleanup_keep_worker(self): """Cleanup on a resource that will remove worker when cleaning up.""" vol = utils.create_volume(self.context, status='deleting') db.worker_create(self.context, status='deleting', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id, keep_after_clean=True) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) vol.refresh() self.assertEqual('deleting_cleaned', vol.status)
def test_do_cleanup_resource_changed_status(self): """Cleanup on a resource that's changed status.""" vol = utils.create_volume(self.context, status='available') db.worker_create(self.context, status='creating', resource_type='Volume', resource_id=vol.id, service_id=self.service.id) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertListEqual([], workers) vol.refresh() self.assertEqual('available', vol.status)
def test_do_cleanup_resource_on_another_service(self): """Cleanup on a resource that's been claimed by other service.""" vol = utils.create_volume(self.context, status='deleting') db.worker_create(self.context, status='deleting', resource_type='Volume', resource_id=vol.id, service_id=self.service.id + 1) clean_req = objects.CleanupRequest(service_id=self.service.id) mngr = FakeManager(self.service.id) mngr.do_cleanup(self.context, clean_req) workers = db.worker_get_all(self.context) self.assertEqual(1, len(workers)) vol.refresh() self.assertEqual('deleting', vol.status)
def test_worker_get_all_returns_empty(self): """Test that get_all returns an empty list when there's no results.""" self._create_workers(3, deleted=True) db_workers = db.worker_get_all(self.ctxt) self.assertListEqual([], db_workers)
def cleanup(self, req, body=None): """Do the cleanup on resources from a specific service/host/node.""" # Let the wsgi middleware convert NotAuthorized exceptions ctxt = req.environ['cinder.context'] ctxt.authorize(policy.CLEAN_POLICY) body = body or {} for boolean in ('disabled', 'is_up'): if body.get(boolean) is not None: body[boolean] = strutils.bool_from_string(body[boolean]) resource_type = body.get('resource_type') if resource_type: resource_type = resource_type.title() types = cleanable.CinderCleanableObject.cleanable_resource_types if resource_type not in types: valid_types = utils.build_or_str(types) msg = _('Resource type %(resource_type)s not valid,' ' must be %(valid_types)s') msg = msg % {"resource_type": resource_type, "valid_types": valid_types} raise exception.InvalidInput(reason=msg) body['resource_type'] = resource_type resource_id = body.get('resource_id') if resource_id: # If we have the resource type but we don't have where it is # located, we get it from the DB to limit the distribution of the # request by the scheduler, otherwise it will be distributed to all # the services. location_keys = {'service_id', 'cluster_name', 'host'} if not location_keys.intersection(body): workers = db.worker_get_all(ctxt, resource_id=resource_id, binary=body.get('binary'), resource_type=resource_type) if len(workers) == 0: msg = (_('There is no resource with UUID %s pending ' 'cleanup.'), resource_id) raise exception.InvalidInput(reason=msg) if len(workers) > 1: msg = (_('There are multiple resources with UUID %s ' 'pending cleanup. Please be more specific.'), resource_id) raise exception.InvalidInput(reason=msg) worker = workers[0] body.update(service_id=worker.service_id, resource_type=worker.resource_type) body['until'] = timeutils.utcnow() # NOTE(geguileo): If is_up is not specified in the request # CleanupRequest's default will be used (False) cleanup_request = objects.CleanupRequest(**body) cleaning, unavailable = self.sch_api.work_cleanup(ctxt, cleanup_request) return { 'cleaning': workers_view.ViewBuilder.service_list(cleaning), 'unavailable': workers_view.ViewBuilder.service_list(unavailable), }
def do_cleanup(self, context, cleanup_request): LOG.info('Initiating service %s cleanup', cleanup_request.service_id) # If the 'until' field in the cleanup request is not set, we default to # this very moment. until = cleanup_request.until or timeutils.utcnow() keep_entry = False to_clean = db.worker_get_all( context, resource_type=cleanup_request.resource_type, resource_id=cleanup_request.resource_id, service_id=cleanup_request.service_id, until=until) for clean in to_clean: original_service_id = clean.service_id original_time = clean.updated_at # Try to do a soft delete to mark the entry as being cleaned up # by us (setting service id to our service id). res = db.worker_claim_for_cleanup(context, claimer_id=self.service_id, orm_worker=clean) # Claim may fail if entry is being cleaned by another service, has # been removed (finished cleaning) by another service or the user # started a new cleanable operation. # In any of these cases we don't have to do cleanup or remove the # worker entry. if not res: continue # Try to get versioned object for resource we have to cleanup try: vo_cls = getattr(objects, clean.resource_type) vo = vo_cls.get_by_id(context, clean.resource_id) # Set the worker DB entry in the VO and mark it as being a # clean operation clean.cleaning = True vo.worker = clean except exception.NotFound: LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.', {'type': clean.resource_type, 'id': clean.resource_id}) else: # Resource status should match if vo.status != clean.status: LOG.debug('Skipping cleanup for mismatching work on ' '%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.', {'type': clean.resource_type, 'id': clean.resource_id, 'exp_sts': clean.status, 'found_sts': vo.status}) else: LOG.info('Cleaning %(type)s with id %(id)s and status ' '%(status)s', {'type': clean.resource_type, 'id': clean.resource_id, 'status': clean.status}, resource=vo) try: # Some cleanup jobs are performed asynchronously, so # we don't delete the worker entry, they'll take care # of it keep_entry = self._do_cleanup(context, vo) except Exception: LOG.exception('Could not perform cleanup.') # Return the worker DB entry to the original service db.worker_update(context, clean.id, service_id=original_service_id, updated_at=original_time) continue # The resource either didn't exist or was properly cleaned, either # way we can remove the entry from the worker table if the cleanup # method doesn't want to keep the entry (for example for delayed # deletion). if not keep_entry and not db.worker_destroy(context, id=clean.id): LOG.warning('Could not remove worker entry %s.', clean.id) LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
def test_worker_get_all_returns_empty(self): """Test that get_all returns an empty list when there's no results.""" self._create_workers(3, deleted=True) db_workers = db.worker_get_all(self.ctxt) self.assertListEqual([], db_workers)
def cleanup(self, req, body=None): """Do the cleanup on resources from a specific service/host/node.""" # Let the wsgi middleware convert NotAuthorized exceptions ctxt = req.environ['cinder.context'] ctxt.authorize(policy.CLEAN_POLICY) body = body or {} for boolean in ('disabled', 'is_up'): if body.get(boolean) is not None: body[boolean] = strutils.bool_from_string(body[boolean]) resource_type = body.get('resource_type') if resource_type: resource_type = resource_type.title() types = cleanable.CinderCleanableObject.cleanable_resource_types if resource_type not in types: valid_types = utils.build_or_str(types) msg = _('Resource type %(resource_type)s not valid,' ' must be %(valid_types)s') msg = msg % { "resource_type": resource_type, "valid_types": valid_types } raise exception.InvalidInput(reason=msg) body['resource_type'] = resource_type resource_id = body.get('resource_id') if resource_id: # If we have the resource type but we don't have where it is # located, we get it from the DB to limit the distribution of the # request by the scheduler, otherwise it will be distributed to all # the services. location_keys = {'service_id', 'cluster_name', 'host'} if not location_keys.intersection(body): workers = db.worker_get_all(ctxt, resource_id=resource_id, binary=body.get('binary'), resource_type=resource_type) if len(workers) == 0: msg = (_('There is no resource with UUID %s pending ' 'cleanup.'), resource_id) raise exception.InvalidInput(reason=msg) if len(workers) > 1: msg = (_('There are multiple resources with UUID %s ' 'pending cleanup. Please be more specific.'), resource_id) raise exception.InvalidInput(reason=msg) worker = workers[0] body.update(service_id=worker.service_id, resource_type=worker.resource_type) body['until'] = timeutils.utcnow() # NOTE(geguileo): If is_up is not specified in the request # CleanupRequest's default will be used (False) cleanup_request = objects.CleanupRequest(**body) cleaning, unavailable = self.sch_api.work_cleanup( ctxt, cleanup_request) return { 'cleaning': workers_view.ViewBuilder.service_list(cleaning), 'unavailable': workers_view.ViewBuilder.service_list(unavailable), }
def _assert_workers_are_removed(self): workers = db.worker_get_all(self.context, read_deleted='yes') self.assertListEqual([], workers)
def do_cleanup(self, context, cleanup_request): LOG.info('Initiating service %s cleanup', cleanup_request.service_id) # If the 'until' field in the cleanup request is not set, we default to # this very moment. until = cleanup_request.until or timeutils.utcnow() keep_entry = False to_clean = db.worker_get_all( context, resource_type=cleanup_request.resource_type, resource_id=cleanup_request.resource_id, service_id=cleanup_request.service_id, until=until) for clean in to_clean: original_service_id = clean.service_id original_time = clean.updated_at # Try to do a soft delete to mark the entry as being cleaned up # by us (setting service id to our service id). res = db.worker_claim_for_cleanup(context, claimer_id=self.service_id, orm_worker=clean) # Claim may fail if entry is being cleaned by another service, has # been removed (finished cleaning) by another service or the user # started a new cleanable operation. # In any of these cases we don't have to do cleanup or remove the # worker entry. if not res: continue # Try to get versioned object for resource we have to cleanup try: vo_cls = getattr(objects, clean.resource_type) vo = vo_cls.get_by_id(context, clean.resource_id) # Set the worker DB entry in the VO and mark it as being a # clean operation clean.cleaning = True vo.worker = clean except exception.NotFound: LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.', { 'type': clean.resource_type, 'id': clean.resource_id }) else: # Resource status should match if vo.status != clean.status: LOG.debug( 'Skipping cleanup for mismatching work on ' '%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.', { 'type': clean.resource_type, 'id': clean.resource_id, 'exp_sts': clean.status, 'found_sts': vo.status }) else: LOG.info( 'Cleaning %(type)s with id %(id)s and status ' '%(status)s', { 'type': clean.resource_type, 'id': clean.resource_id, 'status': clean.status }, resource=vo) try: # Some cleanup jobs are performed asynchronously, so # we don't delete the worker entry, they'll take care # of it keep_entry = self._do_cleanup(context, vo) except Exception: LOG.exception('Could not perform cleanup.') # Return the worker DB entry to the original service db.worker_update(context, clean.id, service_id=original_service_id, updated_at=original_time) continue # The resource either didn't exist or was properly cleaned, either # way we can remove the entry from the worker table if the cleanup # method doesn't want to keep the entry (for example for delayed # deletion). if not keep_entry and not db.worker_destroy(context, id=clean.id): LOG.warning('Could not remove worker entry %s.', clean.id) LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
def _assert_workers_are_removed(self): workers = db.worker_get_all(self.context, read_deleted='yes') self.assertListEqual([], workers)