def test_worker_claim_fails_this_service_claimed(self): """Test claim fails when worker was already claimed by this service.""" service_id = 1 worker = db.worker_create(self.ctxt, resource_type='Volume', resource_id=fake.VOLUME_ID, status='creating', service_id=service_id) # Read it back to have the updated_at value worker = db.worker_get(self.ctxt, id=worker.id) claimed_worker = db.worker_get(self.ctxt, id=worker.id) time.sleep(0.1) # Simulate that this service starts processing this entry res = db.worker_claim_for_cleanup(self.ctxt, service_id, claimed_worker) self.assertEqual(1, res) res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker) self.assertEqual(0, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self._assertEqualObjects(claimed_worker, db_worker) self._assertEqualObjects(worker, db_worker, ['updated_at', 'race_preventer']) self.assertNotEqual(worker.updated_at, db_worker.updated_at) self.assertEqual(worker.race_preventer + 1, db_worker.race_preventer)
def test_worker_claim_fails_this_service_claimed(self): """Test claim fails when worker was already claimed by this service.""" service_id = 1 worker = db.worker_create(self.ctxt, resource_type='Volume', resource_id=fake.VOLUME_ID, status='creating', service_id=service_id) # Read it back to have the updated_at value worker = db.worker_get(self.ctxt, id=worker.id) claimed_worker = db.worker_get(self.ctxt, id=worker.id) time.sleep(0.1) # Simulate that this service starts processing this entry res = db.worker_claim_for_cleanup(self.ctxt, service_id, claimed_worker) self.assertEqual(1, res) res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker) self.assertEqual(0, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self._assertEqualObjects(claimed_worker, db_worker) self._assertEqualObjects(worker, db_worker, ['updated_at']) self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def test_worker_claim_fails_status_change(self): """Test that claim fails if the work entry has changed its status.""" worker = db.worker_create(self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting") worker.status = "creating" res = db.worker_claim_for_cleanup(self.ctxt, 1, worker) self.assertEqual(0, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self._assertEqualObjects(worker, db_worker, ["status"]) self.assertIsNone(db_worker.service_id)
def test_worker_claim(self): """Test worker claim of normal DB entry.""" service_id = 1 worker = db.worker_create(self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting") res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker) self.assertEqual(1, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self._assertEqualObjects(worker, db_worker, ["updated_at"]) self.assertEqual(service_id, db_worker.service_id) self.assertEqual(worker.service_id, db_worker.service_id)
def test_worker_claim_fails_status_change(self): """Test that claim fails if the work entry has changed its status.""" worker = db.worker_create(self.ctxt, resource_type='Volume', resource_id=fake.VOLUME_ID, status='deleting') worker.status = 'creating' res = db.worker_claim_for_cleanup(self.ctxt, 1, worker) self.assertEqual(0, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self._assertEqualObjects(worker, db_worker, ['status']) self.assertIsNone(db_worker.service_id)
def test_worker_claim_fails_service_change(self): """Test that claim fails on worker service change.""" failed_service = 1 working_service = 2 this_service = 3 worker = db.worker_create( self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting", service_id=working_service ) worker.service_id = failed_service res = db.worker_claim_for_cleanup(self.ctxt, this_service, worker) self.assertEqual(0, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self.assertEqual(working_service, db_worker.service_id)
def test_worker_claim_fails_service_change(self): """Test that claim fails on worker service change.""" failed_service = 1 working_service = 2 this_service = 3 worker = db.worker_create(self.ctxt, resource_type='Volume', resource_id=fake.VOLUME_ID, status='deleting', service_id=working_service) worker.service_id = failed_service res = db.worker_claim_for_cleanup(self.ctxt, this_service, worker) self.assertEqual(0, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self.assertEqual(working_service, db_worker.service_id)
def test_worker_claim(self): """Test worker claim of normal DB entry.""" service_id = 1 worker = db.worker_create(self.ctxt, resource_type='Volume', resource_id=fake.VOLUME_ID, status='deleting') res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker) self.assertEqual(1, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self._assertEqualObjects(worker, db_worker, ['updated_at']) self.assertEqual(service_id, db_worker.service_id) self.assertEqual(worker.service_id, db_worker.service_id)
def test_worker_claim_same_service(self): """Test worker claim of a DB entry that has our service_id.""" service_id = 1 worker = db.worker_create( self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting", service_id=service_id ) # Read from DB to get updated_at field worker = db.worker_get(self.ctxt, id=worker.id) claimed_worker = db.worker_get(self.ctxt, id=worker.id) res = db.worker_claim_for_cleanup(self.ctxt, service_id, claimed_worker) self.assertEqual(1, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self._assertEqualObjects(claimed_worker, db_worker) self._assertEqualObjects(worker, db_worker, ["updated_at"]) self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def test_worker_claim_same_service(self): """Test worker claim of a DB entry that has our service_id.""" service_id = 1 worker = db.worker_create(self.ctxt, resource_type='Volume', resource_id=fake.VOLUME_ID, status='deleting', service_id=service_id) # Read from DB to get updated_at field worker = db.worker_get(self.ctxt, id=worker.id) claimed_worker = db.worker_get(self.ctxt, id=worker.id) res = db.worker_claim_for_cleanup(self.ctxt, service_id, claimed_worker) self.assertEqual(1, res) db_worker = db.worker_get(self.ctxt, id=worker.id) self._assertEqualObjects(claimed_worker, db_worker) self._assertEqualObjects(worker, db_worker, ['updated_at']) self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def do_cleanup(self, context, cleanup_request): LOG.info('Initiating service %s cleanup', cleanup_request.service_id) # If the 'until' field in the cleanup request is not set, we default to # this very moment. until = cleanup_request.until or timeutils.utcnow() keep_entry = False to_clean = db.worker_get_all( context, resource_type=cleanup_request.resource_type, resource_id=cleanup_request.resource_id, service_id=cleanup_request.service_id, until=until) for clean in to_clean: original_service_id = clean.service_id original_time = clean.updated_at # Try to do a soft delete to mark the entry as being cleaned up # by us (setting service id to our service id). res = db.worker_claim_for_cleanup(context, claimer_id=self.service_id, orm_worker=clean) # Claim may fail if entry is being cleaned by another service, has # been removed (finished cleaning) by another service or the user # started a new cleanable operation. # In any of these cases we don't have to do cleanup or remove the # worker entry. if not res: continue # Try to get versioned object for resource we have to cleanup try: vo_cls = getattr(objects, clean.resource_type) vo = vo_cls.get_by_id(context, clean.resource_id) # Set the worker DB entry in the VO and mark it as being a # clean operation clean.cleaning = True vo.worker = clean except exception.NotFound: LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.', { 'type': clean.resource_type, 'id': clean.resource_id }) else: # Resource status should match if vo.status != clean.status: LOG.debug( 'Skipping cleanup for mismatching work on ' '%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.', { 'type': clean.resource_type, 'id': clean.resource_id, 'exp_sts': clean.status, 'found_sts': vo.status }) else: LOG.info( 'Cleaning %(type)s with id %(id)s and status ' '%(status)s', { 'type': clean.resource_type, 'id': clean.resource_id, 'status': clean.status }, resource=vo) try: # Some cleanup jobs are performed asynchronously, so # we don't delete the worker entry, they'll take care # of it keep_entry = self._do_cleanup(context, vo) except Exception: LOG.exception('Could not perform cleanup.') # Return the worker DB entry to the original service db.worker_update(context, clean.id, service_id=original_service_id, updated_at=original_time) continue # The resource either didn't exist or was properly cleaned, either # way we can remove the entry from the worker table if the cleanup # method doesn't want to keep the entry (for example for delayed # deletion). if not keep_entry and not db.worker_destroy(context, id=clean.id): LOG.warning('Could not remove worker entry %s.', clean.id) LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
def do_cleanup(self, context, cleanup_request): LOG.info('Initiating service %s cleanup', cleanup_request.service_id) # If the 'until' field in the cleanup request is not set, we default to # this very moment. until = cleanup_request.until or timeutils.utcnow() keep_entry = False to_clean = db.worker_get_all( context, resource_type=cleanup_request.resource_type, resource_id=cleanup_request.resource_id, service_id=cleanup_request.service_id, until=until) for clean in to_clean: original_service_id = clean.service_id original_time = clean.updated_at # Try to do a soft delete to mark the entry as being cleaned up # by us (setting service id to our service id). res = db.worker_claim_for_cleanup(context, claimer_id=self.service_id, orm_worker=clean) # Claim may fail if entry is being cleaned by another service, has # been removed (finished cleaning) by another service or the user # started a new cleanable operation. # In any of these cases we don't have to do cleanup or remove the # worker entry. if not res: continue # Try to get versioned object for resource we have to cleanup try: vo_cls = getattr(objects, clean.resource_type) vo = vo_cls.get_by_id(context, clean.resource_id) # Set the worker DB entry in the VO and mark it as being a # clean operation clean.cleaning = True vo.worker = clean except exception.NotFound: LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.', {'type': clean.resource_type, 'id': clean.resource_id}) else: # Resource status should match if vo.status != clean.status: LOG.debug('Skipping cleanup for mismatching work on ' '%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.', {'type': clean.resource_type, 'id': clean.resource_id, 'exp_sts': clean.status, 'found_sts': vo.status}) else: LOG.info('Cleaning %(type)s with id %(id)s and status ' '%(status)s', {'type': clean.resource_type, 'id': clean.resource_id, 'status': clean.status}, resource=vo) try: # Some cleanup jobs are performed asynchronously, so # we don't delete the worker entry, they'll take care # of it keep_entry = self._do_cleanup(context, vo) except Exception: LOG.exception('Could not perform cleanup.') # Return the worker DB entry to the original service db.worker_update(context, clean.id, service_id=original_service_id, updated_at=original_time) continue # The resource either didn't exist or was properly cleaned, either # way we can remove the entry from the worker table if the cleanup # method doesn't want to keep the entry (for example for delayed # deletion). if not keep_entry and not db.worker_destroy(context, id=clean.id): LOG.warning('Could not remove worker entry %s.', clean.id) LOG.info('Service %s cleanup completed.', cleanup_request.service_id)