def test_worker_claim_fails_this_service_claimed(self):
"""Test claim fails when worker was already claimed by this service."""
service_id = 1
worker = db.worker_create(self.ctxt, resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='creating',
service_id=service_id)
# Read it back to have the updated_at value
worker = db.worker_get(self.ctxt, id=worker.id)
claimed_worker = db.worker_get(self.ctxt, id=worker.id)
time.sleep(0.1)
# Simulate that this service starts processing this entry
res = db.worker_claim_for_cleanup(self.ctxt,
service_id,
claimed_worker)
self.assertEqual(1, res)
res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(claimed_worker, db_worker)
self._assertEqualObjects(worker, db_worker,
['updated_at', 'race_preventer'])
self.assertNotEqual(worker.updated_at, db_worker.updated_at)
self.assertEqual(worker.race_preventer + 1, db_worker.race_preventer)
def test_worker_claim_fails_this_service_claimed(self):
"""Test claim fails when worker was already claimed by this service."""
service_id = 1
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='creating',
service_id=service_id)
# Read it back to have the updated_at value
worker = db.worker_get(self.ctxt, id=worker.id)
claimed_worker = db.worker_get(self.ctxt, id=worker.id)
time.sleep(0.1)
# Simulate that this service starts processing this entry
res = db.worker_claim_for_cleanup(self.ctxt, service_id,
claimed_worker)
self.assertEqual(1, res)
res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(claimed_worker, db_worker)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def test_worker_claim_fails_status_change(self):
"""Test that claim fails if the work entry has changed its status."""
worker = db.worker_create(self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting")
worker.status = "creating"
res = db.worker_claim_for_cleanup(self.ctxt, 1, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ["status"])
self.assertIsNone(db_worker.service_id)
def test_worker_claim(self):
"""Test worker claim of normal DB entry."""
service_id = 1
worker = db.worker_create(self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting")
res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ["updated_at"])
self.assertEqual(service_id, db_worker.service_id)
self.assertEqual(worker.service_id, db_worker.service_id)
def test_worker_claim_fails_status_change(self):
"""Test that claim fails if the work entry has changed its status."""
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='deleting')
worker.status = 'creating'
res = db.worker_claim_for_cleanup(self.ctxt, 1, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['status'])
self.assertIsNone(db_worker.service_id)
def test_worker_claim_fails_service_change(self):
"""Test that claim fails on worker service change."""
failed_service = 1
working_service = 2
this_service = 3
worker = db.worker_create(
self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting", service_id=working_service
)
worker.service_id = failed_service
res = db.worker_claim_for_cleanup(self.ctxt, this_service, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self.assertEqual(working_service, db_worker.service_id)
def test_worker_claim_fails_service_change(self):
"""Test that claim fails on worker service change."""
failed_service = 1
working_service = 2
this_service = 3
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='deleting',
service_id=working_service)
worker.service_id = failed_service
res = db.worker_claim_for_cleanup(self.ctxt, this_service, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self.assertEqual(working_service, db_worker.service_id)
def test_worker_claim(self):
"""Test worker claim of normal DB entry."""
service_id = 1
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='deleting')
res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
self.assertEqual(service_id, db_worker.service_id)
self.assertEqual(worker.service_id, db_worker.service_id)
def test_worker_claim_same_service(self):
"""Test worker claim of a DB entry that has our service_id."""
service_id = 1
worker = db.worker_create(
self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting", service_id=service_id
)
# Read from DB to get updated_at field
worker = db.worker_get(self.ctxt, id=worker.id)
claimed_worker = db.worker_get(self.ctxt, id=worker.id)
res = db.worker_claim_for_cleanup(self.ctxt, service_id, claimed_worker)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(claimed_worker, db_worker)
self._assertEqualObjects(worker, db_worker, ["updated_at"])
self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def test_worker_claim_same_service(self):
"""Test worker claim of a DB entry that has our service_id."""
service_id = 1
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='deleting',
service_id=service_id)
# Read from DB to get updated_at field
worker = db.worker_get(self.ctxt, id=worker.id)
claimed_worker = db.worker_get(self.ctxt, id=worker.id)
res = db.worker_claim_for_cleanup(self.ctxt, service_id,
claimed_worker)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(claimed_worker, db_worker)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def do_cleanup(self, context, cleanup_request):
LOG.info('Initiating service %s cleanup', cleanup_request.service_id)
# If the 'until' field in the cleanup request is not set, we default to
# this very moment.
until = cleanup_request.until or timeutils.utcnow()
keep_entry = False
to_clean = db.worker_get_all(
context,
resource_type=cleanup_request.resource_type,
resource_id=cleanup_request.resource_id,
service_id=cleanup_request.service_id,
until=until)
for clean in to_clean:
original_service_id = clean.service_id
original_time = clean.updated_at
# Try to do a soft delete to mark the entry as being cleaned up
# by us (setting service id to our service id).
res = db.worker_claim_for_cleanup(context,
claimer_id=self.service_id,
orm_worker=clean)
# Claim may fail if entry is being cleaned by another service, has
# been removed (finished cleaning) by another service or the user
# started a new cleanable operation.
# In any of these cases we don't have to do cleanup or remove the
# worker entry.
if not res:
continue
# Try to get versioned object for resource we have to cleanup
try:
vo_cls = getattr(objects, clean.resource_type)
vo = vo_cls.get_by_id(context, clean.resource_id)
# Set the worker DB entry in the VO and mark it as being a
# clean operation
clean.cleaning = True
vo.worker = clean
except exception.NotFound:
LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.',
{
'type': clean.resource_type,
'id': clean.resource_id
})
else:
# Resource status should match
if vo.status != clean.status:
LOG.debug(
'Skipping cleanup for mismatching work on '
'%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.', {
'type': clean.resource_type,
'id': clean.resource_id,
'exp_sts': clean.status,
'found_sts': vo.status
})
else:
LOG.info(
'Cleaning %(type)s with id %(id)s and status '
'%(status)s', {
'type': clean.resource_type,
'id': clean.resource_id,
'status': clean.status
},
resource=vo)
try:
# Some cleanup jobs are performed asynchronously, so
# we don't delete the worker entry, they'll take care
# of it
keep_entry = self._do_cleanup(context, vo)
except Exception:
LOG.exception('Could not perform cleanup.')
# Return the worker DB entry to the original service
db.worker_update(context,
clean.id,
service_id=original_service_id,
updated_at=original_time)
continue
# The resource either didn't exist or was properly cleaned, either
# way we can remove the entry from the worker table if the cleanup
# method doesn't want to keep the entry (for example for delayed
# deletion).
if not keep_entry and not db.worker_destroy(context, id=clean.id):
LOG.warning('Could not remove worker entry %s.', clean.id)
LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
def do_cleanup(self, context, cleanup_request):
LOG.info('Initiating service %s cleanup',
cleanup_request.service_id)
# If the 'until' field in the cleanup request is not set, we default to
# this very moment.
until = cleanup_request.until or timeutils.utcnow()
keep_entry = False
to_clean = db.worker_get_all(
context,
resource_type=cleanup_request.resource_type,
resource_id=cleanup_request.resource_id,
service_id=cleanup_request.service_id,
until=until)
for clean in to_clean:
original_service_id = clean.service_id
original_time = clean.updated_at
# Try to do a soft delete to mark the entry as being cleaned up
# by us (setting service id to our service id).
res = db.worker_claim_for_cleanup(context,
claimer_id=self.service_id,
orm_worker=clean)
# Claim may fail if entry is being cleaned by another service, has
# been removed (finished cleaning) by another service or the user
# started a new cleanable operation.
# In any of these cases we don't have to do cleanup or remove the
# worker entry.
if not res:
continue
# Try to get versioned object for resource we have to cleanup
try:
vo_cls = getattr(objects, clean.resource_type)
vo = vo_cls.get_by_id(context, clean.resource_id)
# Set the worker DB entry in the VO and mark it as being a
# clean operation
clean.cleaning = True
vo.worker = clean
except exception.NotFound:
LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.',
{'type': clean.resource_type,
'id': clean.resource_id})
else:
# Resource status should match
if vo.status != clean.status:
LOG.debug('Skipping cleanup for mismatching work on '
'%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.',
{'type': clean.resource_type,
'id': clean.resource_id,
'exp_sts': clean.status,
'found_sts': vo.status})
else:
LOG.info('Cleaning %(type)s with id %(id)s and status '
'%(status)s',
{'type': clean.resource_type,
'id': clean.resource_id,
'status': clean.status},
resource=vo)
try:
# Some cleanup jobs are performed asynchronously, so
# we don't delete the worker entry, they'll take care
# of it
keep_entry = self._do_cleanup(context, vo)
except Exception:
LOG.exception('Could not perform cleanup.')
# Return the worker DB entry to the original service
db.worker_update(context, clean.id,
service_id=original_service_id,
updated_at=original_time)
continue
# The resource either didn't exist or was properly cleaned, either
# way we can remove the entry from the worker table if the cleanup
# method doesn't want to keep the entry (for example for delayed
# deletion).
if not keep_entry and not db.worker_destroy(context, id=clean.id):
LOG.warning('Could not remove worker entry %s.', clean.id)
LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
