def test_do_cleanup_not_cleaning_already_claimed(self):
"""Basic cleanup that doesn't touch already cleaning works."""
vol = utils.create_volume(self.context, status='creating')
worker1 = db.worker_create(self.context, status='creating',
resource_type='Volume', resource_id=vol.id,
service_id=self.service.id)
worker1 = db.worker_get(self.context, id=worker1.id)
vol2 = utils.create_volume(self.context, status='deleting')
worker2 = db.worker_create(self.context, status='deleting',
resource_type='Volume', resource_id=vol2.id,
service_id=self.service.id + 1)
worker2 = db.worker_get(self.context, id=worker2.id)
# Simulate that the change to vol2 worker happened between
# worker_get_all and trying to claim a work for cleanup
worker2.service_id = self.service.id
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
with mock.patch('cinder.db.worker_get_all') as get_all_mock:
get_all_mock.return_value = [worker1, worker2]
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
self.assertEqual(worker2.id, workers[0].id)
vol.refresh()
self.assertEqual('creating_cleaned', vol.status)
vol2.refresh()
self.assertEqual('deleting', vol2.status)
def _prepare_params(self, ctxt, params, allowed):
if not allowed.issuperset(params):
invalid_keys = set(params).difference(allowed)
msg = _('Invalid filter keys: %s') % ', '.join(invalid_keys)
raise exception.InvalidInput(reason=msg)
if params.get('binary') not in (None, 'cinder-volume',
'cinder-scheduler'):
msg = _('binary must be empty or set to cinder-volume or '
'cinder-scheduler')
raise exception.InvalidInput(reason=msg)
for boolean in ('disabled', 'is_up'):
if params.get(boolean) is not None:
params[boolean] = utils.get_bool_param(boolean, params)
resource_type = params.get('resource_type')
if resource_type:
resource_type = resource_type.title()
types = cleanable.CinderCleanableObject.cleanable_resource_types
if resource_type not in types:
msg = (_('Resource type %s not valid, must be ') %
resource_type)
msg = utils.build_or_str(types, msg + '%s.')
raise exception.InvalidInput(reason=msg)
params['resource_type'] = resource_type
resource_id = params.get('resource_id')
if resource_id:
if not uuidutils.is_uuid_like(resource_id):
msg = (_('Resource ID must be a UUID, and %s is not.') %
resource_id)
raise exception.InvalidInput(reason=msg)
# If we have the resource type but we don't have where it is
# located, we get it from the DB to limit the distribution of the
# request by the scheduler, otherwise it will be distributed to all
# the services.
location_keys = {'service_id', 'cluster_name', 'host'}
if not location_keys.intersection(params):
workers = db.worker_get_all(ctxt, resource_id=resource_id,
binary=params.get('binary'),
resource_type=resource_type)
if len(workers) == 0:
msg = (_('There is no resource with UUID %s pending '
'cleanup.'), resource_id)
raise exception.InvalidInput(reason=msg)
if len(workers) > 1:
msg = (_('There are multiple resources with UUID %s '
'pending cleanup. Please be more specific.'),
resource_id)
raise exception.InvalidInput(reason=msg)
worker = workers[0]
params.update(service_id=worker.service_id,
resource_type=worker.resource_type)
return params
def test_do_cleanup_not_cleaning_already_claimed(self):
"""Basic cleanup that doesn't touch already cleaning works."""
vol = utils.create_volume(self.context, status='creating')
worker1 = db.worker_create(self.context,
status='creating',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id)
worker1 = db.worker_get(self.context, id=worker1.id)
vol2 = utils.create_volume(self.context, status='deleting')
worker2 = db.worker_create(self.context,
status='deleting',
resource_type='Volume',
resource_id=vol2.id,
service_id=self.service.id + 1)
worker2 = db.worker_get(self.context, id=worker2.id)
# Simulate that the change to vol2 worker happened between
# worker_get_all and trying to claim a work for cleanup
worker2.service_id = self.service.id
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
with mock.patch('cinder.db.worker_get_all') as get_all_mock:
get_all_mock.return_value = [worker1, worker2]
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
self.assertEqual(worker2.id, workers[0].id)
vol.refresh()
self.assertEqual('creating_cleaned', vol.status)
vol2.refresh()
self.assertEqual('deleting', vol2.status)
def test_worker_destroy(self):
"""Test that worker destroy really deletes the DB entry."""
worker = self._create_workers(1)[0]
res = db.worker_destroy(self.ctxt, id=worker.id)
self.assertEqual(1, res)
db_workers = db.worker_get_all(self.ctxt, read_deleted='yes')
self.assertListEqual([], db_workers)
def test_worker_destroy(self):
"""Test that worker destroy really deletes the DB entry."""
worker = self._create_workers(1)[0]
res = db.worker_destroy(self.ctxt, id=worker.id)
self.assertEqual(1, res)
db_workers = db.worker_get_all(self.ctxt, read_deleted='yes')
self.assertListEqual([], db_workers)
def test_worker_get_all(self):
"""Test basic get_all method."""
self._create_workers(1)
service = db.service_create(self.ctxt, {})
workers = self._create_workers(3, service_id=service.id)
db_workers = db.worker_get_all(self.ctxt, service_id=service.id)
self._assertEqualListsOfObjects(workers, db_workers)
def test_worker_get_all(self):
"""Test basic get_all method."""
self._create_workers(1)
service = db.service_create(self.ctxt, {})
workers = self._create_workers(3, service_id=service.id)
db_workers = db.worker_get_all(self.ctxt, service_id=service.id)
self._assertEqualListsOfObjects(workers, db_workers)
def test_worker_get_all_until(self):
"""Test get_all until a specific time."""
workers = self._create_workers(3, read_back=True)
timestamp = workers[-1].updated_at
time.sleep(0.1)
self._create_workers(3)
db_workers = db.worker_get_all(self.ctxt, until=timestamp)
self._assertEqualListsOfObjects(workers, db_workers)
def test_worker_get_all_until(self):
"""Test get_all until a specific time."""
workers = self._create_workers(3, read_back=True)
timestamp = workers[-1].updated_at
time.sleep(0.1)
self._create_workers(3)
db_workers = db.worker_get_all(self.ctxt, until=timestamp)
self._assertEqualListsOfObjects(workers, db_workers)
def test_do_cleanup_resource_deleted(self):
"""Cleanup on a resource that's been already deleted."""
vol = utils.create_volume(self.context, status='creating')
db.worker_create(self.context, status='creating',
resource_type='Volume', resource_id=vol.id,
service_id=self.service.id)
vol.destroy()
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertListEqual([], workers)
def test_do_cleanup(self):
"""Basic successful cleanup."""
vol = utils.create_volume(self.context, status='creating')
db.worker_create(self.context, status='creating',
resource_type='Volume', resource_id=vol.id,
service_id=self.service.id)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
self.assertListEqual([], db.worker_get_all(self.context))
vol.refresh()
self.assertEqual('creating_cleaned', vol.status)
def test_do_cleanup_revive_on_cleanup_fail(self, mock_clean):
"""Cleanup will revive a worker if cleanup fails."""
vol = utils.create_volume(self.context, status='creating')
db.worker_create(self.context, status='creating',
resource_type='Volume', resource_id=vol.id,
service_id=self.service.id)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
vol.refresh()
self.assertEqual('creating', vol.status)
def test_do_cleanup_keep_worker(self):
"""Cleanup on a resource that will remove worker when cleaning up."""
vol = utils.create_volume(self.context, status='deleting')
db.worker_create(self.context, status='deleting',
resource_type='Volume', resource_id=vol.id,
service_id=self.service.id)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id, keep_after_clean=True)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
vol.refresh()
self.assertEqual('deleting_cleaned', vol.status)
def test_do_cleanup(self):
"""Basic successful cleanup."""
vol = utils.create_volume(self.context, status='creating')
db.worker_create(self.context,
status='creating',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
self.assertListEqual([], db.worker_get_all(self.context))
vol.refresh()
self.assertEqual('creating_cleaned', vol.status)
def test_do_cleanup_resource_deleted(self):
"""Cleanup on a resource that's been already deleted."""
vol = utils.create_volume(self.context, status='creating')
db.worker_create(self.context,
status='creating',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id)
vol.destroy()
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertListEqual([], workers)
def test_do_cleanup_resource_on_another_service(self):
"""Cleanup on a resource that's been claimed by other service."""
vol = utils.create_volume(self.context, status='deleting')
db.worker_create(self.context, status='deleting',
resource_type='Volume', resource_id=vol.id,
service_id=self.service.id + 1)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
vol.refresh()
self.assertEqual('deleting', vol.status)
def test_do_cleanup_resource_changed_status(self):
"""Cleanup on a resource that's changed status."""
vol = utils.create_volume(self.context, status='available')
db.worker_create(self.context, status='creating',
resource_type='Volume', resource_id=vol.id,
service_id=self.service.id)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertListEqual([], workers)
vol.refresh()
self.assertEqual('available', vol.status)
def test_do_cleanup_revive_on_cleanup_fail(self, mock_clean):
"""Cleanup will revive a worker if cleanup fails."""
vol = utils.create_volume(self.context, status='creating')
db.worker_create(self.context,
status='creating',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
vol.refresh()
self.assertEqual('creating', vol.status)
def test_do_cleanup_not_cleaning_already_claimed_by_us(self):
"""Basic cleanup that doesn't touch other thread's claimed works."""
now = timeutils.utcnow()
delta = timeutils.datetime.timedelta(seconds=1)
original_time = now - delta
# Creating the worker in the future, and then changing the in-memory
# value of worker2.updated_at to an earlier time, we effectively
# simulate that the worker entry was created in the past and that it
# has been just updated between worker_get_all and trying
# to claim a work for cleanup
other_thread_claimed_time = now + delta
vol = utils.create_volume(self.context, status='creating')
worker1 = db.worker_create(self.context,
status='creating',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id,
updated_at=original_time)
worker1 = db.worker_get(self.context, id=worker1.id)
vol2 = utils.create_volume(self.context, status='deleting')
worker2 = db.worker_create(self.context,
status='deleting',
resource_type='Volume',
resource_id=vol2.id,
service_id=self.service.id,
updated_at=other_thread_claimed_time)
worker2 = db.worker_get(self.context, id=worker2.id)
# This with the mock below simulates worker2 was created in the past
# and updated right between worker_get_all and worker_claim_for_cleanup
worker2.updated_at = original_time
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
with mock.patch('cinder.manager.timeutils.utcnow', return_value=now),\
mock.patch('cinder.db.worker_get_all') as get_all_mock:
get_all_mock.return_value = [worker1, worker2]
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
self.assertEqual(worker2.id, workers[0].id)
vol.refresh()
self.assertEqual('creating_cleaned', vol.status)
vol2.refresh()
self.assertEqual('deleting', vol2.status)
def test_do_cleanup_keep_worker(self):
"""Cleanup on a resource that will remove worker when cleaning up."""
vol = utils.create_volume(self.context, status='deleting')
db.worker_create(self.context,
status='deleting',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id, keep_after_clean=True)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
vol.refresh()
self.assertEqual('deleting_cleaned', vol.status)
def test_do_cleanup_resource_changed_status(self):
"""Cleanup on a resource that's changed status."""
vol = utils.create_volume(self.context, status='available')
db.worker_create(self.context,
status='creating',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertListEqual([], workers)
vol.refresh()
self.assertEqual('available', vol.status)
def test_do_cleanup_resource_on_another_service(self):
"""Cleanup on a resource that's been claimed by other service."""
vol = utils.create_volume(self.context, status='deleting')
db.worker_create(self.context,
status='deleting',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id + 1)
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
vol.refresh()
self.assertEqual('deleting', vol.status)
def test_worker_get_all_returns_empty(self):
"""Test that get_all returns an empty list when there's no results."""
self._create_workers(3, deleted=True)
db_workers = db.worker_get_all(self.ctxt)
self.assertListEqual([], db_workers)
def cleanup(self, req, body=None):
"""Do the cleanup on resources from a specific service/host/node."""
# Let the wsgi middleware convert NotAuthorized exceptions
ctxt = req.environ['cinder.context']
ctxt.authorize(policy.CLEAN_POLICY)
body = body or {}
for boolean in ('disabled', 'is_up'):
if body.get(boolean) is not None:
body[boolean] = strutils.bool_from_string(body[boolean])
resource_type = body.get('resource_type')
if resource_type:
resource_type = resource_type.title()
types = cleanable.CinderCleanableObject.cleanable_resource_types
if resource_type not in types:
valid_types = utils.build_or_str(types)
msg = _('Resource type %(resource_type)s not valid,'
' must be %(valid_types)s')
msg = msg % {"resource_type": resource_type,
"valid_types": valid_types}
raise exception.InvalidInput(reason=msg)
body['resource_type'] = resource_type
resource_id = body.get('resource_id')
if resource_id:
# If we have the resource type but we don't have where it is
# located, we get it from the DB to limit the distribution of the
# request by the scheduler, otherwise it will be distributed to all
# the services.
location_keys = {'service_id', 'cluster_name', 'host'}
if not location_keys.intersection(body):
workers = db.worker_get_all(ctxt, resource_id=resource_id,
binary=body.get('binary'),
resource_type=resource_type)
if len(workers) == 0:
msg = (_('There is no resource with UUID %s pending '
'cleanup.'), resource_id)
raise exception.InvalidInput(reason=msg)
if len(workers) > 1:
msg = (_('There are multiple resources with UUID %s '
'pending cleanup. Please be more specific.'),
resource_id)
raise exception.InvalidInput(reason=msg)
worker = workers[0]
body.update(service_id=worker.service_id,
resource_type=worker.resource_type)
body['until'] = timeutils.utcnow()
# NOTE(geguileo): If is_up is not specified in the request
# CleanupRequest's default will be used (False)
cleanup_request = objects.CleanupRequest(**body)
cleaning, unavailable = self.sch_api.work_cleanup(ctxt,
cleanup_request)
return {
'cleaning': workers_view.ViewBuilder.service_list(cleaning),
'unavailable': workers_view.ViewBuilder.service_list(unavailable),
}
def do_cleanup(self, context, cleanup_request):
LOG.info('Initiating service %s cleanup',
cleanup_request.service_id)
# If the 'until' field in the cleanup request is not set, we default to
# this very moment.
until = cleanup_request.until or timeutils.utcnow()
keep_entry = False
to_clean = db.worker_get_all(
context,
resource_type=cleanup_request.resource_type,
resource_id=cleanup_request.resource_id,
service_id=cleanup_request.service_id,
until=until)
for clean in to_clean:
original_service_id = clean.service_id
original_time = clean.updated_at
# Try to do a soft delete to mark the entry as being cleaned up
# by us (setting service id to our service id).
res = db.worker_claim_for_cleanup(context,
claimer_id=self.service_id,
orm_worker=clean)
# Claim may fail if entry is being cleaned by another service, has
# been removed (finished cleaning) by another service or the user
# started a new cleanable operation.
# In any of these cases we don't have to do cleanup or remove the
# worker entry.
if not res:
continue
# Try to get versioned object for resource we have to cleanup
try:
vo_cls = getattr(objects, clean.resource_type)
vo = vo_cls.get_by_id(context, clean.resource_id)
# Set the worker DB entry in the VO and mark it as being a
# clean operation
clean.cleaning = True
vo.worker = clean
except exception.NotFound:
LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.',
{'type': clean.resource_type,
'id': clean.resource_id})
else:
# Resource status should match
if vo.status != clean.status:
LOG.debug('Skipping cleanup for mismatching work on '
'%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.',
{'type': clean.resource_type,
'id': clean.resource_id,
'exp_sts': clean.status,
'found_sts': vo.status})
else:
LOG.info('Cleaning %(type)s with id %(id)s and status '
'%(status)s',
{'type': clean.resource_type,
'id': clean.resource_id,
'status': clean.status},
resource=vo)
try:
# Some cleanup jobs are performed asynchronously, so
# we don't delete the worker entry, they'll take care
# of it
keep_entry = self._do_cleanup(context, vo)
except Exception:
LOG.exception('Could not perform cleanup.')
# Return the worker DB entry to the original service
db.worker_update(context, clean.id,
service_id=original_service_id,
updated_at=original_time)
continue
# The resource either didn't exist or was properly cleaned, either
# way we can remove the entry from the worker table if the cleanup
# method doesn't want to keep the entry (for example for delayed
# deletion).
if not keep_entry and not db.worker_destroy(context, id=clean.id):
LOG.warning('Could not remove worker entry %s.', clean.id)
LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
def test_worker_get_all_returns_empty(self):
"""Test that get_all returns an empty list when there's no results."""
self._create_workers(3, deleted=True)
db_workers = db.worker_get_all(self.ctxt)
self.assertListEqual([], db_workers)
def cleanup(self, req, body=None):
"""Do the cleanup on resources from a specific service/host/node."""
# Let the wsgi middleware convert NotAuthorized exceptions
ctxt = req.environ['cinder.context']
ctxt.authorize(policy.CLEAN_POLICY)
body = body or {}
for boolean in ('disabled', 'is_up'):
if body.get(boolean) is not None:
body[boolean] = strutils.bool_from_string(body[boolean])
resource_type = body.get('resource_type')
if resource_type:
resource_type = resource_type.title()
types = cleanable.CinderCleanableObject.cleanable_resource_types
if resource_type not in types:
valid_types = utils.build_or_str(types)
msg = _('Resource type %(resource_type)s not valid,'
' must be %(valid_types)s')
msg = msg % {
"resource_type": resource_type,
"valid_types": valid_types
}
raise exception.InvalidInput(reason=msg)
body['resource_type'] = resource_type
resource_id = body.get('resource_id')
if resource_id:
# If we have the resource type but we don't have where it is
# located, we get it from the DB to limit the distribution of the
# request by the scheduler, otherwise it will be distributed to all
# the services.
location_keys = {'service_id', 'cluster_name', 'host'}
if not location_keys.intersection(body):
workers = db.worker_get_all(ctxt,
resource_id=resource_id,
binary=body.get('binary'),
resource_type=resource_type)
if len(workers) == 0:
msg = (_('There is no resource with UUID %s pending '
'cleanup.'), resource_id)
raise exception.InvalidInput(reason=msg)
if len(workers) > 1:
msg = (_('There are multiple resources with UUID %s '
'pending cleanup. Please be more specific.'),
resource_id)
raise exception.InvalidInput(reason=msg)
worker = workers[0]
body.update(service_id=worker.service_id,
resource_type=worker.resource_type)
body['until'] = timeutils.utcnow()
# NOTE(geguileo): If is_up is not specified in the request
# CleanupRequest's default will be used (False)
cleanup_request = objects.CleanupRequest(**body)
cleaning, unavailable = self.sch_api.work_cleanup(
ctxt, cleanup_request)
return {
'cleaning': workers_view.ViewBuilder.service_list(cleaning),
'unavailable': workers_view.ViewBuilder.service_list(unavailable),
}
def _assert_workers_are_removed(self):
workers = db.worker_get_all(self.context, read_deleted='yes')
self.assertListEqual([], workers)
def do_cleanup(self, context, cleanup_request):
LOG.info('Initiating service %s cleanup', cleanup_request.service_id)
# If the 'until' field in the cleanup request is not set, we default to
# this very moment.
until = cleanup_request.until or timeutils.utcnow()
keep_entry = False
to_clean = db.worker_get_all(
context,
resource_type=cleanup_request.resource_type,
resource_id=cleanup_request.resource_id,
service_id=cleanup_request.service_id,
until=until)
for clean in to_clean:
original_service_id = clean.service_id
original_time = clean.updated_at
# Try to do a soft delete to mark the entry as being cleaned up
# by us (setting service id to our service id).
res = db.worker_claim_for_cleanup(context,
claimer_id=self.service_id,
orm_worker=clean)
# Claim may fail if entry is being cleaned by another service, has
# been removed (finished cleaning) by another service or the user
# started a new cleanable operation.
# In any of these cases we don't have to do cleanup or remove the
# worker entry.
if not res:
continue
# Try to get versioned object for resource we have to cleanup
try:
vo_cls = getattr(objects, clean.resource_type)
vo = vo_cls.get_by_id(context, clean.resource_id)
# Set the worker DB entry in the VO and mark it as being a
# clean operation
clean.cleaning = True
vo.worker = clean
except exception.NotFound:
LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.',
{
'type': clean.resource_type,
'id': clean.resource_id
})
else:
# Resource status should match
if vo.status != clean.status:
LOG.debug(
'Skipping cleanup for mismatching work on '
'%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.', {
'type': clean.resource_type,
'id': clean.resource_id,
'exp_sts': clean.status,
'found_sts': vo.status
})
else:
LOG.info(
'Cleaning %(type)s with id %(id)s and status '
'%(status)s', {
'type': clean.resource_type,
'id': clean.resource_id,
'status': clean.status
},
resource=vo)
try:
# Some cleanup jobs are performed asynchronously, so
# we don't delete the worker entry, they'll take care
# of it
keep_entry = self._do_cleanup(context, vo)
except Exception:
LOG.exception('Could not perform cleanup.')
# Return the worker DB entry to the original service
db.worker_update(context,
clean.id,
service_id=original_service_id,
updated_at=original_time)
continue
# The resource either didn't exist or was properly cleaned, either
# way we can remove the entry from the worker table if the cleanup
# method doesn't want to keep the entry (for example for delayed
# deletion).
if not keep_entry and not db.worker_destroy(context, id=clean.id):
LOG.warning('Could not remove worker entry %s.', clean.id)
LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
def _assert_workers_are_removed(self):
workers = db.worker_get_all(self.context, read_deleted='yes')
self.assertListEqual([], workers)
