def test_do_cleanup_not_cleaning_already_claimed(self):
"""Basic cleanup that doesn't touch already cleaning works."""
vol = utils.create_volume(self.context, status='creating')
worker1 = db.worker_create(self.context, status='creating',
resource_type='Volume', resource_id=vol.id,
service_id=self.service.id)
worker1 = db.worker_get(self.context, id=worker1.id)
vol2 = utils.create_volume(self.context, status='deleting')
worker2 = db.worker_create(self.context, status='deleting',
resource_type='Volume', resource_id=vol2.id,
service_id=self.service.id + 1)
worker2 = db.worker_get(self.context, id=worker2.id)
# Simulate that the change to vol2 worker happened between
# worker_get_all and trying to claim a work for cleanup
worker2.service_id = self.service.id
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
with mock.patch('cinder.db.worker_get_all') as get_all_mock:
get_all_mock.return_value = [worker1, worker2]
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
self.assertEqual(worker2.id, workers[0].id)
vol.refresh()
self.assertEqual('creating_cleaned', vol.status)
vol2.refresh()
self.assertEqual('deleting', vol2.status)
def test_do_cleanup_not_cleaning_already_claimed(self):
"""Basic cleanup that doesn't touch already cleaning works."""
vol = utils.create_volume(self.context, status='creating')
worker1 = db.worker_create(self.context,
status='creating',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id)
worker1 = db.worker_get(self.context, id=worker1.id)
vol2 = utils.create_volume(self.context, status='deleting')
worker2 = db.worker_create(self.context,
status='deleting',
resource_type='Volume',
resource_id=vol2.id,
service_id=self.service.id + 1)
worker2 = db.worker_get(self.context, id=worker2.id)
# Simulate that the change to vol2 worker happened between
# worker_get_all and trying to claim a work for cleanup
worker2.service_id = self.service.id
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
with mock.patch('cinder.db.worker_get_all') as get_all_mock:
get_all_mock.return_value = [worker1, worker2]
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
self.assertEqual(worker2.id, workers[0].id)
vol.refresh()
self.assertEqual('creating_cleaned', vol.status)
vol2.refresh()
self.assertEqual('deleting', vol2.status)
def test_worker_claim_fails_this_service_claimed(self):
"""Test claim fails when worker was already claimed by this service."""
service_id = 1
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='creating',
service_id=service_id)
# Read it back to have the updated_at value
worker = db.worker_get(self.ctxt, id=worker.id)
claimed_worker = db.worker_get(self.ctxt, id=worker.id)
time.sleep(0.1)
# Simulate that this service starts processing this entry
res = db.worker_claim_for_cleanup(self.ctxt, service_id,
claimed_worker)
self.assertEqual(1, res)
res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(claimed_worker, db_worker)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def test_worker_claim_fails_this_service_claimed(self):
"""Test claim fails when worker was already claimed by this service."""
service_id = 1
worker = db.worker_create(self.ctxt, resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='creating',
service_id=service_id)
# Read it back to have the updated_at value
worker = db.worker_get(self.ctxt, id=worker.id)
claimed_worker = db.worker_get(self.ctxt, id=worker.id)
time.sleep(0.1)
# Simulate that this service starts processing this entry
res = db.worker_claim_for_cleanup(self.ctxt,
service_id,
claimed_worker)
self.assertEqual(1, res)
res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(claimed_worker, db_worker)
self._assertEqualObjects(worker, db_worker,
['updated_at', 'race_preventer'])
self.assertNotEqual(worker.updated_at, db_worker.updated_at)
self.assertEqual(worker.race_preventer + 1, db_worker.race_preventer)
def test_worker_update(self):
"""Test basic worker update."""
worker = self._create_workers(1)[0]
worker = db.worker_get(self.ctxt, id=worker.id)
res = db.worker_update(self.ctxt, worker.id, service_id=1)
self.assertEqual(1, res)
worker.service_id = 1
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
def test_worker_update(self):
"""Test basic worker update."""
worker = self._create_workers(1)[0]
worker = db.worker_get(self.ctxt, id=worker.id)
res = db.worker_update(self.ctxt, worker.id, service_id=1)
self.assertEqual(1, res)
worker.service_id = 1
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
def test_worker_update_no_subsecond(self):
"""Test basic worker update."""
db.sqlalchemy.api.DB_SUPPORTS_SUBSECOND_RESOLUTION = False
worker = self._create_workers(1)[0]
worker = db.worker_get(self.ctxt, id=worker.id)
now = datetime.utcnow().replace(microsecond=123)
with mock.patch("oslo_utils.timeutils.utcnow", return_value=now):
res = db.worker_update(self.ctxt, worker.id, service_id=1)
self.assertEqual(1, res)
worker.service_id = 1
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ["updated_at"])
self.assertEqual(0, db_worker.updated_at.microsecond)
def test_worker_update_no_subsecond(self):
"""Test basic worker update."""
db.sqlalchemy.api.DB_SUPPORTS_SUBSECOND_RESOLUTION = False
worker = self._create_workers(1)[0]
worker = db.worker_get(self.ctxt, id=worker.id)
now = datetime.utcnow().replace(microsecond=123)
with mock.patch('oslo_utils.timeutils.utcnow', return_value=now):
res = db.worker_update(self.ctxt, worker.id, service_id=1)
self.assertEqual(1, res)
worker.service_id = 1
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
self.assertEqual(0, db_worker.updated_at.microsecond)
def test_worker_create_no_subsecond(self, mock_utcnow):
"""Test basic creation of a worker record."""
db.sqlalchemy.api.DB_SUPPORTS_SUBSECOND_RESOLUTION = False
worker = db.worker_create(self.ctxt, **self.worker_fields)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker)
self.assertEqual(0, db_worker.updated_at.microsecond)
def test_worker_create_no_subsecond(self, mock_utcnow):
"""Test basic creation of a worker record."""
db.sqlalchemy.api.DB_SUPPORTS_SUBSECOND_RESOLUTION = False
worker = db.worker_create(self.ctxt, **self.worker_fields)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker)
self.assertEqual(0, db_worker.updated_at.microsecond)
def test_do_cleanup_not_cleaning_already_claimed_by_us(self):
"""Basic cleanup that doesn't touch other thread's claimed works."""
now = timeutils.utcnow()
delta = timeutils.datetime.timedelta(seconds=1)
original_time = now - delta
# Creating the worker in the future, and then changing the in-memory
# value of worker2.updated_at to an earlier time, we effectively
# simulate that the worker entry was created in the past and that it
# has been just updated between worker_get_all and trying
# to claim a work for cleanup
other_thread_claimed_time = now + delta
vol = utils.create_volume(self.context, status='creating')
worker1 = db.worker_create(self.context,
status='creating',
resource_type='Volume',
resource_id=vol.id,
service_id=self.service.id,
updated_at=original_time)
worker1 = db.worker_get(self.context, id=worker1.id)
vol2 = utils.create_volume(self.context, status='deleting')
worker2 = db.worker_create(self.context,
status='deleting',
resource_type='Volume',
resource_id=vol2.id,
service_id=self.service.id,
updated_at=other_thread_claimed_time)
worker2 = db.worker_get(self.context, id=worker2.id)
# This with the mock below simulates worker2 was created in the past
# and updated right between worker_get_all and worker_claim_for_cleanup
worker2.updated_at = original_time
clean_req = objects.CleanupRequest(service_id=self.service.id)
mngr = FakeManager(self.service.id)
with mock.patch('cinder.manager.timeutils.utcnow', return_value=now),\
mock.patch('cinder.db.worker_get_all') as get_all_mock:
get_all_mock.return_value = [worker1, worker2]
mngr.do_cleanup(self.context, clean_req)
workers = db.worker_get_all(self.context)
self.assertEqual(1, len(workers))
self.assertEqual(worker2.id, workers[0].id)
vol.refresh()
self.assertEqual('creating_cleaned', vol.status)
vol2.refresh()
self.assertEqual('deleting', vol2.status)
def test_worker_update_update_orm(self):
"""Test worker update updating the worker orm object."""
worker = self._create_workers(1)[0]
res = db.worker_update(self.ctxt, worker.id, orm_worker=worker,
service_id=1)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
def test_worker_claim_same_service(self):
"""Test worker claim of a DB entry that has our service_id."""
service_id = 1
worker = db.worker_create(
self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting", service_id=service_id
)
# Read from DB to get updated_at field
worker = db.worker_get(self.ctxt, id=worker.id)
claimed_worker = db.worker_get(self.ctxt, id=worker.id)
res = db.worker_claim_for_cleanup(self.ctxt, service_id, claimed_worker)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(claimed_worker, db_worker)
self._assertEqualObjects(worker, db_worker, ["updated_at"])
self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def test_worker_update_update_orm(self):
"""Test worker update updating the worker orm object."""
worker = self._create_workers(1)[0]
res = db.worker_update(self.ctxt, worker.id, orm_worker=worker, service_id=1)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
# If we are updating the ORM object we don't ignore the update_at field
# because it will get updated in the ORM instance.
self._assertEqualObjects(worker, db_worker)
def test_worker_claim_fails_status_change(self):
"""Test that claim fails if the work entry has changed its status."""
worker = db.worker_create(self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting")
worker.status = "creating"
res = db.worker_claim_for_cleanup(self.ctxt, 1, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ["status"])
self.assertIsNone(db_worker.service_id)
def test_worker_update_update_orm(self):
"""Test worker update updating the worker orm object."""
worker = self._create_workers(1)[0]
res = db.worker_update(self.ctxt,
worker.id,
orm_worker=worker,
service_id=1)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
def test_worker_claim_same_service(self):
"""Test worker claim of a DB entry that has our service_id."""
service_id = 1
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='deleting',
service_id=service_id)
# Read from DB to get updated_at field
worker = db.worker_get(self.ctxt, id=worker.id)
claimed_worker = db.worker_get(self.ctxt, id=worker.id)
res = db.worker_claim_for_cleanup(self.ctxt, service_id,
claimed_worker)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(claimed_worker, db_worker)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
self.assertNotEqual(worker.updated_at, db_worker.updated_at)
def test_worker_claim(self):
"""Test worker claim of normal DB entry."""
service_id = 1
worker = db.worker_create(self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting")
res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ["updated_at"])
self.assertEqual(service_id, db_worker.service_id)
self.assertEqual(worker.service_id, db_worker.service_id)
def test_worker_update_update_orm(self):
"""Test worker update updating the worker orm object."""
worker = self._create_workers(1)[0]
res = db.worker_update(self.ctxt,
worker.id,
orm_worker=worker,
service_id=1)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
# If we are updating the ORM object we don't ignore the update_at field
# because it will get updated in the ORM instance.
self._assertEqualObjects(worker, db_worker)
def set_worker(self):
worker = self.worker
service_id = service.Service.service_id
resource_type = self.__class__.__name__
if worker:
if worker.cleaning:
return
else:
try:
worker = db.worker_get(self._context,
resource_type=resource_type,
resource_id=self.id)
except exception.WorkerNotFound:
# If the call didn't come from an RPC call we still have to
# create the entry in the DB.
try:
self.worker = db.worker_create(self._context,
status=self.status,
resource_type=resource_type,
resource_id=self.id,
service_id=service_id)
return
except exception.WorkerExists:
# If 2 cleanable operations are competing for this resource
# and the other one created the entry first that one won
raise exception.CleanableInUse(type=resource_type,
id=self.id)
# If we have to claim this work or if the status has changed we have
# to update DB.
if (worker.service_id != service_id or worker.status != self.status):
try:
db.worker_update(self._context,
worker.id,
filters={
'service_id': worker.service_id,
'status': worker.status,
'race_preventer': worker.race_preventer,
'updated_at': worker.updated_at
},
service_id=service_id,
status=self.status,
orm_worker=worker)
except exception.WorkerNotFound:
self.worker = None
raise exception.CleanableInUse(type=self.__class__.__name__,
id=self.id)
self.worker = worker
def test_worker_claim_fails_status_change(self):
"""Test that claim fails if the work entry has changed its status."""
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='deleting')
worker.status = 'creating'
res = db.worker_claim_for_cleanup(self.ctxt, 1, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['status'])
self.assertIsNone(db_worker.service_id)
def test_worker_claim_fails_service_change(self):
"""Test that claim fails on worker service change."""
failed_service = 1
working_service = 2
this_service = 3
worker = db.worker_create(
self.ctxt, resource_type="Volume", resource_id=fake.VOLUME_ID, status="deleting", service_id=working_service
)
worker.service_id = failed_service
res = db.worker_claim_for_cleanup(self.ctxt, this_service, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self.assertEqual(working_service, db_worker.service_id)
def _create_workers(self, num, read_back=False, **fields):
workers = []
base_params = self.worker_fields.copy()
base_params.update(fields)
for i in range(num):
params = base_params.copy()
params['resource_id'] = self._uuid()
workers.append(db.worker_create(self.ctxt, **params))
if read_back:
for i in range(len(workers)):
workers[i] = db.worker_get(self.ctxt, id=workers[i].id)
return workers
def _create_workers(self, num, read_back=False, **fields):
workers = []
base_params = self.worker_fields.copy()
base_params.update(fields)
for i in range(num):
params = base_params.copy()
params['resource_id'] = self._uuid()
workers.append(db.worker_create(self.ctxt, **params))
if read_back:
for i in range(len(workers)):
workers[i] = db.worker_get(self.ctxt, id=workers[i].id)
return workers
def set_worker(self):
worker = self.worker
service_id = service.Service.service_id
resource_type = self.__class__.__name__
if worker:
if worker.cleaning:
return
else:
try:
worker = db.worker_get(self._context,
resource_type=resource_type,
resource_id=self.id)
except exception.WorkerNotFound:
# If the call didn't come from an RPC call we still have to
# create the entry in the DB.
try:
self.worker = db.worker_create(self._context,
status=self.status,
resource_type=resource_type,
resource_id=self.id,
service_id=service_id)
return
except exception.WorkerExists:
# If 2 cleanable operations are competing for this resource
# and the other one created the entry first that one won
raise exception.CleanableInUse(type=resource_type,
id=self.id)
# If we have to claim this work or if the status has changed we have
# to update DB.
if (worker.service_id != service_id or worker.status != self.status):
try:
db.worker_update(
self._context, worker.id,
filters={'service_id': worker.service_id,
'status': worker.status,
'race_preventer': worker.race_preventer,
'updated_at': worker.updated_at},
service_id=service_id,
status=self.status,
orm_worker=worker)
except exception.WorkerNotFound:
self.worker = None
raise exception.CleanableInUse(type=self.__class__.__name__,
id=self.id)
self.worker = worker
def test_worker_claim_fails_service_change(self):
"""Test that claim fails on worker service change."""
failed_service = 1
working_service = 2
this_service = 3
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='deleting',
service_id=working_service)
worker.service_id = failed_service
res = db.worker_claim_for_cleanup(self.ctxt, this_service, worker)
self.assertEqual(0, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self.assertEqual(working_service, db_worker.service_id)
def test_worker_claim(self):
"""Test worker claim of normal DB entry."""
service_id = 1
worker = db.worker_create(self.ctxt,
resource_type='Volume',
resource_id=fake.VOLUME_ID,
status='deleting')
res = db.worker_claim_for_cleanup(self.ctxt, service_id, worker)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
self.assertEqual(service_id, db_worker.service_id)
self.assertEqual(worker.service_id, db_worker.service_id)
def test_worker_create_and_get(self):
"""Test basic creation of a worker record."""
worker = db.worker_create(self.ctxt, **self.worker_fields)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker)
def test_worker_create_and_get(self):
"""Test basic creation of a worker record."""
worker = db.worker_create(self.ctxt, **self.worker_fields)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker)
