def create_worker(self, pinned=True):
"""Create a worker entry at the API."""
# This method is mostly called from the rpc layer, therefore it checks
# if it's cleanable given current pinned version.
if not self.is_cleanable(pinned):
return False
resource_type = self.__class__.__name__
entry_in_db = False
# This will only loop on very rare race conditions
while not entry_in_db:
try:
# On the common case there won't be an entry in the DB, that's
# why we try to create first.
db.worker_create(self._context,
status=self.status,
resource_type=resource_type,
resource_id=self.id)
entry_in_db = True
except exception.WorkerExists:
try:
db.worker_update(self._context,
None,
filters={
'resource_type': resource_type,
'resource_id': self.id
},
service_id=None,
status=self.status)
entry_in_db = True
except exception.WorkerNotFound:
pass
return entry_in_db
def create_worker(self, pinned=True):
"""Create a worker entry at the API."""
# This method is mostly called from the rpc layer, therefore it checks
# if it's cleanable given current pinned version.
if not self.is_cleanable(pinned):
return False
resource_type = self.__class__.__name__
entry_in_db = False
# This will only loop on very rare race conditions
while not entry_in_db:
try:
# On the common case there won't be an entry in the DB, that's
# why we try to create first.
db.worker_create(self._context, status=self.status,
resource_type=resource_type,
resource_id=self.id)
entry_in_db = True
except exception.WorkerExists:
try:
db.worker_update(self._context, None,
filters={'resource_type': resource_type,
'resource_id': self.id},
service_id=None,
status=self.status)
entry_in_db = True
except exception.WorkerNotFound:
pass
return entry_in_db
def test_workers_init_not_supported(self):
# Fake a Db that doesn't support sub-second resolution in datetimes
db.worker_update(
self.ctxt, None,
{'resource_type': 'SENTINEL', 'ignore_sentinel': False},
updated_at=datetime.utcnow().replace(microsecond=0))
db.workers_init()
self.assertFalse(db.sqlalchemy.api.DB_SUPPORTS_SUBSECOND_RESOLUTION)
def test_workers_init_not_supported(self):
# Fake a Db that doesn't support sub-second resolution in datetimes
db.worker_update(self.ctxt,
None, {
'resource_type': 'SENTINEL',
'ignore_sentinel': False
},
updated_at=datetime.utcnow().replace(microsecond=0))
db.workers_init()
self.assertFalse(db.sqlalchemy.api.DB_SUPPORTS_SUBSECOND_RESOLUTION)
def set_worker(self):
worker = self.worker
service_id = service.Service.service_id
resource_type = self.__class__.__name__
if worker:
if worker.cleaning:
return
else:
try:
worker = db.worker_get(self._context,
resource_type=resource_type,
resource_id=self.id)
except exception.WorkerNotFound:
# If the call didn't come from an RPC call we still have to
# create the entry in the DB.
try:
self.worker = db.worker_create(self._context,
status=self.status,
resource_type=resource_type,
resource_id=self.id,
service_id=service_id)
return
except exception.WorkerExists:
# If 2 cleanable operations are competing for this resource
# and the other one created the entry first that one won
raise exception.CleanableInUse(type=resource_type,
id=self.id)
# If we have to claim this work or if the status has changed we have
# to update DB.
if (worker.service_id != service_id or worker.status != self.status):
try:
db.worker_update(self._context,
worker.id,
filters={
'service_id': worker.service_id,
'status': worker.status,
'race_preventer': worker.race_preventer,
'updated_at': worker.updated_at
},
service_id=service_id,
status=self.status,
orm_worker=worker)
except exception.WorkerNotFound:
self.worker = None
raise exception.CleanableInUse(type=self.__class__.__name__,
id=self.id)
self.worker = worker
def set_worker(self):
worker = self.worker
service_id = service.Service.service_id
resource_type = self.__class__.__name__
if worker:
if worker.cleaning:
return
else:
try:
worker = db.worker_get(self._context,
resource_type=resource_type,
resource_id=self.id)
except exception.WorkerNotFound:
# If the call didn't come from an RPC call we still have to
# create the entry in the DB.
try:
self.worker = db.worker_create(self._context,
status=self.status,
resource_type=resource_type,
resource_id=self.id,
service_id=service_id)
return
except exception.WorkerExists:
# If 2 cleanable operations are competing for this resource
# and the other one created the entry first that one won
raise exception.CleanableInUse(type=resource_type,
id=self.id)
# If we have to claim this work or if the status has changed we have
# to update DB.
if (worker.service_id != service_id or worker.status != self.status):
try:
db.worker_update(
self._context, worker.id,
filters={'service_id': worker.service_id,
'status': worker.status,
'race_preventer': worker.race_preventer,
'updated_at': worker.updated_at},
service_id=service_id,
status=self.status,
orm_worker=worker)
except exception.WorkerNotFound:
self.worker = None
raise exception.CleanableInUse(type=self.__class__.__name__,
id=self.id)
self.worker = worker
def test_worker_update_update_orm(self):
"""Test worker update updating the worker orm object."""
worker = self._create_workers(1)[0]
res = db.worker_update(self.ctxt, worker.id, orm_worker=worker,
service_id=1)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
def test_worker_update(self):
"""Test basic worker update."""
worker = self._create_workers(1)[0]
worker = db.worker_get(self.ctxt, id=worker.id)
res = db.worker_update(self.ctxt, worker.id, service_id=1)
self.assertEqual(1, res)
worker.service_id = 1
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
def test_worker_update(self):
"""Test basic worker update."""
worker = self._create_workers(1)[0]
worker = db.worker_get(self.ctxt, id=worker.id)
res = db.worker_update(self.ctxt, worker.id, service_id=1)
self.assertEqual(1, res)
worker.service_id = 1
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
def test_worker_update_update_orm(self):
"""Test worker update updating the worker orm object."""
worker = self._create_workers(1)[0]
res = db.worker_update(self.ctxt, worker.id, orm_worker=worker, service_id=1)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
# If we are updating the ORM object we don't ignore the update_at field
# because it will get updated in the ORM instance.
self._assertEqualObjects(worker, db_worker)
def test_worker_update_update_orm(self):
"""Test worker update updating the worker orm object."""
worker = self._create_workers(1)[0]
res = db.worker_update(self.ctxt,
worker.id,
orm_worker=worker,
service_id=1)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
def test_worker_update_update_orm(self):
"""Test worker update updating the worker orm object."""
worker = self._create_workers(1)[0]
res = db.worker_update(self.ctxt,
worker.id,
orm_worker=worker,
service_id=1)
self.assertEqual(1, res)
db_worker = db.worker_get(self.ctxt, id=worker.id)
# If we are updating the ORM object we don't ignore the update_at field
# because it will get updated in the ORM instance.
self._assertEqualObjects(worker, db_worker)
def test_worker_update_no_subsecond(self):
"""Test basic worker update."""
db.sqlalchemy.api.DB_SUPPORTS_SUBSECOND_RESOLUTION = False
worker = self._create_workers(1)[0]
worker = db.worker_get(self.ctxt, id=worker.id)
now = datetime.utcnow().replace(microsecond=123)
with mock.patch('oslo_utils.timeutils.utcnow', return_value=now):
res = db.worker_update(self.ctxt, worker.id, service_id=1)
self.assertEqual(1, res)
worker.service_id = 1
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ['updated_at'])
self.assertEqual(0, db_worker.updated_at.microsecond)
def test_worker_update_no_subsecond(self):
"""Test basic worker update."""
db.sqlalchemy.api.DB_SUPPORTS_SUBSECOND_RESOLUTION = False
worker = self._create_workers(1)[0]
worker = db.worker_get(self.ctxt, id=worker.id)
now = datetime.utcnow().replace(microsecond=123)
with mock.patch("oslo_utils.timeutils.utcnow", return_value=now):
res = db.worker_update(self.ctxt, worker.id, service_id=1)
self.assertEqual(1, res)
worker.service_id = 1
db_worker = db.worker_get(self.ctxt, id=worker.id)
self._assertEqualObjects(worker, db_worker, ["updated_at"])
self.assertEqual(0, db_worker.updated_at.microsecond)
def do_cleanup(self, context, cleanup_request):
LOG.info('Initiating service %s cleanup', cleanup_request.service_id)
# If the 'until' field in the cleanup request is not set, we default to
# this very moment.
until = cleanup_request.until or timeutils.utcnow()
keep_entry = False
to_clean = db.worker_get_all(
context,
resource_type=cleanup_request.resource_type,
resource_id=cleanup_request.resource_id,
service_id=cleanup_request.service_id,
until=until)
for clean in to_clean:
original_service_id = clean.service_id
original_time = clean.updated_at
# Try to do a soft delete to mark the entry as being cleaned up
# by us (setting service id to our service id).
res = db.worker_claim_for_cleanup(context,
claimer_id=self.service_id,
orm_worker=clean)
# Claim may fail if entry is being cleaned by another service, has
# been removed (finished cleaning) by another service or the user
# started a new cleanable operation.
# In any of these cases we don't have to do cleanup or remove the
# worker entry.
if not res:
continue
# Try to get versioned object for resource we have to cleanup
try:
vo_cls = getattr(objects, clean.resource_type)
vo = vo_cls.get_by_id(context, clean.resource_id)
# Set the worker DB entry in the VO and mark it as being a
# clean operation
clean.cleaning = True
vo.worker = clean
except exception.NotFound:
LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.',
{
'type': clean.resource_type,
'id': clean.resource_id
})
else:
# Resource status should match
if vo.status != clean.status:
LOG.debug(
'Skipping cleanup for mismatching work on '
'%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.', {
'type': clean.resource_type,
'id': clean.resource_id,
'exp_sts': clean.status,
'found_sts': vo.status
})
else:
LOG.info(
'Cleaning %(type)s with id %(id)s and status '
'%(status)s', {
'type': clean.resource_type,
'id': clean.resource_id,
'status': clean.status
},
resource=vo)
try:
# Some cleanup jobs are performed asynchronously, so
# we don't delete the worker entry, they'll take care
# of it
keep_entry = self._do_cleanup(context, vo)
except Exception:
LOG.exception('Could not perform cleanup.')
# Return the worker DB entry to the original service
db.worker_update(context,
clean.id,
service_id=original_service_id,
updated_at=original_time)
continue
# The resource either didn't exist or was properly cleaned, either
# way we can remove the entry from the worker table if the cleanup
# method doesn't want to keep the entry (for example for delayed
# deletion).
if not keep_entry and not db.worker_destroy(context, id=clean.id):
LOG.warning('Could not remove worker entry %s.', clean.id)
LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
def do_cleanup(self, context, cleanup_request):
LOG.info('Initiating service %s cleanup',
cleanup_request.service_id)
# If the 'until' field in the cleanup request is not set, we default to
# this very moment.
until = cleanup_request.until or timeutils.utcnow()
keep_entry = False
to_clean = db.worker_get_all(
context,
resource_type=cleanup_request.resource_type,
resource_id=cleanup_request.resource_id,
service_id=cleanup_request.service_id,
until=until)
for clean in to_clean:
original_service_id = clean.service_id
original_time = clean.updated_at
# Try to do a soft delete to mark the entry as being cleaned up
# by us (setting service id to our service id).
res = db.worker_claim_for_cleanup(context,
claimer_id=self.service_id,
orm_worker=clean)
# Claim may fail if entry is being cleaned by another service, has
# been removed (finished cleaning) by another service or the user
# started a new cleanable operation.
# In any of these cases we don't have to do cleanup or remove the
# worker entry.
if not res:
continue
# Try to get versioned object for resource we have to cleanup
try:
vo_cls = getattr(objects, clean.resource_type)
vo = vo_cls.get_by_id(context, clean.resource_id)
# Set the worker DB entry in the VO and mark it as being a
# clean operation
clean.cleaning = True
vo.worker = clean
except exception.NotFound:
LOG.debug('Skipping cleanup for non existent %(type)s %(id)s.',
{'type': clean.resource_type,
'id': clean.resource_id})
else:
# Resource status should match
if vo.status != clean.status:
LOG.debug('Skipping cleanup for mismatching work on '
'%(type)s %(id)s: %(exp_sts)s <> %(found_sts)s.',
{'type': clean.resource_type,
'id': clean.resource_id,
'exp_sts': clean.status,
'found_sts': vo.status})
else:
LOG.info('Cleaning %(type)s with id %(id)s and status '
'%(status)s',
{'type': clean.resource_type,
'id': clean.resource_id,
'status': clean.status},
resource=vo)
try:
# Some cleanup jobs are performed asynchronously, so
# we don't delete the worker entry, they'll take care
# of it
keep_entry = self._do_cleanup(context, vo)
except Exception:
LOG.exception('Could not perform cleanup.')
# Return the worker DB entry to the original service
db.worker_update(context, clean.id,
service_id=original_service_id,
updated_at=original_time)
continue
# The resource either didn't exist or was properly cleaned, either
# way we can remove the entry from the worker table if the cleanup
# method doesn't want to keep the entry (for example for delayed
# deletion).
if not keep_entry and not db.worker_destroy(context, id=clean.id):
LOG.warning('Could not remove worker entry %s.', clean.id)
LOG.info('Service %s cleanup completed.', cleanup_request.service_id)
