def _delete_single_resource(self, resource):
"""Safe resource deletion with retries and timeouts.
Send request to delete resource, in case of failures repeat it few
times. After that pull status of resource until it's deleted.
Writes in LOG warning with UUID of resource that wasn't deleted
:param resource: instance of resource manager initiated with resource
that should be deleted.
"""
msg_kw = {
"uuid": resource.id(),
"name": resource.name() or "",
"service": resource._service,
"resource": resource._resource,
}
LOG.debug("Deleting %(service)s %(resource)s object %(name)s (%(uuid)s)" % msg_kw)
try:
rutils.retry(resource._max_attempts, resource.delete)
except Exception as e:
msg_kw["reason"] = e
LOG.warning(
_(
"Resource deletion failed, max retries exceeded for "
"%(service)s.%(resource)s: %(uuid)s. Reason: %(reason)s"
)
% msg_kw
)
if logging.is_debug():
LOG.exception(e)
else:
started = time.time()
failures_count = 0
while time.time() - started < resource._timeout:
try:
if resource.is_deleted():
return
except Exception as e:
LOG.warning(
_("Seems like %s.%s.is_deleted(self) method is broken " "It shouldn't raise any exceptions.")
% (resource.__module__, type(resource).__name__)
)
LOG.exception(e)
# NOTE(boris-42): Avoid LOG spamming in case of bad
# is_deleted() method
failures_count += 1
if failures_count > resource._max_attempts:
break
finally:
time.sleep(resource._interval)
LOG.warning(
_("Resource deletion failed, timeout occurred for " "%(service)s.%(resource)s: %(uuid)s.") % msg_kw
)
def _delete_single_resource(self, resource):
"""Safe resource deletion with retries and timeouts.
Send request to delete resource, in case of failures repeat it few
times. After that pull status of resource until it's deleted.
Writes in LOG warning with UUID of resource that wasn't deleted
:param resource: instance of resource manager initiated with resource
that should be deleted.
"""
msg_kw = {
"uuid": resource.id(),
"name": resource.name() or "",
"service": resource._service,
"resource": resource._resource
}
LOG.debug(
"Deleting %(service)s %(resource)s object %(name)s (%(uuid)s)" %
msg_kw)
try:
rutils.retry(resource._max_attempts, resource.delete)
except Exception as e:
msg_kw["reason"] = e
LOG.warning(
_("Resource deletion failed, max retries exceeded for "
"%(service)s.%(resource)s: %(uuid)s. Reason: %(reason)s") %
msg_kw)
if logging.is_debug():
LOG.exception(e)
else:
started = time.time()
failures_count = 0
while time.time() - started < resource._timeout:
try:
if resource.is_deleted():
return
except Exception as e:
LOG.warning(
_("Seems like %s.%s.is_deleted(self) method is broken "
"It shouldn't raise any exceptions.") %
(resource.__module__, type(resource).__name__))
LOG.exception(e)
# NOTE(boris-42): Avoid LOG spamming in case of bad
# is_deleted() method
failures_count += 1
if failures_count > resource._max_attempts:
break
finally:
rutils.interruptable_sleep(resource._interval)
LOG.warning(
_("Resource deletion failed, timeout occurred for "
"%(service)s.%(resource)s: %(uuid)s.") % msg_kw)
def cleanup(self):
"""Delete created flavors."""
clients = osclients.Clients(self.context["admin"]["endpoint"])
for flavor in self.context["flavors"].values():
with logging.ExceptionLogger(
LOG, _("Can't delete flavor %s") % flavor["id"]):
rutils.retry(3, clients.nova().flavors.delete, flavor["id"])
LOG.debug("Flavor is deleted %s" % flavor["id"])
def _publish(admin, user, manager):
try:
for raw_resource in rutils.retry(3, manager.list):
queue.append((admin, user, raw_resource))
except Exception:
LOG.exception("Seems like %s.%s.list(self) method is broken. "
"It shouldn't raise any exceptions." %
(manager.__module__, type(manager).__name__))
def _publish(admin, user, manager):
try:
for raw_resource in rutils.retry(3, manager.list):
queue.append((admin, user, raw_resource))
except Exception:
LOG.exception(
"Seems like %s.%s.list(self) method is broken. "
"It shouldn't raise any exceptions."
% (manager.__module__, type(manager).__name__))
def _publisher(self, queue):
"""Publisher for deletion jobs.
This method lists all resources (using manager_cls) and puts jobs for
deletion.
"""
try:
for raw_resource in rutils.retry(3, self.manager_cls.list,
self.client):
queue.append(raw_resource)
except Exception:
LOG.exception(
"Seems like %s.%s.list(self) method is broken. "
"It shouldn't raise any exceptions." %
(self.manager_cls.__module__, self.manager_cls.__name__))