def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
if get_device_conf_func(device) is None:
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception():
errcode = ex.get_error_code()
if errcode == libvirt.VIR_ERR_OPERATION_FAILED:
errmsg = ex.get_error_message()
if 'not found' in errmsg:
# This will be raised if the live domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
elif errcode == libvirt.VIR_ERR_INVALID_ARG:
errmsg = ex.get_error_message()
if 'no target device' in errmsg:
# This will be raised if the persistent domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
if get_device_conf_func(device) is None:
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception(reraise=False) as ctx:
errcode = ex.get_error_code()
if errcode in (libvirt.VIR_ERR_OPERATION_FAILED,
libvirt.VIR_ERR_INTERNAL_ERROR):
errmsg = ex.get_error_message()
if 'not found' in errmsg:
# This will be raised if the live domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
elif errcode == libvirt.VIR_ERR_INVALID_ARG:
errmsg = ex.get_error_message()
if 'no target device' in errmsg:
# This will be raised if the persistent domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
# Re-raise the original exception if we're not raising
# DeviceNotFound instead. This will avoid logging of a
# "Original exception being dropped" traceback.
ctx.reraise = True
def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
if get_device_conf_func(device) is None:
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception(reraise=False) as ctx:
code = ex.get_error_code()
msg = ex.get_error_message()
if code == libvirt.VIR_ERR_DEVICE_MISSING:
raise exception.DeviceNotFound(
device=alternative_device_name)
# NOTE(lyarwood): https://bugzilla.redhat.com/1878659
# Ignore this known QEMU bug for the time being allowing
# our retry logic to fire again and hopefully see that
# the device has been removed asynchronously by QEMU
# in the meantime when the next call to detach raises
# VIR_ERR_DEVICE_MISSING.
if (code == libvirt.VIR_ERR_INTERNAL_ERROR and msg
and 'already in the process of unplug' in msg):
LOG.debug('Ignoring QEMU rejecting our request to '
'detach as it is caused by a previous '
'request still being in progress.')
return
# Re-raise the original exception if we're not raising
# DeviceNotFound instead. This will avoid logging of a
# "Original exception being dropped" traceback.
ctx.reraise = True
def detach_device_with_retry(self,
get_device_conf_func,
device,
persistent,
live,
max_retry_count=7,
inc_sleep_time=2,
max_sleep_time=30):
"""Detaches a device from the guest. After the initial detach request,
a function is returned which can be used to ensure the device is
successfully removed from the guest domain (retrying the removal as
necessary).
:param get_device_conf_func: function which takes device as a parameter
and returns the configuration for device
:param device: device to detach
:param persistent: bool to indicate whether the change is
persistent or not
:param live: bool to indicate whether it affects the guest in running
state
:param max_retry_count: number of times the returned function will
retry a detach before failing
:param inc_sleep_time: incremental time to sleep in seconds between
detach retries
:param max_sleep_time: max sleep time in seconds beyond which the sleep
time will not be incremented using param
inc_sleep_time. On reaching this threshold,
max_sleep_time will be used as the sleep time.
"""
conf = get_device_conf_func(device)
if conf is None:
raise exception.DeviceNotFound(device=device)
self.detach_device(conf, persistent, live)
@loopingcall.RetryDecorator(max_retry_count=max_retry_count,
inc_sleep_time=inc_sleep_time,
max_sleep_time=max_sleep_time,
exceptions=exception.DeviceDetachFailed)
def _do_wait_and_retry_detach():
config = get_device_conf_func(device)
if config is not None:
try:
# Device is already detached from persistent domain
# and only transient domain needs update
self.detach_device(config, persistent=False, live=live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception():
errcode = ex.get_error_code()
if errcode == libvirt.VIR_ERR_OPERATION_FAILED:
errmsg = ex.get_error_message()
if 'not found' in errmsg:
raise exception.DeviceNotFound(device=device)
reason = _("Unable to detach from guest transient domain.")
raise exception.DeviceDetachFailed(device=device,
reason=reason)
return _do_wait_and_retry_detach
def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception():
errcode = ex.get_error_code()
if errcode == libvirt.VIR_ERR_OPERATION_FAILED:
errmsg = ex.get_error_message()
if 'not found' in errmsg:
# This will be raised if the live domain
# detach fails because the device is not found
raise exception.DeviceNotFound(device=device)
elif errcode == libvirt.VIR_ERR_INVALID_ARG:
errmsg = ex.get_error_message()
if 'no target device' in errmsg:
# This will be raised if the persistent domain
# detach fails because the device is not found
raise exception.DeviceNotFound(device=device)
def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
if get_device_conf_func(device) is None:
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception(reraise=False) as ctx:
errcode = ex.get_error_code()
# TODO(lyarwood): Remove libvirt.VIR_ERR_OPERATION_FAILED
# and libvirt.VIR_ERR_INTERNAL_ERROR once
# MIN_LIBVIRT_VERSION is >= 4.1.0
if supports_device_missing_error_code:
unplug_libvirt_error_codes.add(
libvirt.VIR_ERR_DEVICE_MISSING)
if errcode in unplug_libvirt_error_codes:
# TODO(lyarwood): Remove the following error message
# check once we only care about VIR_ERR_DEVICE_MISSING
errmsg = ex.get_error_message()
if 'not found' in errmsg:
# This will be raised if the live domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
# TODO(lyarwood): Remove libvirt.VIR_ERR_INVALID_ARG once
# MIN_LIBVIRT_VERSION is >= 4.1.0
elif errcode == libvirt.VIR_ERR_INVALID_ARG:
errmsg = ex.get_error_message()
if 'no target device' in errmsg:
# This will be raised if the persistent domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
# Re-raise the original exception if we're not raising
# DeviceNotFound instead. This will avoid logging of a
# "Original exception being dropped" traceback.
ctx.reraise = True
def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
if get_device_conf_func(device) is None:
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception(reraise=False) as ctx:
if ex.get_error_code() == libvirt.VIR_ERR_DEVICE_MISSING:
raise exception.DeviceNotFound(
device=alternative_device_name)
# Re-raise the original exception if we're not raising
# DeviceNotFound instead. This will avoid logging of a
# "Original exception being dropped" traceback.
ctx.reraise = True
def _do_wait_and_retry_detach():
config = get_device_conf_func(device)
if config is not None:
try:
# Device is already detached from persistent domain
# and only transient domain needs update
self.detach_device(config, persistent=False, live=live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception():
errcode = ex.get_error_code()
if errcode == libvirt.VIR_ERR_OPERATION_FAILED:
errmsg = ex.get_error_message()
if 'not found' in errmsg:
raise exception.DeviceNotFound(device=device)
reason = _("Unable to detach from guest transient domain.")
raise exception.DeviceDetachFailed(device=device,
reason=reason)
def _try_detach_device(conf, persistent=False, host=None):
# WRS: live flag is recalculated because the domain may have
# changed state.
state = self.get_power_state(host)
live = state in (power_state.RUNNING, power_state.PAUSED)
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
errcode = ex.get_error_code()
# if the state of the domain changed, we need to retry
if errcode not in (libvirt.VIR_ERR_OPERATION_INVALID,
libvirt.VIR_ERR_OPERATION_FAILED):
with excutils.save_and_reraise_exception():
if errcode == libvirt.VIR_ERR_INVALID_ARG:
errmsg = ex.get_error_message()
if 'no target device' in errmsg:
# This will be raised if the persistent domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
def detach_device_with_retry(self,
get_device_conf_func,
device,
live,
max_retry_count=7,
inc_sleep_time=10,
max_sleep_time=60,
alternative_device_name=None):
"""Detaches a device from the guest. After the initial detach request,
a function is returned which can be used to ensure the device is
successfully removed from the guest domain (retrying the removal as
necessary).
:param get_device_conf_func: function which takes device as a parameter
and returns the configuration for device
:param device: device to detach
:param live: bool to indicate whether it affects the guest in running
state
:param max_retry_count: number of times the returned function will
retry a detach before failing
:param inc_sleep_time: incremental time to sleep in seconds between
detach retries
:param max_sleep_time: max sleep time in seconds beyond which the sleep
time will not be incremented using param
inc_sleep_time. On reaching this threshold,
max_sleep_time will be used as the sleep time.
:param alternative_device_name: This is an alternative identifier for
the device if device is not an ID, used solely for error messages.
"""
alternative_device_name = alternative_device_name or device
def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
if get_device_conf_func(device) is None:
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception(reraise=False) as ctx:
code = ex.get_error_code()
msg = ex.get_error_message()
if code == libvirt.VIR_ERR_DEVICE_MISSING:
raise exception.DeviceNotFound(
device=alternative_device_name)
# NOTE(lyarwood): https://bugzilla.redhat.com/1878659
# Ignore this known QEMU bug for the time being allowing
# our retry logic to fire again and hopefully see that
# the device has been removed asynchronously by QEMU
# in the meantime when the next call to detach raises
# VIR_ERR_DEVICE_MISSING.
if (code == libvirt.VIR_ERR_INTERNAL_ERROR and msg
and 'already in the process of unplug' in msg):
LOG.debug('Ignoring QEMU rejecting our request to '
'detach as it is caused by a previous '
'request still being in progress.')
return
# Re-raise the original exception if we're not raising
# DeviceNotFound instead. This will avoid logging of a
# "Original exception being dropped" traceback.
ctx.reraise = True
conf = get_device_conf_func(device)
if conf is None:
raise exception.DeviceNotFound(device=alternative_device_name)
persistent = self.has_persistent_configuration()
LOG.debug('Attempting initial detach for device %s',
alternative_device_name)
try:
_try_detach_device(conf, persistent, live)
except exception.DeviceNotFound:
# NOTE(melwitt): There are effectively two configs for an instance.
# The persistent config (affects instance upon next boot) and the
# live config (affects running instance). When we detach a device,
# we need to detach it from both configs if the instance has a
# persistent config and a live config. If we tried to detach the
# device with persistent=True and live=True and it was not found,
# we should still try to detach from the live config, so continue.
if persistent and live:
pass
else:
raise
LOG.debug('Start retrying detach until device %s is gone.',
alternative_device_name)
@loopingcall.RetryDecorator(max_retry_count=max_retry_count,
inc_sleep_time=inc_sleep_time,
max_sleep_time=max_sleep_time,
exceptions=exception.DeviceDetachFailed)
def _do_wait_and_retry_detach():
config = get_device_conf_func(device)
if config is not None:
# Device is already detached from persistent config
# and only the live config needs to be updated.
_try_detach_device(config, persistent=False, live=live)
reason = _("Unable to detach the device from the live config.")
raise exception.DeviceDetachFailed(
device=alternative_device_name, reason=reason)
return _do_wait_and_retry_detach
def detach_device_with_retry(self,
get_device_conf_func,
device,
live,
max_retry_count=7,
inc_sleep_time=2,
max_sleep_time=30,
alternative_device_name=None):
"""Detaches a device from the guest. After the initial detach request,
a function is returned which can be used to ensure the device is
successfully removed from the guest domain (retrying the removal as
necessary).
:param get_device_conf_func: function which takes device as a parameter
and returns the configuration for device
:param device: device to detach
:param live: bool to indicate whether it affects the guest in running
state
:param max_retry_count: number of times the returned function will
retry a detach before failing
:param inc_sleep_time: incremental time to sleep in seconds between
detach retries
:param max_sleep_time: max sleep time in seconds beyond which the sleep
time will not be incremented using param
inc_sleep_time. On reaching this threshold,
max_sleep_time will be used as the sleep time.
:param alternative_device_name: This is an alternative identifier for
the device if device is not an ID, used solely for error messages.
"""
alternative_device_name = alternative_device_name or device
def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
if get_device_conf_func(device) is None:
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception(reraise=False) as ctx:
errcode = ex.get_error_code()
# TODO(lyarwood): Remove libvirt.VIR_ERR_OPERATION_FAILED
# and libvirt.VIR_ERR_INTERNAL_ERROR once
# MIN_LIBVIRT_VERSION is >= 4.1.0
if errcode in (libvirt.VIR_ERR_OPERATION_FAILED,
libvirt.VIR_ERR_INTERNAL_ERROR,
libvirt.VIR_ERR_DEVICE_MISSING):
# TODO(lyarwood): Remove the following error message
# check once we only care about VIR_ERR_DEVICE_MISSING
errmsg = ex.get_error_message()
if 'not found' in errmsg:
# This will be raised if the live domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
# TODO(lyarwood): Remove libvirt.VIR_ERR_INVALID_ARG once
# MIN_LIBVIRT_VERSION is >= 4.1.0
elif errcode == libvirt.VIR_ERR_INVALID_ARG:
errmsg = ex.get_error_message()
if 'no target device' in errmsg:
# This will be raised if the persistent domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
# Re-raise the original exception if we're not raising
# DeviceNotFound instead. This will avoid logging of a
# "Original exception being dropped" traceback.
ctx.reraise = True
conf = get_device_conf_func(device)
if conf is None:
raise exception.DeviceNotFound(device=alternative_device_name)
persistent = self.has_persistent_configuration()
LOG.debug('Attempting initial detach for device %s',
alternative_device_name)
try:
_try_detach_device(conf, persistent, live)
except exception.DeviceNotFound:
# NOTE(melwitt): There are effectively two configs for an instance.
# The persistent config (affects instance upon next boot) and the
# live config (affects running instance). When we detach a device,
# we need to detach it from both configs if the instance has a
# persistent config and a live config. If we tried to detach the
# device with persistent=True and live=True and it was not found,
# we should still try to detach from the live config, so continue.
if persistent and live:
pass
else:
raise
LOG.debug('Start retrying detach until device %s is gone.',
alternative_device_name)
@loopingcall.RetryDecorator(max_retry_count=max_retry_count,
inc_sleep_time=inc_sleep_time,
max_sleep_time=max_sleep_time,
exceptions=exception.DeviceDetachFailed)
def _do_wait_and_retry_detach():
config = get_device_conf_func(device)
if config is not None:
# Device is already detached from persistent config
# and only the live config needs to be updated.
_try_detach_device(config, persistent=False, live=live)
reason = _("Unable to detach the device from the live config.")
raise exception.DeviceDetachFailed(
device=alternative_device_name, reason=reason)
return _do_wait_and_retry_detach
def detach_device_with_retry(self,
get_device_conf_func,
device,
host,
max_retry_count=7,
inc_sleep_time=2,
max_sleep_time=30,
alternative_device_name=None):
"""Detaches a device from the guest. After the initial detach request,
a function is returned which can be used to ensure the device is
successfully removed from the guest domain (retrying the removal as
necessary).
:param get_device_conf_func: function which takes device as a parameter
and returns the configuration for device
:param device: device to detach
:param host: host.Host object for guest attached to device
:param max_retry_count: number of times the returned function will
retry a detach before failing
:param inc_sleep_time: incremental time to sleep in seconds between
detach retries
:param max_sleep_time: max sleep time in seconds beyond which the sleep
time will not be incremented using param
inc_sleep_time. On reaching this threshold,
max_sleep_time will be used as the sleep time.
:param alternative_device_name: This is an alternative identifier for
the device if device is not an ID, used solely for error messages.
"""
alternative_device_name = alternative_device_name or device
def _try_detach_device(conf, persistent=False, host=None):
# WRS: live flag is recalculated because the domain may have
# changed state.
state = self.get_power_state(host)
live = state in (power_state.RUNNING, power_state.PAUSED)
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
errcode = ex.get_error_code()
# if the state of the domain changed, we need to retry
if errcode not in (libvirt.VIR_ERR_OPERATION_INVALID,
libvirt.VIR_ERR_OPERATION_FAILED):
with excutils.save_and_reraise_exception():
if errcode == libvirt.VIR_ERR_INVALID_ARG:
errmsg = ex.get_error_message()
if 'no target device' in errmsg:
# This will be raised if the persistent domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
conf = get_device_conf_func(device)
if conf is None:
raise exception.DeviceNotFound(device=alternative_device_name)
persistent = self.has_persistent_configuration()
LOG.debug('Attempting initial detach for device %s',
alternative_device_name)
try:
_try_detach_device(conf, persistent, host)
except exception.DeviceNotFound:
# NOTE(melwitt): There are effectively two configs for an instance.
# The persistent config (affects instance upon next boot) and the
# live config (affects running instance). When we detach a device,
# we need to detach it from both configs if the instance has a
# persistent config and a live config. If we tried to detach the
# device with persistent=True and live=True and it was not found,
# we should still try to detach from the live config, so continue.
state = self.get_power_state(host)
live = state in (power_state.RUNNING, power_state.PAUSED)
if persistent and live:
pass
else:
raise
LOG.debug('Start retrying detach until device %s is gone.',
alternative_device_name)
@loopingcall.RetryDecorator(max_retry_count=max_retry_count,
inc_sleep_time=inc_sleep_time,
max_sleep_time=max_sleep_time,
exceptions=exception.DeviceDetachFailed)
def _do_wait_and_retry_detach():
config = get_device_conf_func(device)
if config is not None:
# Device is already detached from persistent domain
# and only transient domain needs update
_try_detach_device(config, persistent=False, host=host)
reason = _("Unable to detach from guest transient domain.")
raise exception.DeviceDetachFailed(
device=alternative_device_name, reason=reason)
return _do_wait_and_retry_detach
def detach_device_with_retry(self,
get_device_conf_func,
device,
live,
max_retry_count=7,
inc_sleep_time=2,
max_sleep_time=30,
alternative_device_name=None):
"""Detaches a device from the guest. After the initial detach request,
a function is returned which can be used to ensure the device is
successfully removed from the guest domain (retrying the removal as
necessary).
:param get_device_conf_func: function which takes device as a parameter
and returns the configuration for device
:param device: device to detach
:param live: bool to indicate whether it affects the guest in running
state
:param max_retry_count: number of times the returned function will
retry a detach before failing
:param inc_sleep_time: incremental time to sleep in seconds between
detach retries
:param max_sleep_time: max sleep time in seconds beyond which the sleep
time will not be incremented using param
inc_sleep_time. On reaching this threshold,
max_sleep_time will be used as the sleep time.
:param alternative_device_name: This is an alternative identifier for
the device if device is not an ID, used solely for error messages.
"""
alternative_device_name = alternative_device_name or device
def _try_detach_device(conf, persistent=False, live=False):
# Raise DeviceNotFound if the device isn't found during detach
try:
self.detach_device(conf, persistent=persistent, live=live)
LOG.debug(
'Successfully detached device %s from guest. '
'Persistent? %s. Live? %s', device, persistent, live)
except libvirt.libvirtError as ex:
with excutils.save_and_reraise_exception():
errcode = ex.get_error_code()
if errcode == libvirt.VIR_ERR_OPERATION_FAILED:
errmsg = ex.get_error_message()
if 'not found' in errmsg:
# This will be raised if the live domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
elif errcode == libvirt.VIR_ERR_INVALID_ARG:
errmsg = ex.get_error_message()
if 'no target device' in errmsg:
# This will be raised if the persistent domain
# detach fails because the device is not found
raise exception.DeviceNotFound(
device=alternative_device_name)
conf = get_device_conf_func(device)
if conf is None:
raise exception.DeviceNotFound(device=alternative_device_name)
persistent = self.has_persistent_configuration()
LOG.debug('Attempting initial detach for device %s',
alternative_device_name)
_try_detach_device(conf, persistent, live)
LOG.debug('Start retrying detach until device %s is gone.',
alternative_device_name)
@loopingcall.RetryDecorator(max_retry_count=max_retry_count,
inc_sleep_time=inc_sleep_time,
max_sleep_time=max_sleep_time,
exceptions=exception.DeviceDetachFailed)
def _do_wait_and_retry_detach():
config = get_device_conf_func(device)
if config is not None:
# Device is already detached from persistent domain
# and only transient domain needs update
_try_detach_device(config, persistent=False, live=live)
reason = _("Unable to detach from guest transient domain.")
raise exception.DeviceDetachFailed(
device=alternative_device_name, reason=reason)
return _do_wait_and_retry_detach
