def notification_callback(self, event_type, payload):
"""Handle a notification message.
It is used as a callback of a notification-based resource monitor.
:param event_type: an event type of a notification.
:param payload: a payload of a notification.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
LOG.trace('Handling a notification...')
reservation_flags = {}
data = payload.get('nova_object.data', None)
if data:
if data['disabled'] or data['forced_down']:
failed_hosts = db_api.reservable_host_get_all_by_queries(
['hypervisor_hostname == ' + data['host']])
if failed_hosts:
LOG.warn('%s failed.',
failed_hosts[0]['hypervisor_hostname'])
reservation_flags = self._handle_failures(failed_hosts)
else:
recovered_hosts = db_api.host_get_all_by_queries(
['reservable == 0',
'hypervisor_hostname == ' + data['host']])
if recovered_hosts:
db_api.host_update(recovered_hosts[0]['id'],
{'reservable': True})
LOG.warn('%s recovered.',
recovered_hosts[0]['hypervisor_hostname'])
return reservation_flags
def _handle_failures(self, failed_hosts):
"""Handle resource failures.
:param: failed_hosts: a list of failed hosts.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
# Update the computehosts table
for host in failed_hosts:
try:
db_api.host_update(host['id'], {'reservable': False})
except Exception as e:
LOG.exception('Failed to update %s. %s',
host['hypervisor_hostname'], str(e))
# Heal related reservations
return self.heal()
def poll(self):
"""Detect and handle resource failures.
:return: a dictionary of {reservation id: flags to update}
e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
{'missing_resources': True}}
"""
LOG.trace('Poll...')
reservation_flags = {}
failed_hosts, recovered_hosts = self._poll_resource_failures()
if failed_hosts:
for host in failed_hosts:
LOG.warn('%s failed.', host['hypervisor_hostname'])
reservation_flags = self._handle_failures(failed_hosts)
if recovered_hosts:
for host in recovered_hosts:
db_api.host_update(host['id'], {'reservable': True})
LOG.warn('%s recovered.', host['hypervisor_hostname'])
return reservation_flags