def notification_callback(self, event_type, payload): """Handle a notification message. It is used as a callback of a notification-based resource monitor. :param event_type: an event type of a notification. :param payload: a payload of a notification. :return: a dictionary of {reservation id: flags to update} e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': {'missing_resources': True}} """ LOG.trace('Handling a notification...') reservation_flags = {} data = payload.get('nova_object.data', None) if data: if data['disabled'] or data['forced_down']: failed_hosts = db_api.reservable_host_get_all_by_queries( ['hypervisor_hostname == ' + data['host']]) if failed_hosts: LOG.warn('%s failed.', failed_hosts[0]['hypervisor_hostname']) reservation_flags = self._handle_failures(failed_hosts) else: recovered_hosts = db_api.host_get_all_by_queries( ['reservable == 0', 'hypervisor_hostname == ' + data['host']]) if recovered_hosts: db_api.host_update(recovered_hosts[0]['id'], {'reservable': True}) LOG.warn('%s recovered.', recovered_hosts[0]['hypervisor_hostname']) return reservation_flags
def _handle_failures(self, failed_hosts): """Handle resource failures. :param: failed_hosts: a list of failed hosts. :return: a dictionary of {reservation id: flags to update} e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': {'missing_resources': True}} """ # Update the computehosts table for host in failed_hosts: try: db_api.host_update(host['id'], {'reservable': False}) except Exception as e: LOG.exception('Failed to update %s. %s', host['hypervisor_hostname'], str(e)) # Heal related reservations return self.heal()
def poll(self): """Detect and handle resource failures. :return: a dictionary of {reservation id: flags to update} e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': {'missing_resources': True}} """ LOG.trace('Poll...') reservation_flags = {} failed_hosts, recovered_hosts = self._poll_resource_failures() if failed_hosts: for host in failed_hosts: LOG.warn('%s failed.', host['hypervisor_hostname']) reservation_flags = self._handle_failures(failed_hosts) if recovered_hosts: for host in recovered_hosts: db_api.host_update(host['id'], {'reservable': True}) LOG.warn('%s recovered.', host['hypervisor_hostname']) return reservation_flags