def _handle_notification_type_instance(self, context, notification):
if not virt_events.is_valid_event(notification.payload):
LOG.info(
"Notification '%(uuid)s' received with payload "
"%(payload)s is ignored.", {
"uuid": notification.notification_uuid,
"payload": notification.payload
})
return fields.NotificationStatus.IGNORED
notification_status = fields.NotificationStatus.FINISHED
exception_info = None
try:
self.driver.execute_instance_failure(
context, notification.payload.get('instance_uuid'),
notification.notification_uuid)
except exception.IgnoreInstanceRecoveryException as e:
notification_status = fields.NotificationStatus.IGNORED
exception_info = e
except exception.SkipInstanceRecoveryException:
notification_status = fields.NotificationStatus.FINISHED
except (exception.MasakariException,
exception.InstanceRecoveryFailureException) as e:
notification_status = fields.NotificationStatus.ERROR
LOG.error(
"Failed to process notification '%(uuid)s'."
" Reason: %(error)s", {
"uuid": notification.notification_uuid,
"error": e.message
})
exception_info = e
if exception_info:
tb = traceback.format_exc()
engine_utils.notify_about_notification_update(
context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.ERROR,
exception=str(exception_info),
tb=tb)
else:
engine_utils.notify_about_notification_update(
context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.END)
return notification_status
def test_notify_about_notification_update(self, mock_from_exception,
mock_NotificationApiPayload,
mock_NotificationApiNotification,
mock_NotificationPublisher,
mock_EventType):
mock_fault = mock.Mock()
mock_from_exception.return_value = mock_fault
mock_payload = mock.Mock()
mock_NotificationApiPayload.return_value = mock_payload
mock_engine_notification = mock.Mock()
mock_NotificationApiNotification.return_value = (
mock_engine_notification)
mock_engine_notification.emit.return_value = None
mock_publisher = mock.Mock()
mock_NotificationPublisher.return_value = mock_publisher
mock_event_type = mock.Mock()
mock_EventType.return_value = mock_event_type
mock_context = mock.Mock()
notification = notification_obj.Notification()
action = fields.EventNotificationAction.NOTIFICATION_PROCESS
phase = fields.EventNotificationPhase.ERROR
e = Exception()
engine_utils.notify_about_notification_update(mock_context,
notification,
action=action,
phase=phase,
exception=e)
mock_from_exception.assert_called_once_with(e, None)
mock_NotificationApiPayload.assert_called_once_with(
notification=notification, fault=mock_fault)
mock_NotificationApiNotification.assert_called_once_with(
context=mock_context,
priority=fields.EventNotificationPriority.ERROR,
publisher=mock_publisher,
event_type=mock_event_type,
payload=mock_payload)
mock_NotificationPublisher.assert_called_once_with(
context=mock_context,
host=socket.gethostname(),
binary='masakari-engine')
mock_engine_notification.emit.assert_called_once_with(mock_context)
def _handle_notification_type_instance(self, context, notification):
if not virt_events.is_valid_event(notification.payload):
LOG.info("Notification '%(uuid)s' received with payload "
"%(payload)s is ignored.",
{"uuid": notification.notification_uuid,
"payload": notification.payload})
return fields.NotificationStatus.IGNORED
notification_status = fields.NotificationStatus.FINISHED
exception_info = None
try:
self.driver.execute_instance_failure(
context, notification.payload.get('instance_uuid'),
notification.notification_uuid)
except exception.IgnoreInstanceRecoveryException as e:
notification_status = fields.NotificationStatus.IGNORED
exception_info = e
except exception.SkipInstanceRecoveryException as e:
notification_status = fields.NotificationStatus.FINISHED
except (exception.MasakariException,
exception.InstanceRecoveryFailureException) as e:
notification_status = fields.NotificationStatus.ERROR
LOG.error("Failed to process notification '%(uuid)s'."
" Reason: %(error)s",
{"uuid": notification.notification_uuid,
"error": e.message})
exception_info = e
if exception_info:
tb = traceback.format_exc()
engine_utils.notify_about_notification_update(context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.ERROR,
exception=str(exception_info),
tb=tb)
else:
engine_utils.notify_about_notification_update(context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.END)
return notification_status
def test_notify_about_notification_update(
self, mock_from_exception, mock_NotificationApiPayload,
mock_NotificationApiNotification, mock_NotificationPublisher,
mock_EventType):
mock_fault = mock.Mock()
mock_from_exception.return_value = mock_fault
mock_payload = mock.Mock()
mock_NotificationApiPayload.return_value = mock_payload
mock_engine_notification = mock.Mock()
mock_NotificationApiNotification.return_value = (
mock_engine_notification)
mock_engine_notification.emit.return_value = None
mock_publisher = mock.Mock()
mock_NotificationPublisher.return_value = mock_publisher
mock_event_type = mock.Mock()
mock_EventType.return_value = mock_event_type
mock_context = mock.Mock()
notification = notification_obj.Notification()
action = fields.EventNotificationAction.NOTIFICATION_PROCESS
phase = fields.EventNotificationPhase.ERROR
e = Exception()
engine_utils.notify_about_notification_update(mock_context,
notification, action=action, phase=phase, exception=e)
mock_from_exception.assert_called_once_with(e, None)
mock_NotificationApiPayload.assert_called_once_with(
notification=notification, fault=mock_fault)
mock_NotificationApiNotification.assert_called_once_with(
context=mock_context,
priority=fields.EventNotificationPriority.ERROR,
publisher=mock_publisher,
event_type=mock_event_type,
payload=mock_payload)
mock_NotificationPublisher.assert_called_once_with(
context=mock_context, host=socket.gethostname(),
binary='masakari-engine')
mock_engine_notification.emit.assert_called_once_with(mock_context)
def _handle_notification_type_process(self, context, notification):
notification_status = fields.NotificationStatus.FINISHED
notification_event = notification.payload.get('event')
process_name = notification.payload.get('process_name')
exception_info = None
if notification_event.upper() == 'STARTED':
LOG.info(
"Notification type '%(type)s' received for host "
"'%(host_uuid)s': '%(process_name)s' has been "
"%(event)s.", {
'type': notification.type,
'host_uuid': notification.source_host_uuid,
'process_name': process_name,
'event': notification_event
})
elif notification_event.upper() == 'STOPPED':
host_obj = objects.Host.get_by_uuid(context,
notification.source_host_uuid)
host_name = host_obj.name
# Mark host on_maintenance mode as True
update_data = {
'on_maintenance': True,
}
host_obj.update(update_data)
host_obj.save()
try:
self.driver.execute_process_failure(
context, process_name, host_name,
notification.notification_uuid)
except exception.SkipProcessRecoveryException:
notification_status = fields.NotificationStatus.FINISHED
except (exception.MasakariException,
exception.ProcessRecoveryFailureException) as e:
notification_status = fields.NotificationStatus.ERROR
LOG.error(
"Failed to process notification '%(uuid)s'."
" Reason: %(error)s", {
"uuid": notification.notification_uuid,
"error": e.message
})
exception_info = e
else:
LOG.warning(
"Invalid event: %(event)s received for "
"notification type: %(notification_type)s", {
'event': notification_event,
'notification_type': notification.type
})
notification_status = fields.NotificationStatus.IGNORED
if exception_info:
tb = traceback.format_exc()
engine_utils.notify_about_notification_update(
context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.ERROR,
exception=str(exception_info),
tb=tb)
else:
engine_utils.notify_about_notification_update(
context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.END)
return notification_status
def _process_notification(self, context, notification):
@utils.synchronized(notification.source_host_uuid, blocking=True)
def do_process_notification(notification):
LOG.info(
'Processing notification %(notification_uuid)s of '
'type: %(type)s', {
'notification_uuid': notification.notification_uuid,
'type': notification.type
})
# Get notification from db
notification_db = objects.Notification.get_by_uuid(
context, notification.notification_uuid)
# NOTE(tpatil): To fix bug 1773132, process notification only
# if the notification status is New and the current notification
# from DB status is Not New to avoid recovering from failure twice
if (notification.status == fields.NotificationStatus.NEW and
notification_db.status != fields.NotificationStatus.NEW):
LOG.warning(
"Processing of notification is skipped to avoid "
"recovering from failure twice. "
"Notification received is '%(uuid)s' "
"and it's status is '%(new_status)s' and the "
"current status of same notification in db "
"is '%(old_status)s'", {
"uuid": notification.notification_uuid,
"new_status": notification.status,
"old_status": notification_db.status
})
return
update_data = {
'status': fields.NotificationStatus.RUNNING,
}
notification.update(update_data)
notification.save()
if notification.type == fields.NotificationType.PROCESS:
notification_status = self._handle_notification_type_process(
context, notification)
elif notification.type == fields.NotificationType.VM:
notification_status = self._handle_notification_type_instance(
context, notification)
elif notification.type == fields.NotificationType.COMPUTE_HOST:
notification_status = self._handle_notification_type_host(
context, notification)
LOG.info(
"Notification %(notification_uuid)s exits with "
"status: %(status)s.", {
'notification_uuid': notification.notification_uuid,
'status': notification_status
})
update_data = {'status': notification_status}
notification.update(update_data)
notification.save()
engine_utils.notify_about_notification_update(
context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.START)
do_process_notification(notification)
def _handle_notification_type_host(self, context, notification):
host_status = notification.payload.get('host_status')
notification_status = fields.NotificationStatus.FINISHED
notification_event = notification.payload.get('event')
exception_info = None
if host_status.upper() != fields.HostStatusType.NORMAL:
# NOTE(shilpasd): Avoid host recovery for host_status other than
# 'NORMAL' otherwise it could lead to unsafe evacuation of
# instances running on the failed source host.
LOG.warning(
"Notification '%(uuid)s' ignored as host_status"
"is '%(host_status)s'", {
'uuid': notification.notification_uuid,
'host_status': host_status.upper()
})
notification_status = fields.NotificationStatus.IGNORED
elif notification_event.upper() == 'STARTED':
LOG.info(
"Notification type '%(type)s' received for host "
"'%(host_uuid)s' has been %(event)s.", {
'type': notification.type,
'host_uuid': notification.source_host_uuid,
'event': notification_event
})
elif notification_event.upper() == 'STOPPED':
host_obj = objects.Host.get_by_uuid(context,
notification.source_host_uuid)
host_name = host_obj.name
recovery_method = host_obj.failover_segment.recovery_method
# Mark host on_maintenance mode as True
update_data = {
'on_maintenance': True,
}
# Set reserved flag to False if this host is reserved
if host_obj.reserved:
update_data['reserved'] = False
host_obj.update(update_data)
host_obj.save()
reserved_host_list = None
if not recovery_method == (
fields.FailoverSegmentRecoveryMethod.AUTO):
reserved_host_object_list = objects.HostList.get_all(
context,
filters={
'failover_segment_id': host_obj.failover_segment.uuid,
'reserved': True,
'on_maintenance': False
})
# Create list of host name from reserved_host_object_list
reserved_host_list = [
host.name for host in reserved_host_object_list
]
try:
self.driver.execute_host_failure(
context,
host_name,
recovery_method,
notification.notification_uuid,
update_host_method=update_host_method,
reserved_host_list=reserved_host_list)
except exception.SkipHostRecoveryException:
notification_status = fields.NotificationStatus.FINISHED
except (exception.HostRecoveryFailureException,
exception.ReservedHostsUnavailable,
exception.MasakariException) as e:
notification_status = fields.NotificationStatus.ERROR
LOG.error(
"Failed to process notification '%(uuid)s'."
" Reason: %(error)s", {
"uuid": notification.notification_uuid,
"error": e.message
})
exception_info = e
else:
LOG.warning(
"Invalid event: %(event)s received for "
"notification type: %(type)s", {
'event': notification_event,
'type': notification.type
})
notification_status = fields.NotificationStatus.IGNORED
if exception_info:
tb = traceback.format_exc()
engine_utils.notify_about_notification_update(
context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.ERROR,
exception=str(exception_info),
tb=tb)
else:
engine_utils.notify_about_notification_update(
context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.END)
return notification_status
def _handle_notification_type_process(self, context, notification):
notification_status = fields.NotificationStatus.FINISHED
notification_event = notification.payload.get('event')
process_name = notification.payload.get('process_name')
exception_info = None
if notification_event.upper() == 'STARTED':
LOG.info("Notification type '%(type)s' received for host "
"'%(host_uuid)s': '%(process_name)s' has been "
"%(event)s.",
{'type': notification.type,
'host_uuid': notification.source_host_uuid,
'process_name': process_name,
'event': notification_event})
elif notification_event.upper() == 'STOPPED':
host_obj = objects.Host.get_by_uuid(
context, notification.source_host_uuid)
host_name = host_obj.name
# Mark host on_maintenance mode as True
update_data = {
'on_maintenance': True,
}
host_obj.update(update_data)
host_obj.save()
try:
self.driver.execute_process_failure(
context, process_name, host_name,
notification.notification_uuid)
except exception.SkipProcessRecoveryException as e:
notification_status = fields.NotificationStatus.FINISHED
except (exception.MasakariException,
exception.ProcessRecoveryFailureException) as e:
notification_status = fields.NotificationStatus.ERROR
LOG.error("Failed to process notification '%(uuid)s'."
" Reason: %(error)s",
{"uuid": notification.notification_uuid,
"error": e.message})
exception_info = e
else:
LOG.warning("Invalid event: %(event)s received for "
"notification type: %(notification_type)s",
{'event': notification_event,
'notification_type': notification.type})
notification_status = fields.NotificationStatus.IGNORED
if exception_info:
tb = traceback.format_exc()
engine_utils.notify_about_notification_update(context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.ERROR,
exception=str(exception_info),
tb=tb)
else:
engine_utils.notify_about_notification_update(context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.END)
return notification_status
def _process_notification(self, context, notification):
@utils.synchronized(notification.source_host_uuid, blocking=True)
def do_process_notification(notification):
LOG.info('Processing notification %(notification_uuid)s of '
'type: %(type)s',
{'notification_uuid': notification.notification_uuid,
'type': notification.type})
# Get notification from db
notification_db = objects.Notification.get_by_uuid(context,
notification.notification_uuid)
# NOTE(tpatil): To fix bug 1773132, process notification only
# if the notification status is New and the current notification
# from DB status is Not New to avoid recovering from failure twice
if (notification.status == fields.NotificationStatus.NEW and
notification_db.status != fields.NotificationStatus.NEW):
LOG.warning("Processing of notification is skipped to avoid "
"recovering from failure twice. "
"Notification received is '%(uuid)s' "
"and it's status is '%(new_status)s' and the "
"current status of same notification in db "
"is '%(old_status)s'",
{"uuid": notification.notification_uuid,
"new_status": notification.status,
"old_status": notification_db.status})
return
update_data = {
'status': fields.NotificationStatus.RUNNING,
}
notification.update(update_data)
notification.save()
if notification.type == fields.NotificationType.PROCESS:
notification_status = self._handle_notification_type_process(
context, notification)
elif notification.type == fields.NotificationType.VM:
notification_status = self._handle_notification_type_instance(
context, notification)
elif notification.type == fields.NotificationType.COMPUTE_HOST:
notification_status = self._handle_notification_type_host(
context, notification)
LOG.info("Notification %(notification_uuid)s exits with "
"status: %(status)s.",
{'notification_uuid': notification.notification_uuid,
'status': notification_status})
update_data = {
'status': notification_status
}
notification.update(update_data)
notification.save()
engine_utils.notify_about_notification_update(context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.START)
do_process_notification(notification)
def _handle_notification_type_host(self, context, notification):
notification_status = fields.NotificationStatus.FINISHED
notification_event = notification.payload.get('event')
exception_info = None
if notification_event.upper() == 'STARTED':
LOG.info("Notification type '%(type)s' received for host "
"'%(host_uuid)s' has been %(event)s.",
{'type': notification.type,
'host_uuid': notification.source_host_uuid,
'event': notification_event})
elif notification_event.upper() == 'STOPPED':
host_obj = objects.Host.get_by_uuid(
context, notification.source_host_uuid)
host_name = host_obj.name
recovery_method = host_obj.failover_segment.recovery_method
# Mark host on_maintenance mode as True
update_data = {
'on_maintenance': True,
}
# Set reserved flag to False if this host is reserved
if host_obj.reserved:
update_data['reserved'] = False
host_obj.update(update_data)
host_obj.save()
reserved_host_list = None
if not recovery_method == (
fields.FailoverSegmentRecoveryMethod.AUTO):
reserved_host_object_list = objects.HostList.get_all(
context, filters={
'failover_segment_id': host_obj.failover_segment_id,
'reserved': True,
'on_maintenance': False
})
# Create list of host name from reserved_host_object_list
reserved_host_list = [host.name for host in
reserved_host_object_list]
try:
self.driver.execute_host_failure(
context, host_name, recovery_method,
notification.notification_uuid,
update_host_method=update_host_method,
reserved_host_list=reserved_host_list)
except exception.SkipHostRecoveryException as e:
notification_status = fields.NotificationStatus.FINISHED
except (exception.HostRecoveryFailureException,
exception.ReservedHostsUnavailable,
exception.MasakariException) as e:
notification_status = fields.NotificationStatus.ERROR
LOG.error("Failed to process notification '%(uuid)s'."
" Reason: %(error)s",
{"uuid": notification.notification_uuid,
"error": e.message})
exception_info = e
else:
LOG.warning("Invalid event: %(event)s received for "
"notification type: %(type)s",
{'event': notification_event,
'type': notification.type})
notification_status = fields.NotificationStatus.IGNORED
if exception_info:
tb = traceback.format_exc()
engine_utils.notify_about_notification_update(context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.ERROR,
exception=str(exception_info),
tb=tb)
else:
engine_utils.notify_about_notification_update(context,
notification,
action=fields.EventNotificationAction.NOTIFICATION_PROCESS,
phase=fields.EventNotificationPhase.END)
return notification_status
