def _handle_notification_type_instance(self, context, notification): if not virt_events.is_valid_event(notification.payload): LOG.info( "Notification '%(uuid)s' received with payload " "%(payload)s is ignored.", { "uuid": notification.notification_uuid, "payload": notification.payload }) return fields.NotificationStatus.IGNORED notification_status = fields.NotificationStatus.FINISHED exception_info = None try: self.driver.execute_instance_failure( context, notification.payload.get('instance_uuid'), notification.notification_uuid) except exception.IgnoreInstanceRecoveryException as e: notification_status = fields.NotificationStatus.IGNORED exception_info = e except exception.SkipInstanceRecoveryException: notification_status = fields.NotificationStatus.FINISHED except (exception.MasakariException, exception.InstanceRecoveryFailureException) as e: notification_status = fields.NotificationStatus.ERROR LOG.error( "Failed to process notification '%(uuid)s'." " Reason: %(error)s", { "uuid": notification.notification_uuid, "error": e.message }) exception_info = e if exception_info: tb = traceback.format_exc() engine_utils.notify_about_notification_update( context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.ERROR, exception=str(exception_info), tb=tb) else: engine_utils.notify_about_notification_update( context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.END) return notification_status
def test_notify_about_notification_update(self, mock_from_exception, mock_NotificationApiPayload, mock_NotificationApiNotification, mock_NotificationPublisher, mock_EventType): mock_fault = mock.Mock() mock_from_exception.return_value = mock_fault mock_payload = mock.Mock() mock_NotificationApiPayload.return_value = mock_payload mock_engine_notification = mock.Mock() mock_NotificationApiNotification.return_value = ( mock_engine_notification) mock_engine_notification.emit.return_value = None mock_publisher = mock.Mock() mock_NotificationPublisher.return_value = mock_publisher mock_event_type = mock.Mock() mock_EventType.return_value = mock_event_type mock_context = mock.Mock() notification = notification_obj.Notification() action = fields.EventNotificationAction.NOTIFICATION_PROCESS phase = fields.EventNotificationPhase.ERROR e = Exception() engine_utils.notify_about_notification_update(mock_context, notification, action=action, phase=phase, exception=e) mock_from_exception.assert_called_once_with(e, None) mock_NotificationApiPayload.assert_called_once_with( notification=notification, fault=mock_fault) mock_NotificationApiNotification.assert_called_once_with( context=mock_context, priority=fields.EventNotificationPriority.ERROR, publisher=mock_publisher, event_type=mock_event_type, payload=mock_payload) mock_NotificationPublisher.assert_called_once_with( context=mock_context, host=socket.gethostname(), binary='masakari-engine') mock_engine_notification.emit.assert_called_once_with(mock_context)
def _handle_notification_type_instance(self, context, notification): if not virt_events.is_valid_event(notification.payload): LOG.info("Notification '%(uuid)s' received with payload " "%(payload)s is ignored.", {"uuid": notification.notification_uuid, "payload": notification.payload}) return fields.NotificationStatus.IGNORED notification_status = fields.NotificationStatus.FINISHED exception_info = None try: self.driver.execute_instance_failure( context, notification.payload.get('instance_uuid'), notification.notification_uuid) except exception.IgnoreInstanceRecoveryException as e: notification_status = fields.NotificationStatus.IGNORED exception_info = e except exception.SkipInstanceRecoveryException as e: notification_status = fields.NotificationStatus.FINISHED except (exception.MasakariException, exception.InstanceRecoveryFailureException) as e: notification_status = fields.NotificationStatus.ERROR LOG.error("Failed to process notification '%(uuid)s'." " Reason: %(error)s", {"uuid": notification.notification_uuid, "error": e.message}) exception_info = e if exception_info: tb = traceback.format_exc() engine_utils.notify_about_notification_update(context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.ERROR, exception=str(exception_info), tb=tb) else: engine_utils.notify_about_notification_update(context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.END) return notification_status
def test_notify_about_notification_update( self, mock_from_exception, mock_NotificationApiPayload, mock_NotificationApiNotification, mock_NotificationPublisher, mock_EventType): mock_fault = mock.Mock() mock_from_exception.return_value = mock_fault mock_payload = mock.Mock() mock_NotificationApiPayload.return_value = mock_payload mock_engine_notification = mock.Mock() mock_NotificationApiNotification.return_value = ( mock_engine_notification) mock_engine_notification.emit.return_value = None mock_publisher = mock.Mock() mock_NotificationPublisher.return_value = mock_publisher mock_event_type = mock.Mock() mock_EventType.return_value = mock_event_type mock_context = mock.Mock() notification = notification_obj.Notification() action = fields.EventNotificationAction.NOTIFICATION_PROCESS phase = fields.EventNotificationPhase.ERROR e = Exception() engine_utils.notify_about_notification_update(mock_context, notification, action=action, phase=phase, exception=e) mock_from_exception.assert_called_once_with(e, None) mock_NotificationApiPayload.assert_called_once_with( notification=notification, fault=mock_fault) mock_NotificationApiNotification.assert_called_once_with( context=mock_context, priority=fields.EventNotificationPriority.ERROR, publisher=mock_publisher, event_type=mock_event_type, payload=mock_payload) mock_NotificationPublisher.assert_called_once_with( context=mock_context, host=socket.gethostname(), binary='masakari-engine') mock_engine_notification.emit.assert_called_once_with(mock_context)
def _handle_notification_type_process(self, context, notification): notification_status = fields.NotificationStatus.FINISHED notification_event = notification.payload.get('event') process_name = notification.payload.get('process_name') exception_info = None if notification_event.upper() == 'STARTED': LOG.info( "Notification type '%(type)s' received for host " "'%(host_uuid)s': '%(process_name)s' has been " "%(event)s.", { 'type': notification.type, 'host_uuid': notification.source_host_uuid, 'process_name': process_name, 'event': notification_event }) elif notification_event.upper() == 'STOPPED': host_obj = objects.Host.get_by_uuid(context, notification.source_host_uuid) host_name = host_obj.name # Mark host on_maintenance mode as True update_data = { 'on_maintenance': True, } host_obj.update(update_data) host_obj.save() try: self.driver.execute_process_failure( context, process_name, host_name, notification.notification_uuid) except exception.SkipProcessRecoveryException: notification_status = fields.NotificationStatus.FINISHED except (exception.MasakariException, exception.ProcessRecoveryFailureException) as e: notification_status = fields.NotificationStatus.ERROR LOG.error( "Failed to process notification '%(uuid)s'." " Reason: %(error)s", { "uuid": notification.notification_uuid, "error": e.message }) exception_info = e else: LOG.warning( "Invalid event: %(event)s received for " "notification type: %(notification_type)s", { 'event': notification_event, 'notification_type': notification.type }) notification_status = fields.NotificationStatus.IGNORED if exception_info: tb = traceback.format_exc() engine_utils.notify_about_notification_update( context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.ERROR, exception=str(exception_info), tb=tb) else: engine_utils.notify_about_notification_update( context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.END) return notification_status
def _process_notification(self, context, notification): @utils.synchronized(notification.source_host_uuid, blocking=True) def do_process_notification(notification): LOG.info( 'Processing notification %(notification_uuid)s of ' 'type: %(type)s', { 'notification_uuid': notification.notification_uuid, 'type': notification.type }) # Get notification from db notification_db = objects.Notification.get_by_uuid( context, notification.notification_uuid) # NOTE(tpatil): To fix bug 1773132, process notification only # if the notification status is New and the current notification # from DB status is Not New to avoid recovering from failure twice if (notification.status == fields.NotificationStatus.NEW and notification_db.status != fields.NotificationStatus.NEW): LOG.warning( "Processing of notification is skipped to avoid " "recovering from failure twice. " "Notification received is '%(uuid)s' " "and it's status is '%(new_status)s' and the " "current status of same notification in db " "is '%(old_status)s'", { "uuid": notification.notification_uuid, "new_status": notification.status, "old_status": notification_db.status }) return update_data = { 'status': fields.NotificationStatus.RUNNING, } notification.update(update_data) notification.save() if notification.type == fields.NotificationType.PROCESS: notification_status = self._handle_notification_type_process( context, notification) elif notification.type == fields.NotificationType.VM: notification_status = self._handle_notification_type_instance( context, notification) elif notification.type == fields.NotificationType.COMPUTE_HOST: notification_status = self._handle_notification_type_host( context, notification) LOG.info( "Notification %(notification_uuid)s exits with " "status: %(status)s.", { 'notification_uuid': notification.notification_uuid, 'status': notification_status }) update_data = {'status': notification_status} notification.update(update_data) notification.save() engine_utils.notify_about_notification_update( context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.START) do_process_notification(notification)
def _handle_notification_type_host(self, context, notification): host_status = notification.payload.get('host_status') notification_status = fields.NotificationStatus.FINISHED notification_event = notification.payload.get('event') exception_info = None if host_status.upper() != fields.HostStatusType.NORMAL: # NOTE(shilpasd): Avoid host recovery for host_status other than # 'NORMAL' otherwise it could lead to unsafe evacuation of # instances running on the failed source host. LOG.warning( "Notification '%(uuid)s' ignored as host_status" "is '%(host_status)s'", { 'uuid': notification.notification_uuid, 'host_status': host_status.upper() }) notification_status = fields.NotificationStatus.IGNORED elif notification_event.upper() == 'STARTED': LOG.info( "Notification type '%(type)s' received for host " "'%(host_uuid)s' has been %(event)s.", { 'type': notification.type, 'host_uuid': notification.source_host_uuid, 'event': notification_event }) elif notification_event.upper() == 'STOPPED': host_obj = objects.Host.get_by_uuid(context, notification.source_host_uuid) host_name = host_obj.name recovery_method = host_obj.failover_segment.recovery_method # Mark host on_maintenance mode as True update_data = { 'on_maintenance': True, } # Set reserved flag to False if this host is reserved if host_obj.reserved: update_data['reserved'] = False host_obj.update(update_data) host_obj.save() reserved_host_list = None if not recovery_method == ( fields.FailoverSegmentRecoveryMethod.AUTO): reserved_host_object_list = objects.HostList.get_all( context, filters={ 'failover_segment_id': host_obj.failover_segment.uuid, 'reserved': True, 'on_maintenance': False }) # Create list of host name from reserved_host_object_list reserved_host_list = [ host.name for host in reserved_host_object_list ] try: self.driver.execute_host_failure( context, host_name, recovery_method, notification.notification_uuid, update_host_method=update_host_method, reserved_host_list=reserved_host_list) except exception.SkipHostRecoveryException: notification_status = fields.NotificationStatus.FINISHED except (exception.HostRecoveryFailureException, exception.ReservedHostsUnavailable, exception.MasakariException) as e: notification_status = fields.NotificationStatus.ERROR LOG.error( "Failed to process notification '%(uuid)s'." " Reason: %(error)s", { "uuid": notification.notification_uuid, "error": e.message }) exception_info = e else: LOG.warning( "Invalid event: %(event)s received for " "notification type: %(type)s", { 'event': notification_event, 'type': notification.type }) notification_status = fields.NotificationStatus.IGNORED if exception_info: tb = traceback.format_exc() engine_utils.notify_about_notification_update( context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.ERROR, exception=str(exception_info), tb=tb) else: engine_utils.notify_about_notification_update( context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.END) return notification_status
def _handle_notification_type_process(self, context, notification): notification_status = fields.NotificationStatus.FINISHED notification_event = notification.payload.get('event') process_name = notification.payload.get('process_name') exception_info = None if notification_event.upper() == 'STARTED': LOG.info("Notification type '%(type)s' received for host " "'%(host_uuid)s': '%(process_name)s' has been " "%(event)s.", {'type': notification.type, 'host_uuid': notification.source_host_uuid, 'process_name': process_name, 'event': notification_event}) elif notification_event.upper() == 'STOPPED': host_obj = objects.Host.get_by_uuid( context, notification.source_host_uuid) host_name = host_obj.name # Mark host on_maintenance mode as True update_data = { 'on_maintenance': True, } host_obj.update(update_data) host_obj.save() try: self.driver.execute_process_failure( context, process_name, host_name, notification.notification_uuid) except exception.SkipProcessRecoveryException as e: notification_status = fields.NotificationStatus.FINISHED except (exception.MasakariException, exception.ProcessRecoveryFailureException) as e: notification_status = fields.NotificationStatus.ERROR LOG.error("Failed to process notification '%(uuid)s'." " Reason: %(error)s", {"uuid": notification.notification_uuid, "error": e.message}) exception_info = e else: LOG.warning("Invalid event: %(event)s received for " "notification type: %(notification_type)s", {'event': notification_event, 'notification_type': notification.type}) notification_status = fields.NotificationStatus.IGNORED if exception_info: tb = traceback.format_exc() engine_utils.notify_about_notification_update(context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.ERROR, exception=str(exception_info), tb=tb) else: engine_utils.notify_about_notification_update(context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.END) return notification_status
def _process_notification(self, context, notification): @utils.synchronized(notification.source_host_uuid, blocking=True) def do_process_notification(notification): LOG.info('Processing notification %(notification_uuid)s of ' 'type: %(type)s', {'notification_uuid': notification.notification_uuid, 'type': notification.type}) # Get notification from db notification_db = objects.Notification.get_by_uuid(context, notification.notification_uuid) # NOTE(tpatil): To fix bug 1773132, process notification only # if the notification status is New and the current notification # from DB status is Not New to avoid recovering from failure twice if (notification.status == fields.NotificationStatus.NEW and notification_db.status != fields.NotificationStatus.NEW): LOG.warning("Processing of notification is skipped to avoid " "recovering from failure twice. " "Notification received is '%(uuid)s' " "and it's status is '%(new_status)s' and the " "current status of same notification in db " "is '%(old_status)s'", {"uuid": notification.notification_uuid, "new_status": notification.status, "old_status": notification_db.status}) return update_data = { 'status': fields.NotificationStatus.RUNNING, } notification.update(update_data) notification.save() if notification.type == fields.NotificationType.PROCESS: notification_status = self._handle_notification_type_process( context, notification) elif notification.type == fields.NotificationType.VM: notification_status = self._handle_notification_type_instance( context, notification) elif notification.type == fields.NotificationType.COMPUTE_HOST: notification_status = self._handle_notification_type_host( context, notification) LOG.info("Notification %(notification_uuid)s exits with " "status: %(status)s.", {'notification_uuid': notification.notification_uuid, 'status': notification_status}) update_data = { 'status': notification_status } notification.update(update_data) notification.save() engine_utils.notify_about_notification_update(context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.START) do_process_notification(notification)
def _handle_notification_type_host(self, context, notification): notification_status = fields.NotificationStatus.FINISHED notification_event = notification.payload.get('event') exception_info = None if notification_event.upper() == 'STARTED': LOG.info("Notification type '%(type)s' received for host " "'%(host_uuid)s' has been %(event)s.", {'type': notification.type, 'host_uuid': notification.source_host_uuid, 'event': notification_event}) elif notification_event.upper() == 'STOPPED': host_obj = objects.Host.get_by_uuid( context, notification.source_host_uuid) host_name = host_obj.name recovery_method = host_obj.failover_segment.recovery_method # Mark host on_maintenance mode as True update_data = { 'on_maintenance': True, } # Set reserved flag to False if this host is reserved if host_obj.reserved: update_data['reserved'] = False host_obj.update(update_data) host_obj.save() reserved_host_list = None if not recovery_method == ( fields.FailoverSegmentRecoveryMethod.AUTO): reserved_host_object_list = objects.HostList.get_all( context, filters={ 'failover_segment_id': host_obj.failover_segment_id, 'reserved': True, 'on_maintenance': False }) # Create list of host name from reserved_host_object_list reserved_host_list = [host.name for host in reserved_host_object_list] try: self.driver.execute_host_failure( context, host_name, recovery_method, notification.notification_uuid, update_host_method=update_host_method, reserved_host_list=reserved_host_list) except exception.SkipHostRecoveryException as e: notification_status = fields.NotificationStatus.FINISHED except (exception.HostRecoveryFailureException, exception.ReservedHostsUnavailable, exception.MasakariException) as e: notification_status = fields.NotificationStatus.ERROR LOG.error("Failed to process notification '%(uuid)s'." " Reason: %(error)s", {"uuid": notification.notification_uuid, "error": e.message}) exception_info = e else: LOG.warning("Invalid event: %(event)s received for " "notification type: %(type)s", {'event': notification_event, 'type': notification.type}) notification_status = fields.NotificationStatus.IGNORED if exception_info: tb = traceback.format_exc() engine_utils.notify_about_notification_update(context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.ERROR, exception=str(exception_info), tb=tb) else: engine_utils.notify_about_notification_update(context, notification, action=fields.EventNotificationAction.NOTIFICATION_PROCESS, phase=fields.EventNotificationPhase.END) return notification_status