class RecoveryActionEvent: """ Action Event. This class implements an action event object, that is delegated to components for taking the action. """ def __init__(self, healthevent: HealthEvent): """ Init method. """ payload = { HealthAttr.SOURCE.value: healthevent.source, HealthAttr.CLUSTER_ID.value: healthevent.cluster_id, HealthAttr.SITE_ID.value: healthevent.site_id, HealthAttr.RACK_ID.value: healthevent.rack_id, HealthAttr.STORAGESET_ID.value: healthevent.storageset_id, HealthAttr.NODE_ID.value: healthevent.node_id, HealthAttr.RESOURCE_TYPE.value: healthevent.resource_type, HealthAttr.RESOURCE_ID.value: healthevent.resource_id, HealthAttr.RESOURCE_STATUS.value: healthevent.event_type } self.event = HEvent(**payload) self.event.set_specific_info(healthevent.specific_info) def get_event(self): return self.event
class NodeEvent(HaEvent): def __init__(self): logging.debug('Inside NodeEvent') def create_event(self, node_id, node_name, health_status): logging.debug('Inside create_event of NodeEvent') self.payload = { HealthAttr.SOURCE.value: HEALTH_EVENT_SOURCES.HARE.value, HealthAttr.CLUSTER_ID.value: NOT_DEFINED, HealthAttr.SITE_ID.value: NOT_DEFINED, HealthAttr.RACK_ID.value: NOT_DEFINED, HealthAttr.STORAGESET_ID.value: NOT_DEFINED, HealthAttr.NODE_ID.value: node_id, HealthAttr.RESOURCE_TYPE.value: 'node', HealthAttr.RESOURCE_ID.value: node_id, HealthAttr.RESOURCE_STATUS.value: health_status, HealthAttr.SPECIFIC_INFO.value: {} } self.event = HealthEvent(**self.payload) # 'specific_info' is resource type specific information. # For e.g. incase of Node 'generation_id' will be pod name self.event.set_specific_info({"generation_id": node_name}) return self.event.json
def test_health_event_set(self): """Create of object HealthEvent, Test set attributes value.""" he = HealthEvent() self.assertTrue('header>version' in he.get_keys()) self.assertTrue('payload>source' in he.get_keys()) he.set(f'{HealthAttr.STORAGESET_ID}', '1') he.set(f'{HealthAttr.NODE_ID}', '3') self.assertEqual(he.get(f'payload>{HealthAttr.STORAGESET_ID}'), '1') self.assertEqual(he.get(f'payload>{HealthAttr.NODE_ID}'), '3') # Other attributes remain empty strings if not set self.assertEqual(he.get(f'payload>{HealthAttr.RESOURCE_ID}'), '')
def _create_health_alert(self, res_type, res_name, health_status, generation_id): """ Instantiates Event class which creates Health event object with necessary attributes to pass it for further processing. Args: res_type: resource_type for which event needs to be created. Ex: node, disk res_name: actual resource name. Ex: machine_id in case of node alerts health_status: health of that resource. Ex: online, failed generation_id: name of the node in case of node alert """ self.payload[HealthAttr.RESOURCE_TYPE.value] = res_type self.payload[HealthAttr.RESOURCE_ID.value] = res_name self.payload[HealthAttr.NODE_ID.value] = res_name self.payload[HealthAttr.RESOURCE_STATUS.value] = health_status # Create event schema object with payload data also # can change value of payload attributes with set function self.event = HealthEvent(**self.payload) self.event.set_specific_info({"generation_id": generation_id}) return self.event.json
def __init__(self, healthevent: HealthEvent): """ Init method. """ payload = { HealthAttr.SOURCE.value: healthevent.source, HealthAttr.CLUSTER_ID.value: healthevent.cluster_id, HealthAttr.SITE_ID.value: healthevent.site_id, HealthAttr.RACK_ID.value: healthevent.rack_id, HealthAttr.STORAGESET_ID.value: healthevent.storageset_id, HealthAttr.NODE_ID.value: healthevent.node_id, HealthAttr.RESOURCE_TYPE.value: healthevent.resource_type, HealthAttr.RESOURCE_ID.value: healthevent.resource_id, HealthAttr.RESOURCE_STATUS.value: healthevent.event_type } self.event = HEvent(**payload) self.event.set_specific_info(healthevent.specific_info)
def test_health_event_id(self): """Check event id is not floating value.""" he = HealthEvent() self.assertFalse('.' in he.get(f'header>{EventAttr.EVENT_ID}'))
def test_health_event_create(self): """Check by passing values of attributes during creation of object.""" health_attrs = { f'{HealthAttr.SOURCE}': 's1', f'{HealthAttr.CLUSTER_ID}': '1234', f'{HealthAttr.SITE_ID}': '1', f'{HealthAttr.RACK_ID}': '1' } he = HealthEvent(**health_attrs) self.assertEqual(he.get(f'payload>{HealthAttr.SOURCE}'), 's1') self.assertEqual(he.get(f'payload>{HealthAttr.CLUSTER_ID}'), '1234') self.assertEqual(he.get(f'payload>{HealthAttr.SITE_ID}'), '1') self.assertEqual(he.get(f'payload>{HealthAttr.RACK_ID}'), '1') # Way to set specific info he.set_specific_info({'a': 'a1'}) self.assertEqual(he.get(f'payload>{HealthAttr.SPECIFIC_INFO}>a'), 'a1') # One more way to set specific attr he.set_specific_attr('a', 'a2') self.assertEqual(he.get(f'payload>{HealthAttr.SPECIFIC_INFO}>a'), 'a2') # Other attributes remain empty strings if not set self.assertEqual(he.get(f'payload>{HealthAttr.RESOURCE_ID}'), '')
def publish(args: argparse.Namespace) -> None: """ publishes the message on the message bus. Args: args: parsed argument conf_store: ConftStoreSearch object """ try: with open(args.file, 'r') as fi: events_dict = json.load(fi) if _events_key in events_dict.keys(): ConfigManager.init(None) MessageBus.init() message_type = Conf.get( const.HA_GLOBAL_INDEX, f'FAULT_TOLERANCE{const._DELIM}message_type') message_producer = MessageBus.get_producer( "health_event_generator", message_type) cluster_id = Conf.get( const.HA_GLOBAL_INDEX, f'COMMON_CONFIG{const._DELIM}cluster_id') site_id = Conf.get(const.HA_GLOBAL_INDEX, f'COMMON_CONFIG{const._DELIM}site_id') rack_id = Conf.get(const.HA_GLOBAL_INDEX, f'COMMON_CONFIG{const._DELIM}rack_id') storageset_id = '1' # TODO: Read from config when available. for _, value in events_dict[_events_key].items(): resource_type = value[_resource_type_key] resource_type_list = Conf.get( const.HA_GLOBAL_INDEX, f"CLUSTER{const._DELIM}resource_type") if resource_type not in resource_type_list: raise Exception( f'Invalid resource_type: {resource_type}') resource_status = value[_resource_status_key] status_supported = False for status in list(HEALTH_STATUSES): if resource_status == status.value: status_supported = True break if status_supported is False: raise Exception( f'Invalid resource_status: {resource_status}') payload = { f'{HealthAttr.SOURCE}': value[_source_key], f'{HealthAttr.CLUSTER_ID}': cluster_id, f'{HealthAttr.SITE_ID}': site_id, f'{HealthAttr.RACK_ID}': rack_id, f'{HealthAttr.STORAGESET_ID}': storageset_id, f'{HealthAttr.NODE_ID}': value[_node_id_key], f'{HealthAttr.RESOURCE_TYPE}': resource_type, f'{HealthAttr.RESOURCE_ID}': value[_resource_id_key], f'{HealthAttr.RESOURCE_STATUS}': resource_status } health_event = HealthEvent(**payload) health_event.set_specific_info(value[_specific_info_key]) print(f"Publishing health event {health_event.json}") message_producer.publish(health_event.json) if _delay_key in events_dict.keys(): print( f"Sleeping for {events_dict[_delay_key]} seconds") time.sleep(events_dict[_delay_key]) except Exception as err: sys.stderr.write(f"Health event generator failed. Error: {err}\n") return errno.EINVAL
class PodEventParser(ObjectParser): def __init__(self): super().__init__() # Note: The below type is not a Kubernetes 'node'. # in cortx cluster the pod is called or considered as a node. # hence while sending alert to cortx, below type is set to 'node'. self._type = 'node' def _create_health_alert(self, res_type, res_name, health_status, generation_id): """ Instantiates Event class which creates Health event object with necessary attributes to pass it for further processing. Args: res_type: resource_type for which event needs to be created. Ex: node, disk res_name: actual resource name. Ex: machine_id in case of node alerts health_status: health of that resource. Ex: online, failed generation_id: name of the node in case of node alert """ self.payload[HealthAttr.RESOURCE_TYPE.value] = res_type self.payload[HealthAttr.RESOURCE_ID.value] = res_name self.payload[HealthAttr.NODE_ID.value] = res_name self.payload[HealthAttr.RESOURCE_STATUS.value] = health_status # Create event schema object with payload data also # can change value of payload attributes with set function self.event = HealthEvent(**self.payload) self.event.set_specific_info({"generation_id": generation_id}) return self.event.json def parse(self, an_event, cached_state): resource_type = self._type raw_object = an_event[K8SEventsConst.RAW_OBJECT] labels = raw_object[K8SEventsConst.METADATA][K8SEventsConst.LABELS] if K8SEventsConst.MACHINEID in labels: resource_name = labels[K8SEventsConst.MACHINEID] if K8SEventsConst.TYPE in an_event: event_type = an_event[K8SEventsConst.TYPE] # Actual physical host information is not needed now. # If required, can be available in K8SEventsConst.SPEC. # So, removing it from now. if K8SEventsConst.NAME in raw_object[K8SEventsConst.METADATA]: generation_id = raw_object[K8SEventsConst.METADATA][ K8SEventsConst.NAME] ready_status = None try: for a_condition in raw_object[K8SEventsConst.STATUS][ K8SEventsConst.CONDITIONS]: if a_condition[K8SEventsConst.TYPE] == K8SEventsConst.READY: ready_status = a_condition[K8SEventsConst.STATUS] except Exception as e: Log.debug(f"Exception received during parsing {e}") if ready_status is None: Log.debug(f"ready_status is None for pod resource {resource_name}") cached_state[resource_name] = ready_status return (None, None) if an_event[K8SEventsConst.TYPE] == EventStates.ADDED: cached_state[resource_name] = ready_status.lower() if ready_status.lower() != K8SEventsConst.true: Log.debug( f"[EventStates ADDED] No change detected for pod resource {resource_name}" ) return (None, None) else: event_type = AlertStates.ONLINE health_alert = self._create_health_alert( resource_type, resource_name, event_type, generation_id) return health_alert, self.event if event_type == EventStates.MODIFIED: if resource_name in cached_state: if cached_state[ resource_name] != K8SEventsConst.true and ready_status.lower( ) == K8SEventsConst.true: cached_state[resource_name] = ready_status.lower() event_type = AlertStates.ONLINE health_alert = self._create_health_alert( resource_type, resource_name, event_type, generation_id) return health_alert, self.event elif cached_state[ resource_name] == K8SEventsConst.true and ready_status.lower( ) != K8SEventsConst.true: cached_state[resource_name] = ready_status.lower() event_type = AlertStates.FAILED health_alert = self._create_health_alert( resource_type, resource_name, event_type, generation_id) return health_alert, self.event else: Log.debug( f"[EventStates MODIFIED] No change detected for pod resource {resource_name}" ) return (None, None) else: Log.debug( f"[EventStates MODIFIED] No cached state detected for pod resource {resource_name}" ) return (None, None) # Handle DELETED event - Not required for Cortx return (None, None)