def testMisc(self):
''' Test miscellaneous Location capabilities '''
l1 = Location("TS",'foo')
l2 = Location("TC",'R-H-K8-S')
self.assertNotEqual(hash(l1),hash(l2))
self.assertEquals(hash(l1),hash(l1))
self.assertEqual(l1.get_id(),'TS')
self.assertEqual(l2.get_id(),'TC')
def testComplexMatchBad(self):
''' Test matching scopes with complex locations and errors '''
simple1 = Location('TS','a.b.c')
simple2 = Location('TS','a.b.z')
complex1 = Location('TC',"R-H-PS9")
# Can't mix location types
self.assertFalse(simple1.match(complex1))
# Can't use an invalid location component
self.assertRaises(KeyError, simple1.match, simple2,'mom')
def testLocation(self):
''' Test location is handled correctly '''
s = struct.Struct("3p256p")
fmt = 0x5445535400000003
loc = Location('C','MB-SL3-ET1-PT2')
raw_data = s.pack(loc.get_id(),loc.get_location())
ed = ExtensionData(fmt,None, raw_data)
self.assertEqual(raw_data,ed['raw_data'])
self.assertEqual(loc,ed['neighbor'])
self.assertEqual(len(ed.keys()),2)
self.assertEqual(fmt,ed.get_format())
def testNewLocationByScope(self):
''' Test creating a new location with a given scope '''
complex1 = Location('TC',"R-H-K8-S")
complex2 = complex1.new_location_by_scope('kilroy')
self.assertEquals(complex2,Location('TC','R-H-K8'))
complex3 = complex1.new_location_by_scope('home')
complex4 = complex2.new_location_by_scope('home')
self.assertEquals(complex3,complex4)
self.assertRaises(KeyError, complex1.new_location_by_scope, 'bad')
# Scoping lower than the id will just return the id
complex5 = complex3.new_location_by_scope('kilroy')
self.assertEquals(complex3,complex5)
def testComplexBadDefered(self):
''' Test invalid complex locations with validation deferred '''
cbad1 = Location('TZ', 'R-H')
self.assertEqual('TZ: R-H', str(cbad1))
self.assertRaises(KeyError, cbad1.get_substitution_dict)
self.assertRaises(KeyError, cbad1.get_comp_value, 'dummy')
self.assertRaises(KeyError, cbad1.new_location_by_scope, 'dummy')
self.assertEqual('TZ', cbad1.get_id())
self.assertEqual('R-H', cbad1.get_location())
cbad2 = Location('TZ', 'R-H')
cbad3 = Location('TZ', 'R-I')
self.assertTrue(cbad1 == cbad2)
self.assertFalse(cbad1 == cbad3)
self.assertTrue(cbad1.match(cbad2, None))
self.assertRaises(KeyError, cbad1.match, cbad2, 'dummy')
return
def has_common_location(self, loc, alert_time, query, cursor):
''' Query alerts for the common location to indicate whether or not to send an alert.
'''
locParent, locParent_list = self.get_loc_parent(loc)
# Query for the number of alerts for the same parent's location
query = query.replace('LOCATION',loc.get_location())
query = query.replace('PLOC',locParent)
query = query.replace('ALERT_TIME', alert_time)
query = query.replace('WINDOW', self.window_time)
cursor.execute(query)
registry.get_logger().info("Trying to match parent: %s from query: %s", locParent, query)
# Send a common alert if ther are alerts with common location
# Start counting from the current alert (include current alert)
loc_type = loc.get_id()
count = 1
rows = cursor.fetchall()
for r in rows:
r_loc = r[0].strip()
r_loc_object = Location(loc_type, r_loc)
r_loc_parent, r_loc_parent_list = self.get_loc_parent(r_loc_object)
if r_loc_parent == locParent:
registry.get_logger().info("Match: %s with rec: %s", locParent, r_loc_parent)
count += 1
if count >= self.threshold:
return True
else:
registry.get_logger().info("No match: %s with rec: %s", locParent, r_loc_parent)
return False
def testSubstitutionDict(self):
''' Test getting the substitution dictionary '''
l1 = Location("TS",'foo')
sd1 = l1.get_substitution_dict()
self.assertEquals(len(sd1.keys()), 1)
self.assertTrue('parent' in sd1)
self.assertEquals(sd1['parent'], 'foo')
l2 = Location("TC",'R-H-K8-S')
sd2 = l2.get_substitution_dict()
self.assertEquals(len(sd2.keys()), 4)
self.assertTrue('kilroy' in sd2)
self.assertEquals(sd2['kilroy'], 'K8')
self.assertTrue('src' in sd2)
self.assertEquals(sd2['src'], 'S')
self.assertTrue('top' in sd2)
self.assertEquals(sd2['top'], 'R')
self.assertTrue('home' in sd2)
self.assertEquals(sd2['home'], 'H')
return
def testToFromDict(self):
''' Test to and from dict is handled correctly '''
s = struct.Struct("3p256p")
fmt = 0x5445535400000003
loc = Location('C','MB-SL3-ET1-PT2')
raw_data = s.pack(loc.get_id(),loc.get_location())
ed = ExtensionData(fmt,None, raw_data)
self.assertEqual(raw_data,ed['raw_data'])
self.assertEqual(loc,ed['neighbor'])
self.assertEqual(len(ed.keys()),2)
self.assertEqual(fmt,ed.get_format())
tmp_dict = ed.write_to_dictionary()
new_ed = ExtensionData(fmt, None, None, tmp_dict)
self.assertEqual(raw_data, new_ed['raw_data'])
self.assertEqual(loc, new_ed['neighbor'])
self.assertEqual(len(new_ed.keys()), 2)
self.assertEqual(fmt, new_ed.get_format())
return
def testSimpleMatch(self):
simple1 = Location('TS','a.b.c')
simple2 = Location('TS','a.b.z')
simple3 = Location('TS','x.y.z')
self.assertTrue(simple1.match(simple1))
self.assertFalse(simple1.match(simple2))
self.assertTrue(simple1.match(simple2,'child'))
self.assertTrue(simple1.match(simple2,'parent'))
self.assertFalse(simple1.match(simple3))
self.assertFalse(simple1.match(simple3,'child'))
self.assertFalse(simple1.match(simple3,'parent'))
def testSimpleGood(self):
''' Test good path simple locations '''
simple1 = Location('TS','a')
self.assertEqual(simple1.get_location(),'a')
simple2 = Location('TS','a.b')
self.assertEqual(simple2.get_location(),'a.b')
simple3 = Location('TS','a.b.c')
self.assertEqual(simple3.get_location(),'a.b.c')
def send_common_alert(self, loc, cur_alert_recid, event, alert_time, dup_query, cursor):
''' Send an alert for the common location.
'''
# Close current alert prior to creating a new common alert
registry.get_logger().info('Closing current alert recid %d prior to creating a common mode alert', cur_alert_recid)
registry.get_service(SERVICE_ALERT_MGR).close(cur_alert_recid)
# Get the location
loc_name = self.get_loc_name(loc)
loc_type = loc.get_id()
loc_parent, loc_parent_list = self.get_loc_parent(loc)
loc_parent_object = Location(loc_type, loc_parent)
# Removed the duplicate check that was here -- this has already been determined
# Fill in alert info
reason = self.reason.replace('LOC_NAME', loc_name)
reason = reason.replace('LOC_PARENT', loc_parent)
recommendation = self.recommendation.replace('LOC_PARENT', loc_parent)
alert_dict = {alert.ALERT_ATTR_SEVERITY:self.severity,
alert.ALERT_ATTR_URGENCY:'I',
alert.ALERT_ATTR_EVENT_LOC_OBJECT:loc_parent_object,
alert.ALERT_ATTR_RECOMMENDATION:recommendation,
alert.ALERT_ATTR_REASON:reason,
alert.ALERT_ATTR_RAW_DATA:'No raw data',
alert.ALERT_ATTR_SRC_NAME:self.get_name(),
alert.ALERT_ATTR_CONDITION_EVENTS:set((event,))
}
# Get the alert manager to create/allocate/commit the alert
alertMgr = registry.get_service(registry.SERVICE_ALERT_MGR)
bg_alert = alertMgr.allocate(self.alertId, in_dict=alert_dict)
alertMgr.commit(bg_alert, disable_dup=False)
# Now the alert is created, need to put it in the queue so that it can be analyzed
# by alert analyzer (instead of sending it - send_alert, which will get reported
# through the pipeline right away)
registry.get_logger().info("Put alertId = %s with event recid = %d on the alert analyzer queue", self.alertId, event.get_rec_id())
registry.get_service(SERVICE_ALERT_ANALYZER_Q).put(bg_alert)
return
def testComplexGood(self):
''' Test good path complex locations '''
# Minimum tree specified
complex1 = Location('TC',"R")
self.assertEqual(complex1.get_location(),'R')
complex2 = Location('TC',"R-H")
self.assertEqual(complex2.get_location(),'R-H')
# No value allowed
complex3 = Location('TC',"R-H-JC")
self.assertEqual(complex3.get_location(),'R-H-JC')
# min-only specified
complex4 = Location('TC',"R-H-MM12345")
self.assertEqual(complex4.get_location(),'R-H-MM12345')
complex4a = Location('TC',"R-H-MM"+str(sys.maxint))
self.assertEqual(complex4a.get_location(),'R-H-MM'+str(sys.maxint))
# Range specified
complex5 = Location('TC',"R-H-PS9")
self.assertEqual(complex5.get_location(),'R-H-PS9')
# max-only specified
complex6 = Location('TC',"R-H-K0")
self.assertEqual(complex6.get_location(),'R-H-K0')
# Full tree specified
complex7 = Location('TC',"R-H-K20-S")
self.assertEqual(complex7.get_location(),'R-H-K20-S')
def testComplexComponentValues(self):
''' Test getting the value of a component for complex locations '''
complex1 = Location('TC',"R-H-K8-S")
self.assertEquals(complex1.get_comp_value('kilroy'),'8')
def testSimpleComponentValues(self):
''' Test getting the value of a component for simple locations '''
simple1 = Location('TS','a.b.c')
self.assertEquals(simple1.get_comp_value('grandchild'),'c')
self.assertEquals(simple1.get_comp_value('child'),'b')
self.assertEquals(simple1.get_comp_value('parent'),'a')
def testComplexMatch(self):
''' Test matching scopes with complex locations '''
complex1 = Location('TC',"R-H-PS9")
complex2 = Location('TC',"R-H-K8-S")
complex3 = Location('TC',"R-H-K9-S")
self.assertTrue(complex1.match(complex1))
self.assertTrue(complex1.match(complex1))
self.assertFalse(complex1.match(complex2))
self.assertTrue(complex1.match(complex2,'home'))
self.assertTrue(complex1.match(complex2,'top'))
self.assertFalse(complex2.match(complex3))
# Can do different length w/o problems
complex2 = Location('TC','R-H')
complex3 = Location('TC','R-H-K9-S')
self.assertFalse(complex2.match(complex3,'kilroy'))
self.assertFalse(complex3.match(complex2,'kilroy'))
def analyze_alert(self, alert):
'''Analyze an alert
'''
alert_recId = alert.get_rec_id()
alert_id = alert.get_incident_id()
loc_type = alert.event_loc.get_id()
location = alert.event_loc.get_location()
#alert_msgId = alert.get_incident_id()
registry.get_logger().info('Analyzing alert id %d loc_type: %s: %s', alert_recId, loc_type, location)
# There should only be one condition event associated with the alert.
events = alert.condition_events
if len(events) == 0:
registry.get_logger().error('No event associated with the alert recid %d', alert_recId)
registry.get_service(SERVICE_ALERT_DELIVERY_Q).put(alert)
return
event = events.pop()
if (alert_id == 'BQL01'):
# No need to analyze BQL01 alerts, just pass it to the delivery queue
registry.get_logger().info('Nothing to analyze for alert id %s ', alert_id)
registry.get_service(SERVICE_ALERT_DELIVERY_Q).put(alert)
return
# Get the location
loc = Location(loc_type, location)
locName = self.get_loc_name(loc)
# No need to analyze alert with rack location
alert_time = str(alert.get_time_occurred())
if locName == 'rack':
registry.get_logger().info('Nothing to analyze for alert recid %d with rack location', alert_recId)
registry.get_service(SERVICE_ALERT_DELIVERY_Q).put(alert)
return
# Find out if there are other alerts with the same block id (for ENDJOB01 and THRES01)
dup_qry = ''
if (alert_id == 'ENDJOB01' or alert_id == 'THRES01'):
if event.raw_data['block'] is None:
event_block = None
else:
event_block = event.raw_data['block'].strip()
if event_block is None or event_block == BGQ_EVENT_NULL_BLOCK:
# Found no prior alert with the same block id, pass current alert to the delivery queue
registry.get_logger().info('No block id for alert id %d, no common alert generated for block: %s', alert_recId, event_block)
registry.get_service(SERVICE_ALERT_DELIVERY_Q).put(alert)
return
same_block = False
# Get db connection needed for query
dbi = registry.get_service(SERVICE_DB_INTERFACE)
dbConn = dbi.get_connection()
cursor = dbConn.cursor()
if (alert_id == 'ENDJOB01'):
# For ENDJOB01, look for alert id HWERR01 or COMMON01 with the same block id
same_block = self.has_matching_blockId(event_block, alert_time, cursor)
else:
# For THRES0101, look for alert id HWERR01 or COMMON01 or ENDJOB01 with the same block id
same_block = self.has_matching_blockId(event_block, alert_time, cursor)
if same_block:
# Found prior alert with the same block id, close current alert
registry.get_logger().info('Closing current alert recid %d due to prior alert with the same block id', alert_recId)
registry.get_service(SERVICE_ALERT_MGR).close(alert_recId)
else:
# Found no prior alert with the same block id, pass current alert to the delivery queue
registry.get_logger().info('No common block id found for alert id %d within the last %s', alert_recId, self.window_time)
registry.get_service(SERVICE_ALERT_DELIVERY_Q).put(alert)
return
elif (alert_id == 'BQL01'):
# No need to analyze BQL01 alerts, just pass it to the delivery queue
registry.get_logger().info('Nothing to analyze for alert id %s.', alert_id)
registry.get_service(SERVICE_ALERT_DELIVERY_Q).put(alert)
return
# The following will handle the rest of the alert ids (HWERR01 or COMMON01).
# Find out if there is common mode alert already exist for the same location or higher hierarchy
loc_parent, loc_parent_list = self.get_loc_parent(loc)
loc_qry = '('
idx = 0
for pLoc in loc_parent_list:
if idx != 0:
loc_qry += " or "
loc_qry += " \"event_loc\" like '" + pLoc + "'"
idx += 1
dup_qry2 = self.dup_query + loc_qry + ")"
loc_qry += " or \"event_loc\" like '" + location + "')"
dup_qry = self.dup_query + loc_qry
dbi = registry.get_service(SERVICE_DB_INTERFACE)
dbConn = dbi.get_connection()
cursor = dbConn.cursor()
dup = self.has_duplicate(alert_time, dup_qry, cursor)
if dup:
# Found prior alert with the same block id, close current alert
registry.get_logger().info('Closing current alert recid %d due to prior alert with same common location', alert_recId)
registry.get_service(SERVICE_ALERT_MGR).close(alert_recId)
return
# Look for a common hardware problem if there are multiple alerts for different location
# on the same hardware.
sendAlert = self.has_common_location(loc, alert_time, self.query, cursor)
if sendAlert:
# Send commmon alert
self.send_common_alert(loc, alert_recId, event, alert_time, dup_qry2, cursor)
else:
# Pass current alert to the delivery queue
registry.get_logger().info('No common location for %s found for alert id: %d within the last %s ', location, alert_recId, self.window_time)
registry.get_service(SERVICE_ALERT_DELIVERY_Q).put(alert)
return