class WorkerThread(threading.Thread):
def __init__(self, mq, queue):
threading.Thread.__init__(self)
LOG.debug('Initialising %s...', self.getName())
self.input_queue = queue # internal queue
self.mq = mq # message broker
self.db = Mongo() # mongo database
def run(self):
while True:
LOG.debug('Waiting on input queue...')
item = self.input_queue.get()
if not item:
LOG.info('%s is shutting down.', self.getName())
break
# Handle heartbeats
if item.get_type() == 'Heartbeat':
LOG.debug('update heartbeat')
#self.hb.update_hb(alert) # TODO(nsatterl): rename alert to payload or data or something
try:
self.db.update(
{"origin": alert['origin']},
{"origin": alert['origin'], "version": alert['version'], "createTime": createTime,
"receiveTime": receiveTime},
True)
except Exception, e:
LOG.error('Update failed: %s', e)
sys.exit(1)
LOG.info('%s : heartbeat from %s', alert['id'], alert['origin'])
continue
alert = item.get_body()
print alert
# TODO(nsatterl): fix this!!!!
#alert = transform(alert)
if self.db.is_duplicate(alert['environment'], alert['resource'], alert['event'], alert['severity']):
# Duplicate alert .. 1. update existing document with lastReceiveTime, lastReceiveId, text, summary, value, tags and origin
# 2. increment duplicate count
LOG.info('%s : Duplicate alert -> update dup count', alert['id'])
update = {
"lastReceiveTime": alert['receiveTime'],
"expireTime": alert['expireTime'],
"lastReceiveId": alert['id'],
"text": alert['text'],
"summary": alert['summary'],
"value": alert['value'],
"tags": alert['tags'],
"repeat": True,
"origin": alert['origin'],
"trendIndication": 'noChange',
}
self.db.duplicate_alert(alert['environment'], alert['resource'], alert['event'], **update)
if alert['status'] not in [status.OPEN, status.ACK, status.CLOSED]:
if alert['severity'] != 'NORMAL':
current_status = status.OPEN
else:
current_status = status.CLOSED
else:
current_status = status.UNKNOWN
if current_status:
self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status)
self.input_queue.task_done()
elif self.db.is_correlated(alert['environment'], alert['resource'], alert['event']):
# Diff sev alert ... 1. update existing document with severity, createTime, receiveTime, lastReceiveTime, previousSeverity,
# severityCode, lastReceiveId, text, summary, value, tags and origin
# 2. set duplicate count to zero
# 3. push history
previous_severity = self.db.get_severity(alert['environment'], alert['resource'], alert['event'])
LOG.info('%s : Event and/or severity change %s %s -> %s update details', alert['id'], alert['event'],
previous_severity, alert['severity'])
# TODO(nsatterl): determine ti based on current and previous severity
trend_indication = 'moreSevere' or 'lessSevere'
update = {
"event": alert['event'],
"severity": alert['severity'],
"createTime": alert['createTime'],
"receiveTime": alert['receiveTime'],
"lastReceiveTime": alert['receiveTime'],
"expireTime": alert['expireTime'],
"previousSeverity": previous_severity,
"lastReceiveId": alert['id'],
"text": alert['text'],
"summary": alert['summary'],
"value": alert['value'],
"tags": alert['tags'],
"repeat": False,
"origin": alert['origin'],
"thresholdInfo": alert['thresholdInfo'],
"trendIndication": trend_indication,
"duplicateCount": 0
}
enrichedAlert = self.db.modify_alert(alert['environment'], alert['resource'], alert['event'], **update)
current_status = calculate_status(alert['severity'], previous_severity)
if current_status:
self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status)
# Forward alert to notify topic and logger queue
self.mq.send(enrichedAlert, CONF.outbound_queue)
self.mq.send(enrichedAlert, CONF.outbound_topic)
self.input_queue.task_done()
LOG.info('%s : Alert forwarded to %s and %s', alert['id'], CONF.outbound_queue, CONF.outbound_topic)
else:
LOG.info('%s : New alert -> insert', alert['id'])
# New alert so ... 1. insert entire document
# 2. push history
# 3. set duplicate count to zero
trend_indication = 'noChange'
newAlert = Alert(
alertid=alert['id'],
resource=alert['resource'],
event=alert['event'],
correlate=alert['correlatedEvents'],
group=alert['group'],
value=alert['value'],
severity=alert['severity'],
environment=alert['environment'],
service=alert['service'],
text=alert['text'],
event_type=alert['type'],
tags=alert['tags'],
origin=alert['origin'],
threshold_info=alert['thresholdInfo'],
summary=alert['summary'],
timeout=alert['timeout'],
create_time=alert['createTime'],
receive_time=alert['receiveTime'],
last_receive_time=alert['receiveTime'],
duplicate_count=0,
status=status.OPEN,
trend_indication=trend_indication,
last_receive_id=alert['id'],
)
self.db.save_alert(newAlert)
# if alert['severity'] != 'NORMAL':
# status = 'OPEN'
# else:
# status = 'CLOSED'
#
current_status = status.OPEN if alert['severity'] != severity.NORMAL else status.CLOSED
LOG.debug('severity = %s => status = %s', alert['severity'], current_status)
self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status)
# Forward alert to notify topic and logger queue
self.mq.send(newAlert, CONF.outbound_queue)
self.mq.send(newAlert, CONF.outbound_topic)
self.input_queue.task_done()
LOG.info('%s : Alert forwarded to %s and %s', alert['id'], CONF.outbound_queue, CONF.outbound_topic)
self.input_queue.task_done()
