class WorkerThread(threading.Thread):
def __init__(self, mq, queue):
threading.Thread.__init__(self)
LOG.debug('Initialising %s...', self.getName())
self.input_queue = queue # internal queue
self.mq = mq # message broker
self.db = Mongo() # mongo database
def run(self):
while True:
LOG.debug('Waiting on input queue...')
item = self.input_queue.get()
if not item:
LOG.info('%s is shutting down.', self.getName())
break
# Handle heartbeats
if item.get_type() == 'Heartbeat':
LOG.debug('update heartbeat')
#self.hb.update_hb(alert) # TODO(nsatterl): rename alert to payload or data or something
try:
self.db.update(
{"origin": alert['origin']},
{"origin": alert['origin'], "version": alert['version'], "createTime": createTime,
"receiveTime": receiveTime},
True)
except Exception, e:
LOG.error('Update failed: %s', e)
sys.exit(1)
LOG.info('%s : heartbeat from %s', alert['id'], alert['origin'])
continue
alert = item.get_body()
print alert
# TODO(nsatterl): fix this!!!!
#alert = transform(alert)
if self.db.is_duplicate(alert['environment'], alert['resource'], alert['event'], alert['severity']):
# Duplicate alert .. 1. update existing document with lastReceiveTime, lastReceiveId, text, summary, value, tags and origin
# 2. increment duplicate count
LOG.info('%s : Duplicate alert -> update dup count', alert['id'])
update = {
"lastReceiveTime": alert['receiveTime'],
"expireTime": alert['expireTime'],
"lastReceiveId": alert['id'],
"text": alert['text'],
"summary": alert['summary'],
"value": alert['value'],
"tags": alert['tags'],
"repeat": True,
"origin": alert['origin'],
"trendIndication": 'noChange',
}
self.db.duplicate_alert(alert['environment'], alert['resource'], alert['event'], **update)
if alert['status'] not in [status.OPEN, status.ACK, status.CLOSED]:
if alert['severity'] != 'NORMAL':
current_status = status.OPEN
else:
current_status = status.CLOSED
else:
current_status = status.UNKNOWN
if current_status:
self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status)
self.input_queue.task_done()
elif self.db.is_correlated(alert['environment'], alert['resource'], alert['event']):
# Diff sev alert ... 1. update existing document with severity, createTime, receiveTime, lastReceiveTime, previousSeverity,
# severityCode, lastReceiveId, text, summary, value, tags and origin
# 2. set duplicate count to zero
# 3. push history
previous_severity = self.db.get_severity(alert['environment'], alert['resource'], alert['event'])
LOG.info('%s : Event and/or severity change %s %s -> %s update details', alert['id'], alert['event'],
previous_severity, alert['severity'])
# TODO(nsatterl): determine ti based on current and previous severity
trend_indication = 'moreSevere' or 'lessSevere'
update = {
"event": alert['event'],
"severity": alert['severity'],
"createTime": alert['createTime'],
"receiveTime": alert['receiveTime'],
"lastReceiveTime": alert['receiveTime'],
"expireTime": alert['expireTime'],
"previousSeverity": previous_severity,
"lastReceiveId": alert['id'],
"text": alert['text'],
"summary": alert['summary'],
"value": alert['value'],
"tags": alert['tags'],
"repeat": False,
"origin": alert['origin'],
"thresholdInfo": alert['thresholdInfo'],
"trendIndication": trend_indication,
"duplicateCount": 0
}
enrichedAlert = self.db.modify_alert(alert['environment'], alert['resource'], alert['event'], **update)
current_status = calculate_status(alert['severity'], previous_severity)
if current_status:
self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status)
# Forward alert to notify topic and logger queue
self.mq.send(enrichedAlert, CONF.outbound_queue)
self.mq.send(enrichedAlert, CONF.outbound_topic)
self.input_queue.task_done()
LOG.info('%s : Alert forwarded to %s and %s', alert['id'], CONF.outbound_queue, CONF.outbound_topic)
else:
LOG.info('%s : New alert -> insert', alert['id'])
# New alert so ... 1. insert entire document
# 2. push history
# 3. set duplicate count to zero
trend_indication = 'noChange'
newAlert = Alert(
alertid=alert['id'],
resource=alert['resource'],
event=alert['event'],
correlate=alert['correlatedEvents'],
group=alert['group'],
value=alert['value'],
severity=alert['severity'],
environment=alert['environment'],
service=alert['service'],
text=alert['text'],
event_type=alert['type'],
tags=alert['tags'],
origin=alert['origin'],
threshold_info=alert['thresholdInfo'],
summary=alert['summary'],
timeout=alert['timeout'],
create_time=alert['createTime'],
receive_time=alert['receiveTime'],
last_receive_time=alert['receiveTime'],
duplicate_count=0,
status=status.OPEN,
trend_indication=trend_indication,
last_receive_id=alert['id'],
)
self.db.save_alert(newAlert)
# if alert['severity'] != 'NORMAL':
# status = 'OPEN'
# else:
# status = 'CLOSED'
#
current_status = status.OPEN if alert['severity'] != severity.NORMAL else status.CLOSED
LOG.debug('severity = %s => status = %s', alert['severity'], current_status)
self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status)
# Forward alert to notify topic and logger queue
self.mq.send(newAlert, CONF.outbound_queue)
self.mq.send(newAlert, CONF.outbound_topic)
self.input_queue.task_done()
LOG.info('%s : Alert forwarded to %s and %s', alert['id'], CONF.outbound_queue, CONF.outbound_topic)
self.input_queue.task_done()
class WorkerThread(threading.Thread):
def __init__(self, mq, queue, statsd):
threading.Thread.__init__(self)
LOG.debug('Initialising %s...', self.getName())
self.queue = queue # internal queue
self.mq = mq # message broker
self.db = Mongo() # mongo database
self.statsd = statsd # graphite metrics
def run(self):
while True:
LOG.debug('Waiting on input queue...')
try:
incomingAlert = self.queue.get(True, CONF.loop_every)
except Queue.Empty:
LOG.debug('Send heartbeat...')
heartbeat = Heartbeat(version=Version, timeout=CONF.loop_every)
self.mq.send(heartbeat)
continue
if not incomingAlert:
LOG.info('%s is shutting down.', self.getName())
break
if incomingAlert.get_type() == 'Heartbeat':
heartbeat = incomingAlert
LOG.info('Heartbeat received from %s...', heartbeat.origin)
self.db.update_hb(heartbeat)
self.queue.task_done()
continue
else:
LOG.info('Alert received from %s...', incomingAlert.origin)
try:
suppress = incomingAlert.transform_alert()
except RuntimeError:
self.statsd.metric_send('alerta.alerts.error', 1)
self.queue.task_done()
continue
if suppress:
LOG.info('Suppressing alert %s', incomingAlert.get_id())
self.queue.task_done()
continue
if self.db.is_duplicate(incomingAlert, incomingAlert.severity):
# Duplicate alert .. 1. update existing document with lastReceiveTime, lastReceiveId, text, summary,
# value, tags and origin
# 2. increment duplicate count
LOG.info('%s : Duplicate alert -> update dup count', incomingAlert.alertid)
duplicateAlert = self.db.duplicate_alert(incomingAlert)
if CONF.forward_duplicate:
# Forward alert to notify topic and logger queue
self.mq.send(duplicateAlert, CONF.outbound_queue)
self.mq.send(duplicateAlert, CONF.outbound_topic)
LOG.info('%s : Alert forwarded to %s and %s', duplicateAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic)
self.queue.task_done()
elif self.db.is_correlated(incomingAlert):
# Diff sev alert ... 1. update existing document with severity, createTime, receiveTime,
# lastReceiveTime, previousSeverity,
# severityCode, lastReceiveId, text, summary, value, tags and origin
# 2. set duplicate count to zero
# 3. push history
previous_severity = self.db.get_severity(incomingAlert)
LOG.info('%s : Event and/or severity change %s %s -> %s update details', incomingAlert.get_id(),
incomingAlert.event, previous_severity, incomingAlert.severity)
trend_indication = severity_code.trend(previous_severity, incomingAlert.severity)
correlatedAlert = self.db.update_alert(incomingAlert, previous_severity, trend_indication)
status = severity_code.status_from_severity(previous_severity, correlatedAlert.severity)
if status:
self.db.update_status(alert=correlatedAlert, status=status)
correlatedAlert.status = status
# Forward alert to notify topic and logger queue
self.mq.send(correlatedAlert, CONF.outbound_queue)
self.mq.send(correlatedAlert, CONF.outbound_topic)
LOG.info('%s : Alert forwarded to %s and %s', correlatedAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic)
self.queue.task_done()
else:
LOG.info('%s : New alert -> insert', incomingAlert.get_id())
# New alert so ... 1. insert entire document
# 2. push history
# 3. set duplicate count to zero
trend_indication = severity_code.trend(severity_code.UNKNOWN, incomingAlert.severity)
incomingAlert.status = status_code.OPEN
incomingAlert.repeat = False
incomingAlert.duplicate_count = 0
incomingAlert.last_receive_id = incomingAlert.alertid
incomingAlert.last_receive_time = incomingAlert.receive_time
incomingAlert.trend_indication = trend_indication
if incomingAlert.alertid != self.db.save_alert(incomingAlert):
LOG.critical('Alert was not saved with submitted alert id. Race condition?')
status = severity_code.status_from_severity(severity_code.UNKNOWN, incomingAlert.severity)
if status:
self.db.update_status(alert=incomingAlert, status=status)
incomingAlert.status = status
# Forward alert to notify topic and logger queue
self.mq.send(incomingAlert, CONF.outbound_queue)
self.mq.send(incomingAlert, CONF.outbound_topic)
LOG.info('%s : Alert forwarded to %s and %s', incomingAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic)
self.queue.task_done()
# update application stats
self.statsd.metric_send('alerta.alerts.total', 1)
self.statsd.metric_send('alerta.alerts.%s' % incomingAlert.severity, 1)
self.db.update_metrics(incomingAlert.create_time, incomingAlert.receive_time)
self.queue.task_done()
class WorkerThread(threading.Thread):
def __init__(self, mq, queue, statsd):
threading.Thread.__init__(self)
LOG.debug('Initialising %s...', self.getName())
self.queue = queue # internal queue
self.mq = mq # message broker
self.db = Mongo() # mongo database
self.statsd = statsd # graphite metrics
def run(self):
while True:
LOG.debug('Waiting on input queue...')
try:
incomingAlert = self.queue.get(True, CONF.loop_every)
except Queue.Empty:
continue
if not incomingAlert:
LOG.info('%s is shutting down.', self.getName())
break
if incomingAlert.get_type() == 'Heartbeat':
heartbeat = incomingAlert
LOG.info('Heartbeat received from %s...', heartbeat.origin)
self.db.update_hb(heartbeat)
self.queue.task_done()
continue
else:
LOG.info('Alert received from %s...', incomingAlert.origin)
try:
suppress = incomingAlert.transform_alert()
except RuntimeError:
self.statsd.metric_send('alerta.alerts.error', 1)
self.queue.task_done()
continue
if suppress:
LOG.info('Suppressing alert %s', incomingAlert.get_id())
self.queue.task_done()
continue
if self.db.is_duplicate(incomingAlert, incomingAlert.severity):
# Duplicate alert .. 1. update existing document with lastReceiveTime, lastReceiveId, text, summary,
# value, status, tags and origin
# 2. increment duplicate count
# 3. update and push status if changed
LOG.info('%s : Duplicate alert -> update dup count',
incomingAlert.alertid)
duplicateAlert = self.db.duplicate_alert(incomingAlert)
if incomingAlert.status != status_code.UNKNOWN and incomingAlert.status != duplicateAlert.status:
self.db.update_status(alert=duplicateAlert,
status=incomingAlert.status,
text='Alerta server')
duplicateAlert.status = incomingAlert.status
if CONF.forward_duplicate:
# Forward alert to notify topic and logger queue
self.mq.send(duplicateAlert, CONF.outbound_queue)
self.mq.send(duplicateAlert, CONF.outbound_topic)
LOG.info('%s : Alert forwarded to %s and %s',
duplicateAlert.get_id(), CONF.outbound_queue,
CONF.outbound_topic)
self.db.update_timer_metric(duplicateAlert.create_time,
duplicateAlert.last_receive_time)
self.queue.task_done()
elif self.db.is_correlated(incomingAlert):
# Diff sev alert ... 1. update existing document with severity, createTime, receiveTime,
# lastReceiveTime, previousSeverity,
# severityCode, lastReceiveId, text, summary, value, tags and origin
# 2. set duplicate count to zero
# 3. push history and status if changed
previous_severity = self.db.get_severity(incomingAlert)
LOG.info(
'%s : Event and/or severity change %s %s -> %s update details',
incomingAlert.get_id(), incomingAlert.event,
previous_severity, incomingAlert.severity)
trend_indication = severity_code.trend(previous_severity,
incomingAlert.severity)
correlatedAlert = self.db.correlate_alert(
incomingAlert, previous_severity, trend_indication)
if incomingAlert.status == status_code.UNKNOWN:
incomingAlert.status = severity_code.status_from_severity(
previous_severity, incomingAlert.severity,
correlatedAlert.status)
if incomingAlert.status != correlatedAlert.status:
self.db.update_status(alert=correlatedAlert,
status=incomingAlert.status,
text='Alerta server')
correlatedAlert.status = incomingAlert.status
# Forward alert to notify topic and logger queue
self.mq.send(correlatedAlert, CONF.outbound_queue)
self.mq.send(correlatedAlert, CONF.outbound_topic)
LOG.info('%s : Alert forwarded to %s and %s',
correlatedAlert.get_id(), CONF.outbound_queue,
CONF.outbound_topic)
self.db.update_timer_metric(correlatedAlert.create_time,
correlatedAlert.receive_time)
self.queue.task_done()
else:
# New alert so ... 1. insert entire document
# 2. push history and status
# 3. set duplicate count to zero
LOG.info('%s : New alert -> insert', incomingAlert.get_id())
trend_indication = severity_code.trend(severity_code.UNKNOWN,
incomingAlert.severity)
incomingAlert.repeat = False
incomingAlert.duplicate_count = 0
incomingAlert.last_receive_id = incomingAlert.alertid
incomingAlert.last_receive_time = incomingAlert.receive_time
incomingAlert.trend_indication = trend_indication
if incomingAlert.status == status_code.UNKNOWN:
incomingAlert.status = severity_code.status_from_severity(
severity_code.UNKNOWN, incomingAlert.severity)
if incomingAlert.alertid != self.db.save_alert(incomingAlert):
LOG.critical(
'Alert was not saved with submitted alert id. Race condition?'
)
self.db.update_status(alert=incomingAlert,
status=incomingAlert.status,
text='Alerta server')
# Forward alert to notify topic and logger queue
self.mq.send(incomingAlert, CONF.outbound_queue)
self.mq.send(incomingAlert, CONF.outbound_topic)
LOG.info('%s : Alert forwarded to %s and %s',
incomingAlert.get_id(), CONF.outbound_queue,
CONF.outbound_topic)
self.db.update_timer_metric(incomingAlert.create_time,
incomingAlert.receive_time)
self.queue.task_done()
# update application stats
self.statsd.metric_send('alerta.alerts.total', 1)
self.statsd.metric_send(
'alerta.alerts.%s' % incomingAlert.severity, 1)
self.queue.task_done()
