class TestTags(unittest.TestCase): """ Ensures Alert tagging is working as expected. """ def setUp(self): config.parse_args(sys.argv) self.RESOURCE = 'taghost456' self.EVENT = 'TagEvent' self.TAGS = ['location:london', 'vendor:ibm'] self.db = Mongo() def test_save_and_tag_alert(self): """ Save an alert to database and tag it. """ alert = Alert(self.RESOURCE, self.EVENT, receive_time=datetime.datetime.utcnow()) ret = self.db.save_alert(alert) self.assertIsInstance(ret, basestring) self.db.tag_alert(alert.alertid, self.TAGS) self.assertItemsEqual(self.db.get_alert(alert.alertid).tags, [self.TAGS])
def setUp(self): config.parse_args(sys.argv) self.RESOURCE = 'taghost456' self.EVENT = 'TagEvent' self.TAGS = ['location:london', 'vendor:ibm'] self.db = Mongo()
def __init__(self, mq, queue, statsd): threading.Thread.__init__(self) LOG.debug('Initialising %s...', self.getName()) self.queue = queue # internal queue self.mq = mq # message broker self.db = Mongo() # mongo database self.statsd = statsd # graphite metrics
def run(self): self.running = True self.queue = Queue.Queue() # Create internal queue self.db = Mongo() # mongo database self.carbon = Carbon() # carbon metrics self.statsd = StatsD() # graphite metrics # Connect to message queue self.mq = Messaging() self.mq.connect( callback=ServerMessage(self.mq, self.queue, self.statsd)) self.mq.subscribe() # Start worker threads LOG.debug('Starting %s worker threads...', CONF.server_threads) for i in range(CONF.server_threads): w = WorkerThread(self.mq, self.queue, self.statsd) try: w.start() except Exception, e: LOG.error('Worker thread #%s did not start: %s', i, e) continue LOG.info('Started worker thread: %s', w.getName())
def __init__(self, mq, queue): threading.Thread.__init__(self) LOG.debug('Initialising %s...', self.getName()) self.input_queue = queue # internal queue self.mq = mq # message broker self.db = Mongo() # mongo database
import sys from flask import Flask from alerta.common import config from alerta.common import log as logging from alerta.common.mq import Messaging from alerta.server.database import Mongo Version = '2.1.0' LOG = logging.getLogger(__name__) CONF = config.CONF config.parse_args(version=Version) logging.setup('alerta') app = Flask(__name__) app.config.from_object(__name__) db = Mongo() mq = Messaging() mq.connect() import views import management.views
sys.path.insert(0, possible_topdir) from alerta.common import log as logging from alerta.common import config from alerta.server.database import Mongo from alerta.alert import Alert, severity LOG = logging.getLogger('alerta') LOG = logging.getLogger(__name__) CONF = config.CONF print CONF config.parse_args(['--use-stderr', '--debug']) logging.setup('alerta') db = Mongo() #print db.save_alert({''}) alert3 = Alert('router55', 'Node_Down', severity=severity.INDETERMINATE, value='FAILED', timeout=600, environment=['PROD'], receive_time="2013-02-23T09:18:05.303Z", last_receive_time="2013-02-23T09:18:05.303Z", service=['Network', 'Common'], tags=['london', 'location:london', 'dc:location=london'], text="Node is not responding via ping.", origin="test3", correlate=['Node_Up', 'Node_Down'], event_type='myAlert', trend_indication='moreSevere') print alert3 print alert3.get_id() print alert3.get_header() print alert3.get_body() print 'Saving alert...' print db.save_alert(alert3)
class WorkerThread(threading.Thread): def __init__(self, mq, queue, statsd): threading.Thread.__init__(self) LOG.debug('Initialising %s...', self.getName()) self.queue = queue # internal queue self.mq = mq # message broker self.db = Mongo() # mongo database self.statsd = statsd # graphite metrics def run(self): while True: LOG.debug('Waiting on input queue...') try: incomingAlert = self.queue.get(True, CONF.loop_every) except Queue.Empty: LOG.debug('Send heartbeat...') heartbeat = Heartbeat(version=Version, timeout=CONF.loop_every) self.mq.send(heartbeat) continue if not incomingAlert: LOG.info('%s is shutting down.', self.getName()) break if incomingAlert.get_type() == 'Heartbeat': heartbeat = incomingAlert LOG.info('Heartbeat received from %s...', heartbeat.origin) self.db.update_hb(heartbeat) self.queue.task_done() continue else: LOG.info('Alert received from %s...', incomingAlert.origin) try: suppress = incomingAlert.transform_alert() except RuntimeError: self.statsd.metric_send('alerta.alerts.error', 1) self.queue.task_done() continue if suppress: LOG.info('Suppressing alert %s', incomingAlert.get_id()) self.queue.task_done() continue if self.db.is_duplicate(incomingAlert, incomingAlert.severity): # Duplicate alert .. 1. update existing document with lastReceiveTime, lastReceiveId, text, summary, # value, tags and origin # 2. increment duplicate count LOG.info('%s : Duplicate alert -> update dup count', incomingAlert.alertid) duplicateAlert = self.db.duplicate_alert(incomingAlert) if CONF.forward_duplicate: # Forward alert to notify topic and logger queue self.mq.send(duplicateAlert, CONF.outbound_queue) self.mq.send(duplicateAlert, CONF.outbound_topic) LOG.info('%s : Alert forwarded to %s and %s', duplicateAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic) self.queue.task_done() elif self.db.is_correlated(incomingAlert): # Diff sev alert ... 1. update existing document with severity, createTime, receiveTime, # lastReceiveTime, previousSeverity, # severityCode, lastReceiveId, text, summary, value, tags and origin # 2. set duplicate count to zero # 3. push history previous_severity = self.db.get_severity(incomingAlert) LOG.info('%s : Event and/or severity change %s %s -> %s update details', incomingAlert.get_id(), incomingAlert.event, previous_severity, incomingAlert.severity) trend_indication = severity_code.trend(previous_severity, incomingAlert.severity) correlatedAlert = self.db.update_alert(incomingAlert, previous_severity, trend_indication) status = severity_code.status_from_severity(previous_severity, correlatedAlert.severity) if status: self.db.update_status(alert=correlatedAlert, status=status) correlatedAlert.status = status # Forward alert to notify topic and logger queue self.mq.send(correlatedAlert, CONF.outbound_queue) self.mq.send(correlatedAlert, CONF.outbound_topic) LOG.info('%s : Alert forwarded to %s and %s', correlatedAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic) self.queue.task_done() else: LOG.info('%s : New alert -> insert', incomingAlert.get_id()) # New alert so ... 1. insert entire document # 2. push history # 3. set duplicate count to zero trend_indication = severity_code.trend(severity_code.UNKNOWN, incomingAlert.severity) incomingAlert.status = status_code.OPEN incomingAlert.repeat = False incomingAlert.duplicate_count = 0 incomingAlert.last_receive_id = incomingAlert.alertid incomingAlert.last_receive_time = incomingAlert.receive_time incomingAlert.trend_indication = trend_indication if incomingAlert.alertid != self.db.save_alert(incomingAlert): LOG.critical('Alert was not saved with submitted alert id. Race condition?') status = severity_code.status_from_severity(severity_code.UNKNOWN, incomingAlert.severity) if status: self.db.update_status(alert=incomingAlert, status=status) incomingAlert.status = status # Forward alert to notify topic and logger queue self.mq.send(incomingAlert, CONF.outbound_queue) self.mq.send(incomingAlert, CONF.outbound_topic) LOG.info('%s : Alert forwarded to %s and %s', incomingAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic) self.queue.task_done() # update application stats self.statsd.metric_send('alerta.alerts.total', 1) self.statsd.metric_send('alerta.alerts.%s' % incomingAlert.severity, 1) self.db.update_metrics(incomingAlert.create_time, incomingAlert.receive_time) self.queue.task_done()
class WorkerThread(threading.Thread): def __init__(self, mq, queue): threading.Thread.__init__(self) LOG.debug('Initialising %s...', self.getName()) self.input_queue = queue # internal queue self.mq = mq # message broker self.db = Mongo() # mongo database def run(self): while True: LOG.debug('Waiting on input queue...') item = self.input_queue.get() if not item: LOG.info('%s is shutting down.', self.getName()) break # Handle heartbeats if item.get_type() == 'Heartbeat': LOG.debug('update heartbeat') #self.hb.update_hb(alert) # TODO(nsatterl): rename alert to payload or data or something try: self.db.update( {"origin": alert['origin']}, {"origin": alert['origin'], "version": alert['version'], "createTime": createTime, "receiveTime": receiveTime}, True) except Exception, e: LOG.error('Update failed: %s', e) sys.exit(1) LOG.info('%s : heartbeat from %s', alert['id'], alert['origin']) continue alert = item.get_body() print alert # TODO(nsatterl): fix this!!!! #alert = transform(alert) if self.db.is_duplicate(alert['environment'], alert['resource'], alert['event'], alert['severity']): # Duplicate alert .. 1. update existing document with lastReceiveTime, lastReceiveId, text, summary, value, tags and origin # 2. increment duplicate count LOG.info('%s : Duplicate alert -> update dup count', alert['id']) update = { "lastReceiveTime": alert['receiveTime'], "expireTime": alert['expireTime'], "lastReceiveId": alert['id'], "text": alert['text'], "summary": alert['summary'], "value": alert['value'], "tags": alert['tags'], "repeat": True, "origin": alert['origin'], "trendIndication": 'noChange', } self.db.duplicate_alert(alert['environment'], alert['resource'], alert['event'], **update) if alert['status'] not in [status.OPEN, status.ACK, status.CLOSED]: if alert['severity'] != 'NORMAL': current_status = status.OPEN else: current_status = status.CLOSED else: current_status = status.UNKNOWN if current_status: self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status) self.input_queue.task_done() elif self.db.is_correlated(alert['environment'], alert['resource'], alert['event']): # Diff sev alert ... 1. update existing document with severity, createTime, receiveTime, lastReceiveTime, previousSeverity, # severityCode, lastReceiveId, text, summary, value, tags and origin # 2. set duplicate count to zero # 3. push history previous_severity = self.db.get_severity(alert['environment'], alert['resource'], alert['event']) LOG.info('%s : Event and/or severity change %s %s -> %s update details', alert['id'], alert['event'], previous_severity, alert['severity']) # TODO(nsatterl): determine ti based on current and previous severity trend_indication = 'moreSevere' or 'lessSevere' update = { "event": alert['event'], "severity": alert['severity'], "createTime": alert['createTime'], "receiveTime": alert['receiveTime'], "lastReceiveTime": alert['receiveTime'], "expireTime": alert['expireTime'], "previousSeverity": previous_severity, "lastReceiveId": alert['id'], "text": alert['text'], "summary": alert['summary'], "value": alert['value'], "tags": alert['tags'], "repeat": False, "origin": alert['origin'], "thresholdInfo": alert['thresholdInfo'], "trendIndication": trend_indication, "duplicateCount": 0 } enrichedAlert = self.db.modify_alert(alert['environment'], alert['resource'], alert['event'], **update) current_status = calculate_status(alert['severity'], previous_severity) if current_status: self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status) # Forward alert to notify topic and logger queue self.mq.send(enrichedAlert, CONF.outbound_queue) self.mq.send(enrichedAlert, CONF.outbound_topic) self.input_queue.task_done() LOG.info('%s : Alert forwarded to %s and %s', alert['id'], CONF.outbound_queue, CONF.outbound_topic) else: LOG.info('%s : New alert -> insert', alert['id']) # New alert so ... 1. insert entire document # 2. push history # 3. set duplicate count to zero trend_indication = 'noChange' newAlert = Alert( alertid=alert['id'], resource=alert['resource'], event=alert['event'], correlate=alert['correlatedEvents'], group=alert['group'], value=alert['value'], severity=alert['severity'], environment=alert['environment'], service=alert['service'], text=alert['text'], event_type=alert['type'], tags=alert['tags'], origin=alert['origin'], threshold_info=alert['thresholdInfo'], summary=alert['summary'], timeout=alert['timeout'], create_time=alert['createTime'], receive_time=alert['receiveTime'], last_receive_time=alert['receiveTime'], duplicate_count=0, status=status.OPEN, trend_indication=trend_indication, last_receive_id=alert['id'], ) self.db.save_alert(newAlert) # if alert['severity'] != 'NORMAL': # status = 'OPEN' # else: # status = 'CLOSED' # current_status = status.OPEN if alert['severity'] != severity.NORMAL else status.CLOSED LOG.debug('severity = %s => status = %s', alert['severity'], current_status) self.db.update_status(alert['environment'], alert['resource'], alert['event'], current_status) # Forward alert to notify topic and logger queue self.mq.send(newAlert, CONF.outbound_queue) self.mq.send(newAlert, CONF.outbound_topic) self.input_queue.task_done() LOG.info('%s : Alert forwarded to %s and %s', alert['id'], CONF.outbound_queue, CONF.outbound_topic) self.input_queue.task_done()
class WorkerThread(threading.Thread): def __init__(self, mq, queue, statsd): threading.Thread.__init__(self) LOG.debug('Initialising %s...', self.getName()) self.queue = queue # internal queue self.mq = mq # message broker self.db = Mongo() # mongo database self.statsd = statsd # graphite metrics def run(self): while True: LOG.debug('Waiting on input queue...') try: incomingAlert = self.queue.get(True, CONF.loop_every) except Queue.Empty: continue if not incomingAlert: LOG.info('%s is shutting down.', self.getName()) break if incomingAlert.get_type() == 'Heartbeat': heartbeat = incomingAlert LOG.info('Heartbeat received from %s...', heartbeat.origin) self.db.update_hb(heartbeat) self.queue.task_done() continue else: LOG.info('Alert received from %s...', incomingAlert.origin) try: suppress = incomingAlert.transform_alert() except RuntimeError: self.statsd.metric_send('alerta.alerts.error', 1) self.queue.task_done() continue if suppress: LOG.info('Suppressing alert %s', incomingAlert.get_id()) self.queue.task_done() continue if self.db.is_duplicate(incomingAlert, incomingAlert.severity): # Duplicate alert .. 1. update existing document with lastReceiveTime, lastReceiveId, text, summary, # value, status, tags and origin # 2. increment duplicate count # 3. update and push status if changed LOG.info('%s : Duplicate alert -> update dup count', incomingAlert.alertid) duplicateAlert = self.db.duplicate_alert(incomingAlert) if incomingAlert.status != status_code.UNKNOWN and incomingAlert.status != duplicateAlert.status: self.db.update_status(alert=duplicateAlert, status=incomingAlert.status, text='Alerta server') duplicateAlert.status = incomingAlert.status if CONF.forward_duplicate: # Forward alert to notify topic and logger queue self.mq.send(duplicateAlert, CONF.outbound_queue) self.mq.send(duplicateAlert, CONF.outbound_topic) LOG.info('%s : Alert forwarded to %s and %s', duplicateAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic) self.db.update_timer_metric(duplicateAlert.create_time, duplicateAlert.last_receive_time) self.queue.task_done() elif self.db.is_correlated(incomingAlert): # Diff sev alert ... 1. update existing document with severity, createTime, receiveTime, # lastReceiveTime, previousSeverity, # severityCode, lastReceiveId, text, summary, value, tags and origin # 2. set duplicate count to zero # 3. push history and status if changed previous_severity = self.db.get_severity(incomingAlert) LOG.info( '%s : Event and/or severity change %s %s -> %s update details', incomingAlert.get_id(), incomingAlert.event, previous_severity, incomingAlert.severity) trend_indication = severity_code.trend(previous_severity, incomingAlert.severity) correlatedAlert = self.db.correlate_alert( incomingAlert, previous_severity, trend_indication) if incomingAlert.status == status_code.UNKNOWN: incomingAlert.status = severity_code.status_from_severity( previous_severity, incomingAlert.severity, correlatedAlert.status) if incomingAlert.status != correlatedAlert.status: self.db.update_status(alert=correlatedAlert, status=incomingAlert.status, text='Alerta server') correlatedAlert.status = incomingAlert.status # Forward alert to notify topic and logger queue self.mq.send(correlatedAlert, CONF.outbound_queue) self.mq.send(correlatedAlert, CONF.outbound_topic) LOG.info('%s : Alert forwarded to %s and %s', correlatedAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic) self.db.update_timer_metric(correlatedAlert.create_time, correlatedAlert.receive_time) self.queue.task_done() else: # New alert so ... 1. insert entire document # 2. push history and status # 3. set duplicate count to zero LOG.info('%s : New alert -> insert', incomingAlert.get_id()) trend_indication = severity_code.trend(severity_code.UNKNOWN, incomingAlert.severity) incomingAlert.repeat = False incomingAlert.duplicate_count = 0 incomingAlert.last_receive_id = incomingAlert.alertid incomingAlert.last_receive_time = incomingAlert.receive_time incomingAlert.trend_indication = trend_indication if incomingAlert.status == status_code.UNKNOWN: incomingAlert.status = severity_code.status_from_severity( severity_code.UNKNOWN, incomingAlert.severity) if incomingAlert.alertid != self.db.save_alert(incomingAlert): LOG.critical( 'Alert was not saved with submitted alert id. Race condition?' ) self.db.update_status(alert=incomingAlert, status=incomingAlert.status, text='Alerta server') # Forward alert to notify topic and logger queue self.mq.send(incomingAlert, CONF.outbound_queue) self.mq.send(incomingAlert, CONF.outbound_topic) LOG.info('%s : Alert forwarded to %s and %s', incomingAlert.get_id(), CONF.outbound_queue, CONF.outbound_topic) self.db.update_timer_metric(incomingAlert.create_time, incomingAlert.receive_time) self.queue.task_done() # update application stats self.statsd.metric_send('alerta.alerts.total', 1) self.statsd.metric_send( 'alerta.alerts.%s' % incomingAlert.severity, 1) self.queue.task_done()