Example #1
0
    def on_message(self, body, message):

        try:
            alert = AlertDocument.parse_alert(body)
            alertid = alert.get_id()
        except Exception as e:
            LOG.warn(e)
            return

        if alert.repeat:
            message.ack()
            return

        if alert.status not in ["open", "closed"]:
            message.ack()
            return

        if alert.severity not in ["critical", "major"] and alert.previous_severity not in ["critical", "major"]:
            message.ack()
            return

        if alertid in on_hold:
            if alert.severity in ["normal", "ok", "cleared"]:
                try:
                    del on_hold[alertid]
                except KeyError:
                    pass
                message.ack()
            else:
                on_hold[alertid] = (alert, time.time() + HOLD_TIME)
                message.ack()
        else:
            on_hold[alertid] = (alert, time.time() + HOLD_TIME)
            message.ack()
Example #2
0
    def test_alertdoc(self):
        """
        Ensure a valid alert document is created with all assigned values
        """

        alert = AlertDocument(id=self.ALERTID,
                              resource=self.RESOURCE,
                              event=self.EVENT,
                              environment=self.ENVIRONMENT,
                              severity=self.SEVERITY,
                              correlate=self.CORRELATE,
                              status=self.STATUS,
                              service=self.SERVICE,
                              group=self.GROUP,
                              value=self.VALUE,
                              text=self.TEXT,
                              tags=self.TAGS,
                              attributes={
                                  'thresholdInfo': self.THRESHOLD_INFO,
                                  'moreInfo': self.MORE_INFO,
                                  'graphUrls': self.GRAPH_URLS
                              },
                              origin=self.ORIGIN,
                              event_type=self.EVENT_TYPE,
                              create_time=self.CREATE_TIME,
                              timeout=self.TIMEOUT,
                              raw_data=self.RAW_DATA,
                              duplicate_count=self.DUPLICATE_COUNT,
                              repeat=self.REPEAT,
                              previous_severity=self.PREVIOUS_SEVERITY,
                              trend_indication=self.TREND_INDICATION,
                              receive_time=self.RECEIVE_TIME,
                              last_receive_id=self.ALERTID,
                              last_receive_time=self.RECEIVE_TIME,
                              history=self.HISTORY)

        self.assertEquals(alert.resource, self.RESOURCE)
        self.assertEquals(alert.event, self.EVENT)
        self.assertEquals(alert.correlate, self.CORRELATE)
        self.assertEquals(alert.group, self.GROUP)
        self.assertEquals(alert.value, self.VALUE)
        self.assertEquals(alert.status, self.STATUS)
        self.assertEquals(alert.severity, self.SEVERITY)
        self.assertEquals(alert.previous_severity, self.PREVIOUS_SEVERITY)
        self.assertEquals(alert.environment, self.ENVIRONMENT)
        self.assertEquals(alert.service, self.SERVICE)
        self.assertEquals(alert.text, self.TEXT)
        self.assertEquals(alert.event_type, self.EVENT_TYPE)
        self.assertEquals(alert.tags, self.TAGS)
        self.assertEquals(alert.origin, self.ORIGIN)
        self.assertEquals(alert.repeat, self.REPEAT)
        self.assertEquals(alert.duplicate_count, self.DUPLICATE_COUNT)
        self.assertEquals(alert.timeout, self.TIMEOUT)
        self.assertEquals(alert.id, self.ALERTID)
        self.assertEquals(alert.last_receive_id, self.ALERTID)
        self.assertEquals(alert.create_time, self.CREATE_TIME)
        self.assertEquals(alert.receive_time, self.RECEIVE_TIME)
        self.assertEquals(alert.trend_indication, self.TREND_INDICATION)
        self.assertEquals(alert.raw_data, self.RAW_DATA)
        self.assertEquals(alert.history, self.HISTORY)
Example #3
0
def test_rules_evaluation(alert_spec, input_rules, expected_contacts):
    '''
    Test that rules are properly evaluated
    '''
    with patch.dict(mailer.OPTIONS, mailer.DEFAULT_OPTIONS):
        mailer.OPTIONS['mail_to'] = []
        mailer.OPTIONS['group_rules'] = input_rules
        mail_sender = mailer.MailSender()
        with patch.object(mail_sender, '_send_email_message') as _sem:
            alert = AlertDocument.parse_alert(alert_spec)
            _, emailed_contacts = mail_sender.send_email(alert)
            assert _sem.call_count == 1
            assert emailed_contacts == expected_contacts
Example #4
0
    def get_alert(self, id):

        if len(id) == 8:
            query = {
                '$or': [{
                    '_id': {
                        '$regex': '^' + id
                    }
                }, {
                    'lastReceiveId': {
                        '$regex': '^' + id
                    }
                }]
            }
        else:
            query = {'$or': [{'_id': id}, {'lastReceiveId': id}]}

        response = self._db.alerts.find_one(query)
        if not response:
            return

        return AlertDocument(id=response['_id'],
                             resource=response['resource'],
                             event=response['event'],
                             environment=response['environment'],
                             severity=response['severity'],
                             correlate=response['correlate'],
                             status=response['status'],
                             service=response['service'],
                             group=response['group'],
                             value=response['value'],
                             text=response['text'],
                             tags=response['tags'],
                             attributes=response['attributes'],
                             origin=response['origin'],
                             event_type=response['type'],
                             create_time=response['createTime'],
                             timeout=response['timeout'],
                             raw_data=response['rawData'],
                             customer=response.get('customer', None),
                             duplicate_count=response['duplicateCount'],
                             repeat=response['repeat'],
                             previous_severity=response['previousSeverity'],
                             trend_indication=response['trendIndication'],
                             receive_time=response['receiveTime'],
                             last_receive_id=response['lastReceiveId'],
                             last_receive_time=response['lastReceiveTime'],
                             history=response['history'])
    def test_date_formats(self):

        self.CREATE_TIME = datetime.datetime(year=2012, month=11, day=10, hour=9, minute=8, second=7, microsecond=543210)

        alert = AlertDocument(
            id=self.ALERTID, resource=self.RESOURCE, event=self.EVENT, environment=self.ENVIRONMENT,
            severity=self.SEVERITY, correlate=self.CORRELATE, status=self.STATUS, service=self.SERVICE, group=self.GROUP,
            value=self.VALUE, text=self.TEXT, tags=self.TAGS,
            attributes={'thresholdInfo': self.THRESHOLD_INFO, 'moreInfo': self.MORE_INFO, 'graphUrls': self.GRAPH_URLS},
            origin=self.ORIGIN, event_type=self.EVENT_TYPE, create_time=self.CREATE_TIME, timeout=self.TIMEOUT,
            raw_data=self.RAW_DATA, duplicate_count=self.DUPLICATE_COUNT, repeat=self.REPEAT,
            previous_severity=self.PREVIOUS_SEVERITY, trend_indication=self.TREND_INDICATION, receive_time=self.RECEIVE_TIME,
            last_receive_id=self.ALERTID, last_receive_time=self.RECEIVE_TIME, history=self.HISTORY, customer=self.CUSTOMER)

        self.assertEqual(alert.get_date('create_time', 'local'), '2012/11/10 09:08:07')
        self.assertEqual(alert.get_date('create_time', 'iso'), '2012-11-10T09:08:07.543Z')
        self.assertEqual(alert.get_date('create_time', 'iso8601'), '2012-11-10T09:08:07.543Z')
        self.assertEqual(alert.get_date('create_time', 'rfc'), 'Sat, 10 Nov 2012 09:08:07 +0000')
        self.assertEqual(alert.get_date('create_time', 'rfc2822'), 'Sat, 10 Nov 2012 09:08:07 +0000')
        self.assertEqual(alert.get_date('create_time', 'short'), 'Sat 10 09:08:07')
        self.assertEqual(alert.get_date('create_time', 'epoch'), 1352538487.0)
        self.assertEqual(alert.get_date('create_time', 'raw'), self.CREATE_TIME)
        self.assertEqual(alert.get_date('create_time'), '2012-11-10T09:08:07.543Z')
        self.assertRaises(ValueError, alert.get_date, 'create_time', 'invalid')
Example #6
0
    def get_alerts(self, tenant, query=None, fields=None, sort=None, page=1, limit=0):

        dBase = self._client[tenant]

        if 'status' not in query:
            query['status'] = {'$ne': "expired"}

        responses = dBase.alerts.find(query, projection=fields, sort=sort).skip((page-1)*limit).limit(limit)

        alerts = list()
        for response in responses:
            alerts.append(
                AlertDocument(
                    id=response['_id'],
                    resource=response['resource'],
                    event=response['event'],
                    environment=response['environment'],
                    severity=response['severity'],
                    correlate=response['correlate'],
                    status=response['status'],
                    service=response['service'],
                    group=response['group'],
                    value=response['value'],
                    text=response['text'],
                    tags=response['tags'],
                    attributes=response['attributes'],
                    origin=response['origin'],
                    event_type=response['type'],
                    create_time=response['createTime'],
                    timeout=response['timeout'],
                    raw_data=response['rawData'],
                    customer=response.get('customer', None),
                    duplicate_count=response['duplicateCount'],
                    repeat=response['repeat'],
                    previous_severity=response['previousSeverity'],
                    trend_indication=response['trendIndication'],
                    receive_time=response['receiveTime'],
                    last_receive_id=response['lastReceiveId'],
                    last_receive_time=response['lastReceiveTime'],
                    history=response['history']
                )
            )
        return alerts
Example #7
0
    def set_status(self, id, status, text=None):
        """
        Set status and update history.
        """
        query = {'_id': {'$regex': '^' + id}}

        event = self.db.alerts.find_one(query, fields={
            "event": 1,
            "_id": 0
        })['event']
        if not event:
            return False

        now = datetime.datetime.utcnow()
        update = {
            '$set': {
                "status": status
            },
            '$push': {
                "history": {
                    "event": event,
                    "status": status,
                    "text": text,
                    "id": id,
                    "updateTime": now
                }
            }
        }

        no_obj_error = "No matching object found"
        response = self.db.command("findAndModify",
                                   'alerts',
                                   allowable_errors=[no_obj_error],
                                   query=query,
                                   update=update,
                                   new=True,
                                   fields={"history": 0})["value"]

        return AlertDocument(id=response['_id'],
                             resource=response['resource'],
                             event=response['event'],
                             environment=response['environment'],
                             severity=response['severity'],
                             correlate=response['correlate'],
                             status=response['status'],
                             service=response['service'],
                             group=response['group'],
                             value=response['value'],
                             text=response['text'],
                             tags=response['tags'],
                             attributes=response['attributes'],
                             origin=response['origin'],
                             event_type=response['type'],
                             create_time=response['createTime'],
                             timeout=response['timeout'],
                             raw_data=response['rawData'],
                             duplicate_count=response['duplicateCount'],
                             repeat=response['repeat'],
                             previous_severity=response['previousSeverity'],
                             trend_indication=response['trendIndication'],
                             receive_time=response['receiveTime'],
                             last_receive_id=response['lastReceiveId'],
                             last_receive_time=response['lastReceiveTime'],
                             history=list())
Example #8
0
    def create_alert(self, alert):

        trend_indication = severity_code.trend(severity_code.UNKNOWN,
                                               alert.severity)
        if alert.status == status_code.UNKNOWN:
            status = severity_code.status_from_severity(
                severity_code.UNKNOWN, alert.severity)
        else:
            status = alert.status

        now = datetime.datetime.utcnow()
        history = [{
            "id": alert.id,
            "event": alert.event,
            "severity": alert.severity,
            "value": alert.value,
            "text": alert.text,
            "updateTime": alert.create_time
        }]
        if status != alert.status:
            history.append({
                "event": alert.event,
                "status": status,
                "text": "new alert status change",
                "id": alert.id,
                "updateTime": now
            })

        alert = {
            "_id": alert.id,
            "resource": alert.resource,
            "event": alert.event,
            "environment": alert.environment,
            "severity": alert.severity,
            "correlate": alert.correlate,
            "status": status,
            "service": alert.service,
            "group": alert.group,
            "value": alert.value,
            "text": alert.text,
            "tags": alert.tags,
            "attributes": alert.attributes,
            "origin": alert.origin,
            "type": alert.event_type,
            "createTime": alert.create_time,
            "timeout": alert.timeout,
            "rawData": alert.raw_data,
            "duplicateCount": 0,
            "repeat": False,
            "previousSeverity": severity_code.UNKNOWN,
            "trendIndication": trend_indication,
            "receiveTime": now,
            "lastReceiveId": alert.id,
            "lastReceiveTime": now,
            "history": history
        }

        LOG.debug('Insert new alert in database: %s', alert)

        response = self.db.alerts.insert(alert)

        if not response:
            return

        return AlertDocument(id=alert['_id'],
                             resource=alert['resource'],
                             event=alert['event'],
                             environment=alert['environment'],
                             severity=alert['severity'],
                             correlate=alert['correlate'],
                             status=alert['status'],
                             service=alert['service'],
                             group=alert['group'],
                             value=alert['value'],
                             text=alert['text'],
                             tags=alert['tags'],
                             attributes=alert['attributes'],
                             origin=alert['origin'],
                             event_type=alert['type'],
                             create_time=alert['createTime'],
                             timeout=alert['timeout'],
                             raw_data=alert['rawData'],
                             duplicate_count=alert['duplicateCount'],
                             repeat=alert['repeat'],
                             previous_severity=alert['previousSeverity'],
                             trend_indication=alert['trendIndication'],
                             receive_time=alert['receiveTime'],
                             last_receive_id=alert['lastReceiveId'],
                             last_receive_time=alert['lastReceiveTime'],
                             history=list())
Example #9
0
    def save_correlated(self, alert):
        """
        Update alert key attributes, reset duplicate count and set repeat=False, keep track of last
        receive id and time, appending all to history. Append to history again if status changes.
        """

        previous_severity = self.get_severity(alert)
        previous_status = self.get_status(alert)
        trend_indication = severity_code.trend(previous_severity,
                                               alert.severity)
        if alert.status == status_code.UNKNOWN:
            status = severity_code.status_from_severity(
                previous_severity, alert.severity, previous_status)
        else:
            status = alert.status

        query = {
            "environment":
            alert.environment,
            "resource":
            alert.resource,
            '$or': [{
                "event": alert.event,
                "severity": {
                    '$ne': alert.severity
                }
            }, {
                "event": {
                    '$ne': alert.event
                },
                "correlate": alert.event,
                "severity": alert.severity
            }, {
                "event": {
                    '$ne': alert.event
                },
                "correlate": alert.event,
                "severity": {
                    '$ne': alert.severity
                }
            }]
        }

        now = datetime.datetime.utcnow()
        update = {
            '$set': {
                "event": alert.event,
                "severity": alert.severity,
                "status": status,
                "value": alert.value,
                "text": alert.text,
                "tags": alert.tags,
                "attributes": alert.attributes,
                "createTime": alert.create_time,
                "rawData": alert.raw_data,
                "duplicateCount": 0,
                "repeat": False,
                "previousSeverity": previous_severity,
                "trendIndication": trend_indication,
                "receiveTime": now,
                "lastReceiveId": alert.id,
                "lastReceiveTime": now
            },
            '$pushAll': {
                "history": [{
                    "event": alert.event,
                    "severity": alert.severity,
                    "value": alert.value,
                    "text": alert.text,
                    "id": alert.id,
                    "updateTime": alert.create_time
                }]
            }
        }

        if status != previous_status:
            update['$pushAll']['history'].append({
                "event": alert.event,
                "status": status,
                "text": "correlated alert status change",
                "id": alert.id,
                "updateTime": now
            })

        LOG.debug('Update correlated alert in database: %s', update)

        no_obj_error = "No matching object found"
        response = self.db.command("findAndModify",
                                   'alerts',
                                   allowable_errors=[no_obj_error],
                                   query=query,
                                   update=update,
                                   new=True,
                                   fields={"history": 0})["value"]

        return AlertDocument(id=response['_id'],
                             resource=response['resource'],
                             event=response['event'],
                             environment=response['environment'],
                             severity=response['severity'],
                             correlate=response['correlate'],
                             status=response['status'],
                             service=response['service'],
                             group=response['group'],
                             value=response['value'],
                             text=response['text'],
                             tags=response['tags'],
                             attributes=response['attributes'],
                             origin=response['origin'],
                             event_type=response['type'],
                             create_time=response['createTime'],
                             timeout=response['timeout'],
                             raw_data=response['rawData'],
                             duplicate_count=response['duplicateCount'],
                             repeat=response['repeat'],
                             previous_severity=response['previousSeverity'],
                             trend_indication=response['trendIndication'],
                             receive_time=response['receiveTime'],
                             last_receive_id=response['lastReceiveId'],
                             last_receive_time=response['lastReceiveTime'],
                             history=list())
Example #10
0
    def save_duplicate(self, alert):
        """
        Update alert value, text and rawData, increment duplicate count and set repeat=True, and
        keep track of last receive id and time but don't append to history unless status changes.
        """

        previous_status = self.get_status(alert)
        if alert.status != status_code.UNKNOWN and alert.status != previous_status:
            status = alert.status
        else:
            status = previous_status

        query = {
            "environment": alert.environment,
            "resource": alert.resource,
            "event": alert.event,
            "severity": alert.severity
        }

        now = datetime.datetime.utcnow()
        update = {
            '$set': {
                "status": status,
                "value": alert.value,
                "text": alert.text,
                "rawData": alert.raw_data,
                "repeat": True,
                "lastReceiveId": alert.id,
                "lastReceiveTime": now
            },
            '$inc': {
                "duplicateCount": 1
            }
        }
        if status != previous_status:
            update['$push'] = {
                "history": {
                    "event": alert.event,
                    "status": status,
                    "text": "duplicate alert status change",
                    "id": alert.id,
                    "updateTime": now
                }
            }

        LOG.debug('Update duplicate alert in database: %s', update)

        no_obj_error = "No matching object found"
        response = self.db.command("findAndModify",
                                   'alerts',
                                   allowable_errors=[no_obj_error],
                                   query=query,
                                   update=update,
                                   new=True,
                                   fields={"history": 0})["value"]

        return AlertDocument(id=response['_id'],
                             resource=response['resource'],
                             event=response['event'],
                             environment=response['environment'],
                             severity=response['severity'],
                             correlate=response['correlate'],
                             status=response['status'],
                             service=response['service'],
                             group=response['group'],
                             value=response['value'],
                             text=response['text'],
                             tags=response['tags'],
                             attributes=response['attributes'],
                             origin=response['origin'],
                             event_type=response['type'],
                             create_time=response['createTime'],
                             timeout=response['timeout'],
                             raw_data=response['rawData'],
                             duplicate_count=response['duplicateCount'],
                             repeat=response['repeat'],
                             previous_severity=response['previousSeverity'],
                             trend_indication=response['trendIndication'],
                             receive_time=response['receiveTime'],
                             last_receive_id=response['lastReceiveId'],
                             last_receive_time=response['lastReceiveTime'],
                             history=list())
Example #11
0
    def query(self, args, from_date=None):

        response = self._alerts(args.filters, from_date)
        alerts = response['alerts']

        if args.output == "json":
            print(json.dumps(alerts, indent=4))
            sys.exit(0)

        for alert in reversed(alerts):

            a = AlertDocument.parse_alert(alert)

            line_color = ''
            end_color = _ENDC

            if args.color:
                line_color = _COLOR_MAP.get(a.severity, _COLOR_MAP['unknown'])

            print(
                line_color +
                '{0}|{1}|{2}|{3:5d}|{4}|{5:<5s}|{6:<10s}|{7:<18s}|{8:12s}|{9:16s}|{10:12s}'
                .format(
                    a.id[0:8],
                    a.get_date('last_receive_time', 'local',
                               args.timezone), a.severity, a.duplicate_count,
                    a.customer or "-", a.environment, ','.join(a.service),
                    a.resource, a.group, a.event, a.value) + end_color)
            print(line_color + '   |{}'.format(a.text) + end_color)

            if args.details:
                print(line_color + '    severity   | {} -> {}'.format(
                    a.previous_severity, a.severity) + end_color)
                print(line_color +
                      '    trend      | {}'.format(a.trend_indication) +
                      end_color)
                print(line_color + '    status     | {}'.format(a.status) +
                      end_color)
                print(line_color + '    resource   | {}'.format(a.resource) +
                      end_color)
                print(line_color + '    group      | {}'.format(a.group) +
                      end_color)
                print(line_color + '    event      | {}'.format(a.event) +
                      end_color)
                print(line_color + '    value      | {}'.format(a.value) +
                      end_color)
                print(line_color +
                      '    tags       | {}'.format(' '.join(a.tags)) +
                      end_color)

                for key, value in a.attributes.items():
                    print(line_color +
                          '    {} | {}'.format(key.ljust(10), value) +
                          end_color)

                latency = a.receive_time - a.create_time

                print(line_color + '        time created  | {}'.format(
                    a.get_date('create_time', 'iso', args.timezone)) +
                      end_color)
                print(line_color + '        time received | {}'.format(
                    a.get_date('receive_time', 'iso', args.timezone)) +
                      end_color)
                print(line_color + '        last received | {}'.format(
                    a.get_date('last_receive_time', 'iso', args.timezone)) +
                      end_color)
                print(line_color + '        latency       | {}ms'.format(
                    (latency.microseconds / 1000)) + end_color)
                print(line_color +
                      '        timeout       | {}s'.format(a.timeout) +
                      end_color)

                print(line_color +
                      '            alert id     | {}'.format(a.id) + end_color)
                print(
                    line_color +
                    '            last recv id | {}'.format(a.last_receive_id) +
                    end_color)
                print(line_color +
                      '            customer     | {}'.format(a.customer) +
                      end_color)
                print(line_color +
                      '            environment  | {}'.format(a.environment) +
                      end_color)
                print(line_color + '            service      | {}'.format(
                    ','.join(a.service)) + end_color)
                print(line_color +
                      '            resource     | {}'.format(a.resource) +
                      end_color)
                print(line_color +
                      '            type         | {}'.format(a.event_type) +
                      end_color)
                print(line_color +
                      '            repeat       | {}'.format(a.repeat) +
                      end_color)
                print(line_color +
                      '            origin       | {}'.format(a.origin) +
                      end_color)
                print(line_color + '            correlate    | {}'.format(
                    ','.join(a.correlate)) + end_color)

        return response.get('lastTime', '')
Example #12
0
    def set_status(self, tenant, id, status, text=None):
        """
        Set status and update history.
        """

        dBase = self._client[tenant]

        query = {'_id': {'$regex': '^' + id}}


        event = dBase.alerts.find_one(query, projection={"event": 1, "_id": 0})['event']

        if not event:
            return False

        now = datetime.datetime.utcnow()
        update = {
            '$set': {"status": status},
            '$push': {
                "history": {
                    "event": event,
                    "status": status,
                    "text": text,
                    "id": id,
                    "updateTime": now
                }
            }
        }

        response = dBase.alerts.find_one_and_update(
            query,
            update=update,
            projection={"history": 0},
            return_document=ReturnDocument.AFTER
        )

        return AlertDocument(
            id=response['_id'],
            resource=response['resource'],
            event=response['event'],
            environment=response['environment'],
            severity=response['severity'],
            correlate=response['correlate'],
            status=response['status'],
            service=response['service'],
            group=response['group'],
            value=response['value'],
            text=response['text'],
            tags=response['tags'],
            attributes=response['attributes'],
            origin=response['origin'],
            event_type=response['type'],
            create_time=response['createTime'],
            timeout=response['timeout'],
            raw_data=response['rawData'],
            customer=response.get('customer', None),
            duplicate_count=response['duplicateCount'],
            repeat=response['repeat'],
            previous_severity=response['previousSeverity'],
            trend_indication=response['trendIndication'],
            receive_time=response['receiveTime'],
            last_receive_id=response['lastReceiveId'],
            last_receive_time=response['lastReceiveTime'],
            history=list()
        )
Example #13
0
    def save_correlated(self, alert, tenant):
        """
        Update alert key attributes, reset duplicate count and set repeat=False, keep track of last
        receive id and time, appending all to history. Append to history again if status changes.
        """
        dBase = self._client[tenant]

        previous_severity = self.get_severity(alert, tenant)
        previous_status = self.get_status(alert, tenant)
        trend_indication = severity_code.trend(previous_severity, alert.severity)
        if alert.status == status_code.UNKNOWN:
            status = severity_code.status_from_severity(previous_severity, alert.severity, previous_status)
        else:
            status = alert.status

        query = {
            "environment": alert.environment,
            "resource": alert.resource,
            '$or': [
                {
                    "event": alert.event,
                    "severity": {'$ne': alert.severity}
                },
                {
                    "event": {'$ne': alert.event},
                    "correlate": alert.event
                }],
            "customer": alert.customer
        }

        now = datetime.datetime.utcnow()
        update = {
            '$set': {
                "event": alert.event,
                "severity": alert.severity,
                "status": status,
                "value": alert.value,
                "text": alert.text,
                "tags": alert.tags,
                "attributes": alert.attributes,
                "createTime": alert.create_time,
                "rawData": alert.raw_data,
                "duplicateCount": 0,
                "repeat": False,
                "previousSeverity": previous_severity,
                "trendIndication": trend_indication,
                "receiveTime": now,
                "lastReceiveId": alert.id,
                "lastReceiveTime": now
            },
            '$pushAll': {
                "history": [{
                    "event": alert.event,
                    "severity": alert.severity,
                    "value": alert.value,
                    "text": alert.text,
                    "id": alert.id,
                    "updateTime": alert.create_time
                }]
            }
        }

        if status != previous_status:
            update['$pushAll']['history'].append({
                "event": alert.event,
                "status": status,
                "text": "correlated alert status change",
                "id": alert.id,
                "updateTime": now
            })

        LOG.debug('Update correlated alert in database: %s', update)
        response = dBase.alerts.find_one_and_update(
            query,
            update=update,
            projection={"history": 0},
            return_document=ReturnDocument.AFTER
        )

        return AlertDocument(
            id=response['_id'],
            resource=response['resource'],
            event=response['event'],
            environment=response['environment'],
            severity=response['severity'],
            correlate=response['correlate'],
            status=response['status'],
            service=response['service'],
            group=response['group'],
            value=response['value'],
            text=response['text'],
            tags=response['tags'],
            attributes=response['attributes'],
            origin=response['origin'],
            event_type=response['type'],
            create_time=response['createTime'],
            timeout=response['timeout'],
            raw_data=response['rawData'],
            customer=response.get('customer', None),
            duplicate_count=response['duplicateCount'],
            repeat=response['repeat'],
            previous_severity=response['previousSeverity'],
            trend_indication=response['trendIndication'],
            receive_time=response['receiveTime'],
            last_receive_id=response['lastReceiveId'],
            last_receive_time=response['lastReceiveTime'],
            history=list()
        )
Example #14
0
    def save_duplicate(self, alert, tenant):
        """
        Update alert value, text and rawData, increment duplicate count and set repeat=True, and
        keep track of last receive id and time but don't append to history unless status changes.
        """
        dBase = self._client[tenant]

        previous_status = self.get_status(alert, tenant)
        if alert.status != status_code.UNKNOWN and alert.status != previous_status:
            status = alert.status
        else:
            status = severity_code.status_from_severity(alert.severity, alert.severity, previous_status)

        query = {
            "environment": alert.environment,
            "resource": alert.resource,
            "event": alert.event,
            "severity": alert.severity,
            "customer": alert.customer
        }

        now = datetime.datetime.utcnow()
        update = {
            '$set': {
                "status": status,
                "value": alert.value,
                "text": alert.text,
                "rawData": alert.raw_data,
                "repeat": True,
                "lastReceiveId": alert.id,
                "lastReceiveTime": now
            },
            '$inc': {"duplicateCount": 1}
        }
        if status != previous_status:
            update['$push'] = {
                "history": {
                    "event": alert.event,
                    "status": status,
                    "text": "duplicate alert status change",
                    "id": alert.id,
                    "updateTime": now
                }
            }

        LOG.debug('Update duplicate alert in database: %s', update)
        response = dBase.alerts.find_one_and_update(
            query,
            update=update,
            projection={"history": 0},
            return_document=ReturnDocument.AFTER
        )

        return AlertDocument(
            id=response['_id'],
            resource=response['resource'],
            event=response['event'],
            environment=response['environment'],
            severity=response['severity'],
            correlate=response['correlate'],
            status=response['status'],
            service=response['service'],
            group=response['group'],
            value=response['value'],
            text=response['text'],
            tags=response['tags'],
            attributes=response['attributes'],
            origin=response['origin'],
            event_type=response['type'],
            create_time=response['createTime'],
            timeout=response['timeout'],
            raw_data=response['rawData'],
            customer=response.get('customer', None),
            duplicate_count=response['duplicateCount'],
            repeat=response['repeat'],
            previous_severity=response['previousSeverity'],
            trend_indication=response['trendIndication'],
            receive_time=response['receiveTime'],
            last_receive_id=response['lastReceiveId'],
            last_receive_time=response['lastReceiveTime'],
            history=list()
        )
Example #15
0
    def test_date_formats(self):

        self.CREATE_TIME = datetime.datetime(year=2012,
                                             month=11,
                                             day=10,
                                             hour=9,
                                             minute=8,
                                             second=7,
                                             microsecond=543210)

        alert = AlertDocument(id=self.ALERTID,
                              resource=self.RESOURCE,
                              event=self.EVENT,
                              environment=self.ENVIRONMENT,
                              severity=self.SEVERITY,
                              correlate=self.CORRELATE,
                              status=self.STATUS,
                              service=self.SERVICE,
                              group=self.GROUP,
                              value=self.VALUE,
                              text=self.TEXT,
                              tags=self.TAGS,
                              attributes={
                                  'thresholdInfo': self.THRESHOLD_INFO,
                                  'moreInfo': self.MORE_INFO,
                                  'graphUrls': self.GRAPH_URLS
                              },
                              origin=self.ORIGIN,
                              event_type=self.EVENT_TYPE,
                              create_time=self.CREATE_TIME,
                              timeout=self.TIMEOUT,
                              raw_data=self.RAW_DATA,
                              duplicate_count=self.DUPLICATE_COUNT,
                              repeat=self.REPEAT,
                              previous_severity=self.PREVIOUS_SEVERITY,
                              trend_indication=self.TREND_INDICATION,
                              receive_time=self.RECEIVE_TIME,
                              last_receive_id=self.ALERTID,
                              last_receive_time=self.RECEIVE_TIME,
                              history=self.HISTORY)

        self.assertEqual(alert.get_date('create_time', 'local'),
                         '2012/11/10 09:08:07')
        self.assertEqual(alert.get_date('create_time', 'iso'),
                         '2012-11-10T09:08:07.543Z')
        self.assertEqual(alert.get_date('create_time', 'iso8601'),
                         '2012-11-10T09:08:07.543Z')
        self.assertEqual(alert.get_date('create_time', 'rfc'),
                         'Sat, 10 Nov 2012 09:08:07 +0000')
        self.assertEqual(alert.get_date('create_time', 'rfc2822'),
                         'Sat, 10 Nov 2012 09:08:07 +0000')
        self.assertEqual(alert.get_date('create_time', 'short'),
                         'Sat 10 09:08:07')
        self.assertEqual(alert.get_date('create_time', 'epoch'), 1352538487.0)
        self.assertEqual(alert.get_date('create_time', 'raw'),
                         self.CREATE_TIME)
        self.assertEqual(alert.get_date('create_time'),
                         '2012-11-10T09:08:07.543Z')
        self.assertRaises(ValueError, alert.get_date, 'create_time', 'invalid')
Example #16
0
    def query(self, args, from_date=None):

        response = self._alerts(args.filters, from_date)
        alerts = response['alerts']

        if args.output == "json":
            print(json.dumps(alerts, indent=4))
            sys.exit(0)

        for alert in reversed(alerts):

            a = AlertDocument.parse_alert(alert)

            line_color = ''
            end_color = _ENDC

            if args.color:
                line_color = _COLOR_MAP.get(a.severity, _COLOR_MAP['unknown'])

            print(line_color + '{0}|{1}|{2}|{3:5d}|{4}|{5:<5s}|{6:<10s}|{7:<18s}|{8:12s}|{9:16s}|{10:12s}'.format(
                a.id[0:8],
                a.get_date('last_receive_time', 'local', args.timezone),
                a.severity,
                a.duplicate_count,
                a.customer or "-",
                a.environment,
                ','.join(a.service),
                a.resource,
                a.group,
                a.event,
                a.value) + end_color)
            print(line_color + '   |{}'.format(a.text) + end_color)

            if args.details:
                print(line_color + '    severity   | {} -> {}'.format(a.previous_severity, a.severity) + end_color)
                print(line_color + '    trend      | {}'.format(a.trend_indication) + end_color)
                print(line_color + '    status     | {}'.format(a.status) + end_color)
                print(line_color + '    resource   | {}'.format(a.resource) + end_color)
                print(line_color + '    group      | {}'.format(a.group) + end_color)
                print(line_color + '    event      | {}'.format(a.event) + end_color)
                print(line_color + '    value      | {}'.format(a.value) + end_color)
                print(line_color + '    tags       | {}'.format(' '.join(a.tags)) + end_color)

                for key, value in a.attributes.items():
                    print(line_color + '    {} | {}'.format(key.ljust(10), value) + end_color)

                latency = a.receive_time - a.create_time

                print(line_color + '        time created  | {}'.format(a.get_date('create_time', 'iso', args.timezone)) + end_color)
                print(line_color + '        time received | {}'.format(a.get_date('receive_time', 'iso', args.timezone)) + end_color)
                print(line_color + '        last received | {}'.format(a.get_date('last_receive_time', 'iso', args.timezone)) + end_color)
                print(line_color + '        latency       | {}ms'.format((latency.microseconds / 1000)) + end_color)
                print(line_color + '        timeout       | {}s'.format(a.timeout) + end_color)

                print(line_color + '            alert id     | {}'.format(a.id) + end_color)
                print(line_color + '            last recv id | {}'.format(a.last_receive_id) + end_color)
                print(line_color + '            customer     | {}'.format(a.customer) + end_color)
                print(line_color + '            environment  | {}'.format(a.environment) + end_color)
                print(line_color + '            service      | {}'.format(','.join(a.service)) + end_color)
                print(line_color + '            resource     | {}'.format(a.resource) + end_color)
                print(line_color + '            type         | {}'.format(a.event_type) + end_color)
                print(line_color + '            repeat       | {}'.format(a.repeat) + end_color)
                print(line_color + '            origin       | {}'.format(a.origin) + end_color)
                print(line_color + '            correlate    | {}'.format(','.join(a.correlate)) + end_color)

        return response.get('lastTime', '')