Beispiel #1
0
def setIncidentAutoSubsequentResolved(context, index, sessionKey):
    if not context.get('title'):
        query = '{  "alert": "' + context.get(
            'name'
        ) + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "' + context.get(
            'job_id') + '"} }'
    else:
        log.debug(
            "Using title '%s' to search for incidents to auto subsequent resolve."
            % context.get('title'))
        query = '{  "title": "' + context.get(
            'title'
        ) + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "' + context.get(
            'job_id') + '"} }'

    uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
        query)
    prev_incidents = getRestData(uri, sessionKey, output_mode='default')
    if len(prev_incidents) > 0:
        prev_incident = prev_incidents[0]
        log.info("Found '%s' as pre-existing incident" %
                 prev_incident['incident_id'])

        # Set status of current incident and fire event
        setStatus(context.get('_key'), context.get('incident_id'),
                  'auto_subsequent_resolved', sessionKey)
        event = 'severity=INFO origin="alert_handler" user="******" action="auto_subsequent_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (
            context.get('status'), context.get('incident_id'),
            context.get('job_id'))
        createIncidentChangeEvent(event, context.get('job_id'), index)

        ic = IncidentContext(sessionKey, incident_id)
        eh.handleEvent(alert=context.get('name'),
                       event="incident_auto_subsequent_resolved",
                       incident={"owner": context.get("owner")},
                       context=ic.getContext())

        # Update history of pre-existing incident and fire event
        event = 'severity=INFO origin="alert_handler" user="******" action="new_subsequent_incident" incident_id="%s" new_incident_id="%s"' % (
            prev_incident['incident_id'], context.get('incident_id'))
        createIncidentChangeEvent(context.get('event'), context.get('job_id'),
                                  index)

        ic = IncidentContext(sessionKey, prev_incident['incident_id'])
        eh.handleEvent(alert=context.get('name'),
                       event="incident_new_subsequent_incident",
                       incident=prev_incident,
                       context=ic.getContext())
        return True
    else:
        log.info(
            "No pre-existing incidents with matching criteria for auto_subsequent_resolve found, keep this one open."
        )
        return False
Beispiel #2
0
def autoPreviousResolve(alert, job_id, title):
    # Auto Previous resolve
    log.info(
        "auto_previous_resolve is active for alert %s, searching for incidents to resolve..."
        % alert)
    if title == "":
        query = '{  "alert": "' + alert + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + job_id + '"} }'
    else:
        log.debug(
            "Using title (%s) to search for incidents to auto previous resolve."
            % title)
        query = '{  "title": "' + title + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + job_id + '"} }'

    log.debug("Filter for auto_previous_resolve: %s" % query)
    uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
        query)
    serverResponse, serverContent = rest.simpleRequest(uri,
                                                       sessionKey=sessionKey)
    incidents = json.loads(serverContent)
    if len(incidents) > 0:
        log.info("Got %s incidents to auto-resolve" % len(incidents))
        for incident in incidents:
            log.info("Auto-resolving incident with key=%s" % incident['_key'])

            previous_status = incident["status"]
            previous_job_id = incident["job_id"]
            previous_incident_id = incident["incident_id"]
            previous_owner = incident["owner"]

            incident['status'] = 'auto_previous_resolved'
            uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident[
                '_key']
            incident = json.dumps(incident)
            serverResponse, serverContent = rest.simpleRequest(
                uri, sessionKey=sessionKey, jsonargs=incident)

            now = datetime.datetime.now().isoformat()
            event_id = hashlib.md5(job_id + now).hexdigest()
            log.debug("event_id=%s now=%s incident=%s" %
                      (event_id, now, incident))

            event = 'time=%s severity=INFO origin="alert_handler" event_id="%s" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (
                now, event_id, previous_status, previous_incident_id,
                previous_job_id)
            log.debug("Resolve event will be: %s" % event)
            input.submit(event,
                         hostname=socket.gethostname(),
                         sourcetype='incident_change',
                         source='alert_handler.py',
                         index=config['index'])

            ic = IncidentContext(sessionKey, previous_incident_id)
            eh.handleEvent(alert=alert,
                           event="incident_auto_previous_resolved",
                           incident={"owner": previous_owner},
                           context=ic.getContext())
    else:
        log.info(
            "No incidents with matching criteria for auto_previous_resolve found."
        )
def setIncidentsAutoResolved(alert, job_id, title, index, sessionKey):
    if title == "":
        query = (
            '{  "alert": "'
            + alert
            + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'
            + job_id
            + '"} }'
        )
    else:
        log.debug("Using title '%s' to search for incidents to auto previous resolve." % title)
        query = (
            '{  "title": "'
            + title
            + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'
            + job_id
            + '"} }'
        )

    uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s" % urllib.quote(query)
    incidents = getRestData(uri, sessionKey, output_mode="default")
    if len(incidents) > 0:
        log.info("Got %s incidents to auto-resolve" % len(incidents))
        for incident in incidents:
            log.info("Auto-resolving incident with key=%s" % incident["_key"])

            previous_status = incident["status"]
            previous_job_id = incident["job_id"]
            previous_incident_id = incident["incident_id"]
            previous_owner = incident["owner"]

            incident["status"] = "auto_previous_resolved"
            uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s" % incident["_key"]
            getRestData(uri, sessionKey, json.dumps(incident))

            event = (
                'severity=INFO origin="alert_handler" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"'
                % (previous_status, previous_incident_id, previous_job_id)
            )
            createIncidentChangeEvent(event, job_id, index)

            ic = IncidentContext(sessionKey, previous_incident_id)
            eh.handleEvent(
                alert=alert,
                event="incident_auto_previous_resolved",
                incident={"owner": previous_owner},
                context=ic.getContext(),
            )
    else:
        log.info("No incidents with matching criteria for auto_previous_resolve found.")
Beispiel #4
0
def setIncidentsAutoPreviousResolved(context, index, sessionKey):
    if not context.get('title'):
        query = '{  "alert": "' + context.get(
            'name'
        ) + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + context.get(
            'job_id') + '"} }'
    else:
        log.debug(
            "Using title '%s' to search for incidents to auto previous resolve."
            % context.get('title'))
        query = '{  "title": "' + context.get(
            'title'
        ) + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + context.get(
            'job_id') + '"} }'

    uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
        query)
    incidents = getRestData(uri, sessionKey, output_mode='default')
    if len(incidents) > 0:
        log.info("Got %s incidents to auto-resolve" % len(incidents))
        for incident in incidents:
            log.info("Auto-resolving incident with key=%s" % incident['_key'])

            previous_status = incident["status"]
            previous_job_id = incident["job_id"]
            previous_incident_id = incident["incident_id"]
            previous_owner = incident["owner"]

            incident['status'] = 'auto_previous_resolved'
            uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident[
                '_key']
            getRestData(uri, sessionKey, json.dumps(incident))

            event = 'severity=INFO origin="alert_handler" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s" resolving_incident="%s"' % (
                previous_status, previous_incident_id, previous_job_id,
                context.get('incident_id'))
            createIncidentChangeEvent(event, previous_job_id, index)

            ic = IncidentContext(sessionKey, previous_incident_id)
            eh.handleEvent(alert=context.get('name'),
                           event="incident_auto_previous_resolved",
                           incident={"owner": previous_owner},
                           context=ic.getContext())
    else:
        log.info(
            "No incidents with matching criteria for auto_previous_resolve found."
        )
Beispiel #5
0
def setIncidentAutoInfoResolved(context, index, sessionKey, statusval):
    log.info('Resolving incident %s per settings.' %
             context.get('incident_id'))

    # set the status of the incident to the configured resolution status
    setStatus(context.get('_key'), context.get('incident_id'), statusval,
              sessionKey)

    # create and index a change event
    event = 'severity=INFO origin="alert_handler" user="******" action="auto_informational_resolve" previous_status="%s" status="%s" incident_id="%s" job_id="%s"' % (
        context.get('status'), statusval, context.get('incident_id'),
        context.get('job_id'))
    createIncidentChangeEvent(event, context.get('job_id'), index)

    # create a context run the event handler
    ic = IncidentContext(sessionKey, incident_id)
    eh.handleEvent(alert=context.get('name'),
                   event="auto_informational_resolve",
                   incident={"owner": context.get("owner")},
                   context=ic.getContext())
Beispiel #6
0
writeResultToCollection(results)
log.info(
    "Alert results for job_id=%s incident_id=%s result_id=%s written to collection incident_results"
    % (job_id, incident_id, str(result_id)))

# Write metadata to index
writeAlertMetadataToIndex(job, incident_id, result_id)

# Fire incident_created or incident_suppressed event
ic = IncidentContext(sessionKey, incident_id)
if incident_suppressed == False:
    log.info("Firing incident_created event for incident=%s" % incident_id)
    eh.handleEvent(alert=alert,
                   event="incident_created",
                   incident={"owner": config['default_owner']},
                   context=ic.getContext())
else:
    log.info("Firing incident_suppressed event for incident=%s" % incident_id)
    eh.handleEvent(alert=alert,
                   event="incident_suppressed",
                   incident={"owner": config['default_owner']},
                   context=ic.getContext())

# Handle auto-assign
if incident_config['auto_assign'] and incident_config[
        'auto_assign_owner'] != 'unassigned' and incident_suppressed == False:
    log.info("auto_assign is active for %s. Starting to handle it." % alert)
    assignIncident(incident_key, incident_id,
                   incident_config['auto_assign_owner'])
    logAutoAssignEvent(incident_id, job_id, result_id,
                       incident_config['auto_assign_owner'])
Beispiel #7
0
    def save(self, contents, **kwargs):

        logger.info("Saving incident settings contents...")

        user = cherrypy.session["user"]["name"]
        sessionKey = cherrypy.session.get("sessionKey")
        splunk.setDefault("sessionKey", sessionKey)

        eh = EventHandler(sessionKey=sessionKey)

        config = {}
        config["index"] = "alerts"

        restconfig = entity.getEntities("configs/alert_manager", count=-1, sessionKey=sessionKey)
        if len(restconfig) > 0:
            if "index" in restconfig["settings"]:
                config["index"] = restconfig["settings"]["index"]

        logger.debug("Global settings: %s" % config)

        # Parse the JSON
        contents = json.loads(contents)

        logger.debug("Contents: %s" % json.dumps(contents))

        # Get key
        query = {}
        query["incident_id"] = contents["incident_id"]
        logger.debug("Filter: %s" % json.dumps(query))

        uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s" % urllib.quote(
            json.dumps(query)
        )
        serverResponse, incident = rest.simpleRequest(uri, sessionKey=sessionKey)
        logger.debug("Settings for incident: %s" % incident)
        incident = json.loads(incident)

        # Update incident
        uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incidents/" + incident[0]["_key"]
        logger.debug("URI for incident update: %s" % uri)

        # Prepared new entry
        now = datetime.datetime.now().isoformat()
        changed_keys = []
        for key in incident[0].keys():
            if (key in contents) and (incident[0][key] != contents[key]):
                changed_keys.append(key)
                logger.info(
                    "%s for incident %s changed. Writing change event to index %s."
                    % (key, incident[0]["incident_id"], config["index"])
                )
                event_id = hashlib.md5(incident[0]["incident_id"] + now).hexdigest()
                event = (
                    'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"'
                    % (now, event_id, user, incident[0]["incident_id"], key, contents[key], key, incident[0][key])
                )
                logger.debug("Change event will be: %s" % event)
                input.submit(
                    event,
                    hostname=socket.gethostname(),
                    sourcetype="incident_change",
                    source="incident_settings.py",
                    index=config["index"],
                )
                incident[0][key] = contents[key]

            else:
                logger.info("%s for incident %s didn't change." % (key, incident[0]["incident_id"]))

        del incident[0]["_key"]
        contentsStr = json.dumps(incident[0])
        logger.debug("content for update: %s" % contentsStr)
        serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=contentsStr)

        logger.debug("Response from update incident entry was %s " % serverResponse)
        logger.debug("Changed keys: %s" % changed_keys)

        if len(changed_keys) > 0:
            ic = IncidentContext(sessionKey, contents["incident_id"])
            if "owner" in changed_keys:
                eh.handleEvent(
                    alert=incident[0]["alert"], event="incident_assigned", incident=incident[0], context=ic.getContext()
                )
            elif "status" in changed_keys and contents["status"] == "resolved":
                eh.handleEvent(
                    alert=incident[0]["alert"], event="incident_resolved", incident=incident[0], context=ic.getContext()
                )
            else:
                eh.handleEvent(
                    alert=incident[0]["alert"], event="incident_changed", incident=incident[0], context=ic.getContext()
                )

        if contents["comment"] != "":
            event_id = hashlib.md5(incident[0]["incident_id"] + now).hexdigest()
            event = (
                'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"'
                % (now, event_id, user, incident[0]["incident_id"], contents["comment"])
            )
            logger.debug("Comment event will be: %s" % event)
            event = event.encode("utf8")
            input.submit(
                event,
                hostname=socket.gethostname(),
                sourcetype="incident_change",
                source="incident_settings.py",
                index=config["index"],
            )

        return "Done"
    def save(self, contents, **kwargs):


        logger.info("Saving incident settings contents...")

        user = cherrypy.session['user']['name']
        sessionKey = cherrypy.session.get('sessionKey')
        splunk.setDefault('sessionKey', sessionKey)

        eh = EventHandler(sessionKey = sessionKey)

        config = {}
        config['index'] = 'alerts'
        
        restconfig = entity.getEntities('configs/alert_manager', count=-1, sessionKey=sessionKey)
        if len(restconfig) > 0:
            if 'index' in restconfig['settings']:
                config['index'] = restconfig['settings']['index']

        logger.debug("Global settings: %s" % config)

        # Parse the JSON
        contents = json.loads(contents)

        logger.debug("Contents: %s" % json.dumps(contents))

        # Get key
        query = {}
        query['incident_id'] = contents['incident_id']
        logger.debug("Filter: %s" % json.dumps(query))

        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(json.dumps(query))
        serverResponse, incident = rest.simpleRequest(uri, sessionKey=sessionKey)
        logger.debug("Settings for incident: %s" % incident)
        incident = json.loads(incident)

        # Update incident
        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/' + incident[0]['_key']
        logger.debug("URI for incident update: %s" % uri )

        # Prepared new entry
        now = datetime.datetime.now().isoformat()
        changed_keys = []
        for key in incident[0].keys():
            if (key in contents) and (incident[0][key] != contents[key]):
                changed_keys.append(key)
                logger.info("%s for incident %s changed. Writing change event to index %s." % (key, incident[0]['incident_id'], config['index']))
                event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
                event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"' % (now, event_id, user, incident[0]['incident_id'], key, contents[key], key, incident[0][key])
                logger.debug("Change event will be: %s" % event)
                input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
                incident[0][key] = contents[key]

            else:
                logger.info("%s for incident %s didn't change." % (key, incident[0]['incident_id']))

        del incident[0]['_key']
        contentsStr = json.dumps(incident[0])
        logger.debug("content for update: %s" % contentsStr)
        serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=contentsStr)

        logger.debug("Response from update incident entry was %s " % serverResponse)
        logger.debug("Changed keys: %s" % changed_keys)

        if len(changed_keys) > 0:
            ic = IncidentContext(sessionKey, contents['incident_id'])
            if "owner" in changed_keys:
                eh.handleEvent(alert=incident[0]["alert"], event="incident_assigned", incident=incident[0], context=ic.getContext())
            elif "status" in changed_keys and contents["status"] == "resolved":
                eh.handleEvent(alert=incident[0]["alert"], event="incident_resolved", incident=incident[0], context=ic.getContext())
            else:
                eh.handleEvent(alert=incident[0]["alert"], event="incident_changed", incident=incident[0], context=ic.getContext())
        
        if contents['comment'] != "":
            event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
            event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"' % (now, event_id, user, incident[0]['incident_id'], contents['comment'])
            logger.debug("Comment event will be: %s" % event)
            event = event.encode('utf8')
            input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
        
        
        return 'Done'
Beispiel #9
0
if incident_suppressed:
    logSuppressEvent(alert, incident_id, job_id, result_id, rule_names)

# Write results to collection
writeResultToCollection(results)
log.info("Alert results for job_id=%s incident_id=%s result_id=%s written to collection incident_results" % (job_id, incident_id, str(result_id)))

# Write metadata to index
writeAlertMetadataToIndex(job, incident_id, result_id)

# Fire incident_created or incident_suppressed event
ic = IncidentContext(sessionKey, incident_id)
if incident_suppressed == False:
    log.info("Firing incident_created event for incident=%s" % incident_id)
    eh.handleEvent(alert=alert, event="incident_created", incident={"owner": config['default_owner']}, context=ic.getContext())
else:
    log.info("Firing incident_suppressed event for incident=%s" % incident_id)
    eh.handleEvent(alert=alert, event="incident_suppressed", incident={"owner": config['default_owner']}, context=ic.getContext())

# Handle auto-assign
if incident_config['auto_assign'] and incident_config['auto_assign_owner'] != 'unassigned' and incident_suppressed == False:
    log.info("auto_assign is active for %s. Starting to handle it." % alert)
    assignIncident(incident_key, incident_id, incident_config['auto_assign_owner'])
    logAutoAssignEvent(incident_id, job_id, result_id, incident_config['auto_assign_owner'])
    eh.handleEvent(alert=alert, event="incident_auto_assigned", incident={"owner": incident_config["auto_assign_owner"]}, context=ic.getContext())

# Auto Previous Resolve - run only once
if incident_config['auto_previous_resolve'] and incident_suppressed == False:
    log.info("auto_previous_resolve is active for %s. Starting to handle it." % alert)
    autoPreviousResolve(alert, job_id, job['title'])
Beispiel #10
0
def setIncidentAutoSubsequentResolved(context, index, sessionKey):
    if not context.get('title'):
        query = '{  "alert": "'+ context.get('name') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
    else:
        log.debug("Using title '%s' to search for incidents to auto subsequent resolve." % context.get('title'))
        query = '{  "title": "'+ context.get('title') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'

    uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(query)
    prev_incidents = getRestData(uri, sessionKey, output_mode = 'default')
    if len(prev_incidents) > 0:
        prev_incident = prev_incidents[0]
        log.info("Found '%s' as pre-existing incident" % prev_incident['incident_id'])

        # Set status of current incident and fire event
        setStatus(context.get('_key'), context.get('incident_id'), 'auto_subsequent_resolved', sessionKey)
        event = 'severity=INFO origin="alert_handler" user="******" action="auto_subsequent_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (context.get('status'), context.get('incident_id'), context.get('job_id'))
        createIncidentChangeEvent(event, context.get('job_id'), index)

        ic = IncidentContext(sessionKey, incident_id)
        eh.handleEvent(alert=context.get('name'), event="incident_auto_subsequent_resolved", incident={"owner": context.get("owner")}, context=ic.getContext())

        # Update history of pre-existing incident and fire event
        event = 'severity=INFO origin="alert_handler" user="******" action="new_subsequent_incident" incident_id="%s" new_incident_id="%s"' % (prev_incident['incident_id'], context.get('incident_id'))
        createIncidentChangeEvent(context.get('event'), context.get('job_id'), index)

        ic = IncidentContext(sessionKey, prev_incident['incident_id'])
        eh.handleEvent(alert=context.get('name'), event="incident_new_subsequent_incident", incident=prev_incident, context=ic.getContext())
        return True
    else:
        log.info("No pre-existing incidents with matching criteria for auto_subsequent_resolve found, keep this one open.")
        return False
                            log.info("Incident %s (%s) should be resolved. alert_time=%s ttl=%s now=%s" % (incident['incident_id'], incident['_key'], incident['alert_time'], incident['ttl'], time.time()))
                            old_status = incident['status']
                            incident['status'] = 'auto_ttl_resolved'
                            uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
                            incidentStr = json.dumps(incident)
                            serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=incidentStr)
                            
                            now = datetime.datetime.now().isoformat()
                            event_id = hashlib.md5(incident['incident_id'] + now).hexdigest()
                            log.debug("event_id=%s now=%s" % (event_id, now))

                            event = 'time=%s severity=INFO origin="alert_manager_scheduler" event_id="%s" user="******" action="auto_ttl_resolve" previous_status="%s" status="auto_ttl_resolved" incident_id="%s"' % (now, event_id, old_status, incident['incident_id'])
                            log.debug("Event will be: %s" % event)
                            input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'alert_manager_scheduler.py', index = config['index'])
                            ic = IncidentContext(sessionKey, incident["incident_id"])
                            eh.handleEvent(alert=alert["name"], event="incident_auto_ttl_resolved", incident={"owner": incident["owner"]}, context=ic.getContext())
                        else:
                            log.info("Incident %s has not ttl reached yet." % incident['incident_id'])
                else:
                    log.info("No incidents of alert %s to check for reached ttl." % alert['name'])
            log.debug('Alert "%s" is not configured for auto_ttl_resolve, skipping...' % alert['name'])

    #
    # Look for auto_suppress_resolve incidents
    #
    query = {}
    query['auto_suppress_resolve'] = True
    log.debug("Filter: %s" % json.dumps(query))
    uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_settings?query=%s' % urllib.quote(json.dumps(query))
    serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
    try:
    def _update_incident(self, sessionKey, user, post_data):
        logger.debug("START _update_incident()")

        required = ['incident_data']
        missing = [r for r in required if r not in post_data]
        if missing:
            return self.response("Missing required arguments: %s" % missing, httplib.BAD_REQUEST)

        incident_data = post_data.pop('incident_data')

        splunk.setDefault('sessionKey', sessionKey)

        eh = EventHandler(sessionKey = sessionKey)

        config = {}
        config['index'] = 'main'

        restconfig = entity.getEntities('configs/alert_manager', count=-1, sessionKey=sessionKey)
        if len(restconfig) > 0:
            if 'index' in restconfig['settings']:
                config['index'] = restconfig['settings']['index']

        logger.debug("Global settings: %s" % config)

        # Parse the JSON
        incident_data = json.loads(incident_data)

        # Get key
        query = {}
        query['incident_id'] = incident_data['incident_id']
        logger.debug("Filter: %s" % json.dumps(query))

        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(json.dumps(query))
        serverResponse, incident = rest.simpleRequest(uri, sessionKey=sessionKey)
        logger.debug("Settings for incident: %s" % incident)
        incident = json.loads(incident)

        # Update incident
        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/' + incident[0]['_key']
        logger.debug("URI for incident update: %s" % uri )

        # Prepared new entry
        now = datetime.datetime.now().isoformat()
        changed_keys = []
        for key in incident[0].keys():
            if (key in incident_data) and (incident[0][key] != incident_data[key]):
                changed_keys.append(key)
                logger.info("%s for incident %s changed. Writing change event to index %s." % (key, incident[0]['incident_id'], config['index']))
                event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
                event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"' % (now, event_id, user, incident[0]['incident_id'], key, incident_data[key], key, incident[0][key])
                logger.debug("Change event will be: %s" % event)
                input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
                incident[0][key] = incident_data[key]

            else:
                logger.info("%s for incident %s didn't change." % (key, incident[0]['incident_id']))

        del incident[0]['_key']
        contentsStr = json.dumps(incident[0])
        logger.debug("content for update: %s" % contentsStr)
        serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=contentsStr)

        logger.debug("Response from update incident entry was %s " % serverResponse)
        logger.debug("Changed keys: %s" % changed_keys)

        if len(changed_keys) > 0:
            ic = IncidentContext(sessionKey, incident_data['incident_id'])
            if "owner" in changed_keys:
                eh.handleEvent(alert=incident[0]["alert"], event="incident_assigned", incident=incident[0], context=ic.getContext())
            elif "status" in changed_keys and incident_data["status"] == "resolved":
                eh.handleEvent(alert=incident[0]["alert"], event="incident_resolved", incident=incident[0], context=ic.getContext())
            else:
                eh.handleEvent(alert=incident[0]["alert"], event="incident_changed", incident=incident[0], context=ic.getContext())

        if incident_data['comment'] != "":
            incident_data['comment'] = incident_data['comment'].replace('\n', '<br />').replace('\r', '')
            event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
            event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"' % (now, event_id, user, incident[0]['incident_id'], incident_data['comment'])
            logger.debug("Comment event will be: %s" % event)
            event = event.encode('utf8')
            input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
            ic = IncidentContext(sessionKey, incident_data['incident_id'])
            eh.handleEvent(alert=incident[0]["alert"], event="incident_commented", incident=incident[0], context=ic.getContext())


        return self.response('Successfully updated incident.', httplib.OK)
            event = 'severity=INFO origin="alert_handler" user="******" action="suppress" alert="%s" incident_id="%s" job_id="%s" result_id="%s" %s' % ('splunk-system-user', search_name, incident_id, job_id, result_id, rules)
            createIncidentChangeEvent(event, metadata['job_id'], settings.get('index'))

        # Write results to collection
        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_results'
        response = getRestData(uri, sessionKey, json.dumps(results))
        log.info("Results for incident_id=%s written to collection." % (incident_id))

        # Write metadata to index
        createMetadataEvent(metadata, settings.get('index'), sessionKey)

        # Fire incident_created or incident_suppressed event
        ic = IncidentContext(sessionKey, incident_id)
        if incident_suppressed == False:
            log.info("Firing incident_created event for incident=%s" % incident_id)
            eh.handleEvent(alert=search_name, event="incident_created", incident={"owner": settings.get('default_owner')}, context=ic.getContext())
        else:
            log.info("Firing incident_suppressed event for incident=%s" % incident_id)
            eh.handleEvent(alert=search_name, event="incident_suppressed", incident={"owner": settings.get('default_owner')}, context=ic.getContext())

        # Handle auto-assign
        if config['auto_assign_owner'] != '' and config['auto_assign_owner'] != 'unassigned' and incident_suppressed == False:
            log.debug("auto_assign is active for %s. Starting to handle it." % search_name)
            setOwner(incident_key, incident_id, config['auto_assign_owner'], sessionKey)
            setStatus(incident_key, incident_id, 'auto_assigned', sessionKey)
            ic.update("owner", config['auto_assign_owner'])

            event = 'severity=INFO origin="alert_handler" user="******" action="change" incident_id="%s" job_id="%s" result_id="%s" owner="%s" previous_owner="unassigned"' % (incident_id, job_id, result_id, config['auto_assign_owner'])
            createIncidentChangeEvent(event, metadata['job_id'], settings.get('index'))

            event = 'severity=INFO origin="alert_handler" user="******" action="change" incident_id="%s" job_id="%s" result_id="%s" status="auto_assigned" previous_status="new"' % (incident_id, job_id, result_id)
Beispiel #14
0
def autoPreviousResolve(alert, job_id, title):
    # Auto Previous resolve
    log.info("auto_previous_resolve is active for alert %s, searching for incidents to resolve..." % alert)
    if title == "":
        query = '{  "alert": "'+ alert +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ job_id +'"} }'
    else:
        log.debug("Using title (%s) to search for incidents to auto previous resolve." % title)
        query = '{  "title": "'+ title +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ job_id +'"} }'

    log.debug("Filter for auto_previous_resolve: %s" % query)
    uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(query)
    serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
    incidents = json.loads(serverContent)
    if len(incidents) > 0:
        log.info("Got %s incidents to auto-resolve" % len(incidents))
        for incident in incidents:
            log.info("Auto-resolving incident with key=%s" % incident['_key'])

            previous_status = incident["status"]
            previous_job_id = incident["job_id"]
            previous_incident_id = incident["incident_id"]
            previous_owner = incident["owner"]

            incident['status'] = 'auto_previous_resolved'
            uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
            incident = json.dumps(incident)
            serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=incident)

            now = datetime.datetime.now().isoformat()
            event_id = hashlib.md5(job_id + now).hexdigest()
            log.debug("event_id=%s now=%s incident=%s" % (event_id, now, incident))


            event = 'time=%s severity=INFO origin="alert_handler" event_id="%s" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (now, event_id, previous_status, previous_incident_id, previous_job_id)
            log.debug("Resolve event will be: %s" % event)
            input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'alert_handler.py', index = config['index'])

            ic = IncidentContext(sessionKey, previous_incident_id)
            eh.handleEvent(alert=alert, event="incident_auto_previous_resolved", incident={"owner": previous_owner}, context=ic.getContext())
    else:
        log.info("No incidents with matching criteria for auto_previous_resolve found.")
Beispiel #15
0
            setIncidentsAutoPreviousResolved(ic, settings.get('index'), sessionKey)

        elif config['auto_subsequent_resolve'] and incident_suppressed == False:
            log.debug("auto_subsequent_resolve is active for %s. Starting to handle it." % search_name)
            is_subsequent_resolved = setIncidentAutoSubsequentResolved(ic, settings.get('index'), sessionKey)


        # Fire incident_created or incident_suppressed event
        # only if it was not deemed a duplicate
        if is_subsequent_resolved:
            log.info("Skipping firing of incident_created event for incident=%s because it is a duplicate." % incident_id)

        else:
            if incident_suppressed == False:
                log.info("Firing incident_created event for incident=%s" % incident_id)
                eh.handleEvent(alert=search_name, event="incident_created", incident={"owner": settings.get('default_owner')}, context=ic.getContext())
            else:
                log.info("Firing incident_suppressed event for incident=%s" % incident_id)
                eh.handleEvent(alert=search_name, event="incident_suppressed", incident={"owner": settings.get('default_owner')}, context=ic.getContext())

        # If the incident was not resolved already, auto resolved is enabled, and priority is informational - resolve it.
        auto_info_resolved = False
        if is_subsequent_resolved == False:
            # This automatic resolution is optional.
            try:
                if normalize_bool(settings.get('auto_close_info')) and metadata['priority'] == 'informational':
                    log.debug('Auto close informational is on')
                    setIncidentAutoInfoResolved(ic, settings.get('index'), sessionKey, settings.get('auto_close_info_status'))
                    auto_info_resolved = True

            except:
                        log.info("Incident %s (%s) should be resolved. alert_time=%s ttl=%s now=%s" % (incident['incident_id'], incident['_key'], incident['alert_time'], incident['ttl'], time.time()))
                        old_status = incident['status']
                        incident['status'] = 'auto_ttl_resolved'
                        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
                        incidentStr = json.dumps(incident)
                        serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=incidentStr)
                        
                        now = datetime.datetime.now().isoformat()
                        event_id = hashlib.md5(incident['incident_id'] + now).hexdigest()
                        log.debug("event_id=%s now=%s" % (event_id, now))

                        event = 'time=%s severity=INFO origin="alert_manager_scheduler" event_id="%s" user="******" action="auto_ttl_resolve" previous_status="%s" status="auto_ttl_resolved" incident_id="%s"' % (now, event_id, old_status, incident['incident_id'])
                        log.debug("Event will be: %s" % event)
                        input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'alert_manager_scheduler.py', index = config['index'])
                        ic = IncidentContext(sessionKey, incident["incident_id"])
                        eh.handleEvent(alert=alert["name"], event="incident_auto_ttl_resolved", incident={"owner": incident["owner"]}, context=ic.getContext())
                    else:
                        log.info("Incident %s has not ttl reached yet." % incident['incident_id'])
            else:
                log.info("No incidents of alert %s to check for reached ttl." % alert['name'])
        log.debug('Alert "%s" is not configured for auto_ttl_resolve, skipping...' % alert['name'])

# Look for auto_suppress_resolve incidents
query = {}
query['auto_suppress_resolve'] = True
log.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_settings?query=%s' % urllib.quote(json.dumps(query))
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
alerts = json.loads(serverContent)
if len(alerts) >0:
    for alert in alerts:
Beispiel #17
0
def setIncidentsAutoPreviousResolved(context, index, sessionKey):
    if not context.get('title'):
        query = '{  "alert": "'+ context.get('name') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
    else:
        log.debug("Using title '%s' to search for incidents to auto previous resolve." % context.get('title'))
        query = '{  "title": "'+ context.get('title') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'

    uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(query)
    incidents = getRestData(uri, sessionKey, output_mode = 'default')
    if len(incidents) > 0:
        log.info("Got %s incidents to auto-resolve" % len(incidents))
        for incident in incidents:
            log.info("Auto-resolving incident with key=%s" % incident['_key'])

            previous_status = incident["status"]
            previous_job_id = incident["job_id"]
            previous_incident_id = incident["incident_id"]
            previous_owner = incident["owner"]

            incident['status'] = 'auto_previous_resolved'
            uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
            getRestData(uri, sessionKey, json.dumps(incident))

            event = 'severity=INFO origin="alert_handler" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s" resolving_incident="%s"' % (previous_status, previous_incident_id, previous_job_id, context.get('incident_id'))
            createIncidentChangeEvent(event, previous_job_id, index)

            ic = IncidentContext(sessionKey, previous_incident_id)
            eh.handleEvent(alert=context.get('name'), event="incident_auto_previous_resolved", incident={"owner": previous_owner}, context=ic.getContext())
    else:
        log.info("No incidents with matching criteria for auto_previous_resolve found.")
        uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incident_results"
        response = getRestData(uri, sessionKey, json.dumps(results))
        log.info("Results for incident_id=%s written to collection." % (incident_id))

        # Write metadata to index
        createMetadataEvent(metadata, settings.get("index"), sessionKey)

        # Fire incident_created or incident_suppressed event
        ic = IncidentContext(sessionKey, incident_id)
        if incident_suppressed == False:
            log.info("Firing incident_created event for incident=%s" % incident_id)
            eh.handleEvent(
                alert=search_name,
                event="incident_created",
                incident={"owner": settings.get("default_owner")},
                context=ic.getContext(),
            )
        else:
            log.info("Firing incident_suppressed event for incident=%s" % incident_id)
            eh.handleEvent(
                alert=search_name,
                event="incident_suppressed",
                incident={"owner": settings.get("default_owner")},
                context=ic.getContext(),
            )

        # Handle auto-assign
        if (
            config["auto_assign_owner"] != ""
            and config["auto_assign_owner"] != "unassigned"
            and incident_suppressed == False
Beispiel #19
0
def setIncidentAutoInfoResolved(context, index, sessionKey, statusval):
    log.info('Resolving incident %s per settings.' % context.get('incident_id'))

    # set the status of the incident to the configured resolution status
    setStatus(context.get('_key'), context.get('incident_id'), statusval, sessionKey)

    # create and index a change event
    event = 'severity=INFO origin="alert_handler" user="******" action="auto_informational_resolve" previous_status="%s" status="%s" incident_id="%s" job_id="%s"' % (context.get('status'), statusval, context.get('incident_id'), context.get('job_id'))
    createIncidentChangeEvent(event, context.get('job_id'), index)

    # create a context run the event handler
    ic = IncidentContext(sessionKey, incident_id)
    eh.handleEvent(alert=context.get('name'), event="auto_informational_resolve", incident={"owner": context.get("owner")}, context=ic.getContext())
Beispiel #20
0
    def _update_incident(self, sessionKey, user, post_data):
        logger.debug("START _update_incident()")

        required = ['incident_data']
        missing = [r for r in required if r not in post_data]
        if missing:
            return self.response("Missing required arguments: %s" % missing,
                                 httplib.BAD_REQUEST)

        incident_data = post_data.pop('incident_data')

        splunk.setDefault('sessionKey', sessionKey)

        eh = EventHandler(sessionKey=sessionKey)

        config = {}
        config['index'] = 'main'

        restconfig = entity.getEntities('configs/alert_manager',
                                        count=-1,
                                        sessionKey=sessionKey)
        if len(restconfig) > 0:
            if 'index' in restconfig['settings']:
                config['index'] = restconfig['settings']['index']

        logger.debug("Global settings: %s" % config)

        # Parse the JSON
        incident_data = json.loads(incident_data)

        # Get key
        query = {}
        query['incident_id'] = incident_data['incident_id']
        logger.debug("Filter: %s" % json.dumps(query))

        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
            json.dumps(query))
        serverResponse, incident = rest.simpleRequest(uri,
                                                      sessionKey=sessionKey)
        logger.debug("Settings for incident: %s" % incident)
        incident = json.loads(incident)

        # Update incident
        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/' + incident[
            0]['_key']
        logger.debug("URI for incident update: %s" % uri)

        # Prepared new entry
        now = datetime.datetime.now().isoformat()
        changed_keys = []
        for key in incident[0].keys():
            if (key in incident_data) and (incident[0][key] !=
                                           incident_data[key]):
                changed_keys.append(key)
                logger.info(
                    "%s for incident %s changed. Writing change event to index %s."
                    % (key, incident[0]['incident_id'], config['index']))
                event_id = hashlib.md5(incident[0]['incident_id'] +
                                       now).hexdigest()
                event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"' % (
                    now, event_id, user, incident[0]['incident_id'], key,
                    incident_data[key], key, incident[0][key])
                logger.debug("Change event will be: %s" % event)
                input.submit(event,
                             hostname=socket.gethostname(),
                             sourcetype='incident_change',
                             source='incident_settings.py',
                             index=config['index'])
                incident[0][key] = incident_data[key]

            else:
                logger.info("%s for incident %s didn't change." %
                            (key, incident[0]['incident_id']))

        del incident[0]['_key']
        contentsStr = json.dumps(incident[0])
        logger.debug("content for update: %s" % contentsStr)
        serverResponse, serverContent = rest.simpleRequest(
            uri, sessionKey=sessionKey, jsonargs=contentsStr)

        logger.debug("Response from update incident entry was %s " %
                     serverResponse)
        logger.debug("Changed keys: %s" % changed_keys)

        if len(changed_keys) > 0:
            ic = IncidentContext(sessionKey, incident_data['incident_id'])
            if "owner" in changed_keys:
                eh.handleEvent(alert=incident[0]["alert"],
                               event="incident_assigned",
                               incident=incident[0],
                               context=ic.getContext())
            elif "status" in changed_keys and incident_data[
                    "status"] == "resolved":
                eh.handleEvent(alert=incident[0]["alert"],
                               event="incident_resolved",
                               incident=incident[0],
                               context=ic.getContext())
            else:
                eh.handleEvent(alert=incident[0]["alert"],
                               event="incident_changed",
                               incident=incident[0],
                               context=ic.getContext())

        if incident_data['comment'] != "":
            incident_data['comment'] = incident_data['comment'].replace(
                '\n', '<br />').replace('\r', '')
            event_id = hashlib.md5(incident[0]['incident_id'] +
                                   now).hexdigest()
            event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"' % (
                now, event_id, user, incident[0]['incident_id'],
                incident_data['comment'])
            logger.debug("Comment event will be: %s" % event)
            event = event.encode('utf8')
            input.submit(event,
                         hostname=socket.gethostname(),
                         sourcetype='incident_change',
                         source='incident_settings.py',
                         index=config['index'])
            ic = IncidentContext(sessionKey, incident_data['incident_id'])
            eh.handleEvent(alert=incident[0]["alert"],
                           event="incident_commented",
                           incident=incident[0],
                           context=ic.getContext())

        return self.response('Successfully updated incident.', httplib.OK)