def setIncidentAutoSubsequentResolved(context, index, sessionKey):
if not context.get('title'):
query = '{ "alert": "' + context.get(
'name'
) + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "' + context.get(
'job_id') + '"} }'
else:
log.debug(
"Using title '%s' to search for incidents to auto subsequent resolve."
% context.get('title'))
query = '{ "title": "' + context.get(
'title'
) + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "' + context.get(
'job_id') + '"} }'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
query)
prev_incidents = getRestData(uri, sessionKey, output_mode='default')
if len(prev_incidents) > 0:
prev_incident = prev_incidents[0]
log.info("Found '%s' as pre-existing incident" %
prev_incident['incident_id'])
# Set status of current incident and fire event
setStatus(context.get('_key'), context.get('incident_id'),
'auto_subsequent_resolved', sessionKey)
event = 'severity=INFO origin="alert_handler" user="******" action="auto_subsequent_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (
context.get('status'), context.get('incident_id'),
context.get('job_id'))
createIncidentChangeEvent(event, context.get('job_id'), index)
ic = IncidentContext(sessionKey, incident_id)
eh.handleEvent(alert=context.get('name'),
event="incident_auto_subsequent_resolved",
incident={"owner": context.get("owner")},
context=ic.getContext())
# Update history of pre-existing incident and fire event
event = 'severity=INFO origin="alert_handler" user="******" action="new_subsequent_incident" incident_id="%s" new_incident_id="%s"' % (
prev_incident['incident_id'], context.get('incident_id'))
createIncidentChangeEvent(context.get('event'), context.get('job_id'),
index)
ic = IncidentContext(sessionKey, prev_incident['incident_id'])
eh.handleEvent(alert=context.get('name'),
event="incident_new_subsequent_incident",
incident=prev_incident,
context=ic.getContext())
return True
else:
log.info(
"No pre-existing incidents with matching criteria for auto_subsequent_resolve found, keep this one open."
)
return False
def autoPreviousResolve(alert, job_id, title):
# Auto Previous resolve
log.info(
"auto_previous_resolve is active for alert %s, searching for incidents to resolve..."
% alert)
if title == "":
query = '{ "alert": "' + alert + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + job_id + '"} }'
else:
log.debug(
"Using title (%s) to search for incidents to auto previous resolve."
% title)
query = '{ "title": "' + title + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + job_id + '"} }'
log.debug("Filter for auto_previous_resolve: %s" % query)
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
query)
serverResponse, serverContent = rest.simpleRequest(uri,
sessionKey=sessionKey)
incidents = json.loads(serverContent)
if len(incidents) > 0:
log.info("Got %s incidents to auto-resolve" % len(incidents))
for incident in incidents:
log.info("Auto-resolving incident with key=%s" % incident['_key'])
previous_status = incident["status"]
previous_job_id = incident["job_id"]
previous_incident_id = incident["incident_id"]
previous_owner = incident["owner"]
incident['status'] = 'auto_previous_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident[
'_key']
incident = json.dumps(incident)
serverResponse, serverContent = rest.simpleRequest(
uri, sessionKey=sessionKey, jsonargs=incident)
now = datetime.datetime.now().isoformat()
event_id = hashlib.md5(job_id + now).hexdigest()
log.debug("event_id=%s now=%s incident=%s" %
(event_id, now, incident))
event = 'time=%s severity=INFO origin="alert_handler" event_id="%s" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (
now, event_id, previous_status, previous_incident_id,
previous_job_id)
log.debug("Resolve event will be: %s" % event)
input.submit(event,
hostname=socket.gethostname(),
sourcetype='incident_change',
source='alert_handler.py',
index=config['index'])
ic = IncidentContext(sessionKey, previous_incident_id)
eh.handleEvent(alert=alert,
event="incident_auto_previous_resolved",
incident={"owner": previous_owner},
context=ic.getContext())
else:
log.info(
"No incidents with matching criteria for auto_previous_resolve found."
)
def setIncidentsAutoResolved(alert, job_id, title, index, sessionKey):
if title == "":
query = (
'{ "alert": "'
+ alert
+ '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'
+ job_id
+ '"} }'
)
else:
log.debug("Using title '%s' to search for incidents to auto previous resolve." % title)
query = (
'{ "title": "'
+ title
+ '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'
+ job_id
+ '"} }'
)
uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s" % urllib.quote(query)
incidents = getRestData(uri, sessionKey, output_mode="default")
if len(incidents) > 0:
log.info("Got %s incidents to auto-resolve" % len(incidents))
for incident in incidents:
log.info("Auto-resolving incident with key=%s" % incident["_key"])
previous_status = incident["status"]
previous_job_id = incident["job_id"]
previous_incident_id = incident["incident_id"]
previous_owner = incident["owner"]
incident["status"] = "auto_previous_resolved"
uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s" % incident["_key"]
getRestData(uri, sessionKey, json.dumps(incident))
event = (
'severity=INFO origin="alert_handler" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"'
% (previous_status, previous_incident_id, previous_job_id)
)
createIncidentChangeEvent(event, job_id, index)
ic = IncidentContext(sessionKey, previous_incident_id)
eh.handleEvent(
alert=alert,
event="incident_auto_previous_resolved",
incident={"owner": previous_owner},
context=ic.getContext(),
)
else:
log.info("No incidents with matching criteria for auto_previous_resolve found.")
def setIncidentsAutoPreviousResolved(context, index, sessionKey):
if not context.get('title'):
query = '{ "alert": "' + context.get(
'name'
) + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + context.get(
'job_id') + '"} }'
else:
log.debug(
"Using title '%s' to search for incidents to auto previous resolve."
% context.get('title'))
query = '{ "title": "' + context.get(
'title'
) + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + context.get(
'job_id') + '"} }'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
query)
incidents = getRestData(uri, sessionKey, output_mode='default')
if len(incidents) > 0:
log.info("Got %s incidents to auto-resolve" % len(incidents))
for incident in incidents:
log.info("Auto-resolving incident with key=%s" % incident['_key'])
previous_status = incident["status"]
previous_job_id = incident["job_id"]
previous_incident_id = incident["incident_id"]
previous_owner = incident["owner"]
incident['status'] = 'auto_previous_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident[
'_key']
getRestData(uri, sessionKey, json.dumps(incident))
event = 'severity=INFO origin="alert_handler" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s" resolving_incident="%s"' % (
previous_status, previous_incident_id, previous_job_id,
context.get('incident_id'))
createIncidentChangeEvent(event, previous_job_id, index)
ic = IncidentContext(sessionKey, previous_incident_id)
eh.handleEvent(alert=context.get('name'),
event="incident_auto_previous_resolved",
incident={"owner": previous_owner},
context=ic.getContext())
else:
log.info(
"No incidents with matching criteria for auto_previous_resolve found."
)
def setIncidentAutoInfoResolved(context, index, sessionKey, statusval):
log.info('Resolving incident %s per settings.' %
context.get('incident_id'))
# set the status of the incident to the configured resolution status
setStatus(context.get('_key'), context.get('incident_id'), statusval,
sessionKey)
# create and index a change event
event = 'severity=INFO origin="alert_handler" user="******" action="auto_informational_resolve" previous_status="%s" status="%s" incident_id="%s" job_id="%s"' % (
context.get('status'), statusval, context.get('incident_id'),
context.get('job_id'))
createIncidentChangeEvent(event, context.get('job_id'), index)
# create a context run the event handler
ic = IncidentContext(sessionKey, incident_id)
eh.handleEvent(alert=context.get('name'),
event="auto_informational_resolve",
incident={"owner": context.get("owner")},
context=ic.getContext())
writeResultToCollection(results)
log.info(
"Alert results for job_id=%s incident_id=%s result_id=%s written to collection incident_results"
% (job_id, incident_id, str(result_id)))
# Write metadata to index
writeAlertMetadataToIndex(job, incident_id, result_id)
# Fire incident_created or incident_suppressed event
ic = IncidentContext(sessionKey, incident_id)
if incident_suppressed == False:
log.info("Firing incident_created event for incident=%s" % incident_id)
eh.handleEvent(alert=alert,
event="incident_created",
incident={"owner": config['default_owner']},
context=ic.getContext())
else:
log.info("Firing incident_suppressed event for incident=%s" % incident_id)
eh.handleEvent(alert=alert,
event="incident_suppressed",
incident={"owner": config['default_owner']},
context=ic.getContext())
# Handle auto-assign
if incident_config['auto_assign'] and incident_config[
'auto_assign_owner'] != 'unassigned' and incident_suppressed == False:
log.info("auto_assign is active for %s. Starting to handle it." % alert)
assignIncident(incident_key, incident_id,
incident_config['auto_assign_owner'])
logAutoAssignEvent(incident_id, job_id, result_id,
incident_config['auto_assign_owner'])
def save(self, contents, **kwargs):
logger.info("Saving incident settings contents...")
user = cherrypy.session["user"]["name"]
sessionKey = cherrypy.session.get("sessionKey")
splunk.setDefault("sessionKey", sessionKey)
eh = EventHandler(sessionKey=sessionKey)
config = {}
config["index"] = "alerts"
restconfig = entity.getEntities("configs/alert_manager", count=-1, sessionKey=sessionKey)
if len(restconfig) > 0:
if "index" in restconfig["settings"]:
config["index"] = restconfig["settings"]["index"]
logger.debug("Global settings: %s" % config)
# Parse the JSON
contents = json.loads(contents)
logger.debug("Contents: %s" % json.dumps(contents))
# Get key
query = {}
query["incident_id"] = contents["incident_id"]
logger.debug("Filter: %s" % json.dumps(query))
uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s" % urllib.quote(
json.dumps(query)
)
serverResponse, incident = rest.simpleRequest(uri, sessionKey=sessionKey)
logger.debug("Settings for incident: %s" % incident)
incident = json.loads(incident)
# Update incident
uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incidents/" + incident[0]["_key"]
logger.debug("URI for incident update: %s" % uri)
# Prepared new entry
now = datetime.datetime.now().isoformat()
changed_keys = []
for key in incident[0].keys():
if (key in contents) and (incident[0][key] != contents[key]):
changed_keys.append(key)
logger.info(
"%s for incident %s changed. Writing change event to index %s."
% (key, incident[0]["incident_id"], config["index"])
)
event_id = hashlib.md5(incident[0]["incident_id"] + now).hexdigest()
event = (
'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"'
% (now, event_id, user, incident[0]["incident_id"], key, contents[key], key, incident[0][key])
)
logger.debug("Change event will be: %s" % event)
input.submit(
event,
hostname=socket.gethostname(),
sourcetype="incident_change",
source="incident_settings.py",
index=config["index"],
)
incident[0][key] = contents[key]
else:
logger.info("%s for incident %s didn't change." % (key, incident[0]["incident_id"]))
del incident[0]["_key"]
contentsStr = json.dumps(incident[0])
logger.debug("content for update: %s" % contentsStr)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=contentsStr)
logger.debug("Response from update incident entry was %s " % serverResponse)
logger.debug("Changed keys: %s" % changed_keys)
if len(changed_keys) > 0:
ic = IncidentContext(sessionKey, contents["incident_id"])
if "owner" in changed_keys:
eh.handleEvent(
alert=incident[0]["alert"], event="incident_assigned", incident=incident[0], context=ic.getContext()
)
elif "status" in changed_keys and contents["status"] == "resolved":
eh.handleEvent(
alert=incident[0]["alert"], event="incident_resolved", incident=incident[0], context=ic.getContext()
)
else:
eh.handleEvent(
alert=incident[0]["alert"], event="incident_changed", incident=incident[0], context=ic.getContext()
)
if contents["comment"] != "":
event_id = hashlib.md5(incident[0]["incident_id"] + now).hexdigest()
event = (
'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"'
% (now, event_id, user, incident[0]["incident_id"], contents["comment"])
)
logger.debug("Comment event will be: %s" % event)
event = event.encode("utf8")
input.submit(
event,
hostname=socket.gethostname(),
sourcetype="incident_change",
source="incident_settings.py",
index=config["index"],
)
return "Done"
def save(self, contents, **kwargs):
logger.info("Saving incident settings contents...")
user = cherrypy.session['user']['name']
sessionKey = cherrypy.session.get('sessionKey')
splunk.setDefault('sessionKey', sessionKey)
eh = EventHandler(sessionKey = sessionKey)
config = {}
config['index'] = 'alerts'
restconfig = entity.getEntities('configs/alert_manager', count=-1, sessionKey=sessionKey)
if len(restconfig) > 0:
if 'index' in restconfig['settings']:
config['index'] = restconfig['settings']['index']
logger.debug("Global settings: %s" % config)
# Parse the JSON
contents = json.loads(contents)
logger.debug("Contents: %s" % json.dumps(contents))
# Get key
query = {}
query['incident_id'] = contents['incident_id']
logger.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(json.dumps(query))
serverResponse, incident = rest.simpleRequest(uri, sessionKey=sessionKey)
logger.debug("Settings for incident: %s" % incident)
incident = json.loads(incident)
# Update incident
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/' + incident[0]['_key']
logger.debug("URI for incident update: %s" % uri )
# Prepared new entry
now = datetime.datetime.now().isoformat()
changed_keys = []
for key in incident[0].keys():
if (key in contents) and (incident[0][key] != contents[key]):
changed_keys.append(key)
logger.info("%s for incident %s changed. Writing change event to index %s." % (key, incident[0]['incident_id'], config['index']))
event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"' % (now, event_id, user, incident[0]['incident_id'], key, contents[key], key, incident[0][key])
logger.debug("Change event will be: %s" % event)
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
incident[0][key] = contents[key]
else:
logger.info("%s for incident %s didn't change." % (key, incident[0]['incident_id']))
del incident[0]['_key']
contentsStr = json.dumps(incident[0])
logger.debug("content for update: %s" % contentsStr)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=contentsStr)
logger.debug("Response from update incident entry was %s " % serverResponse)
logger.debug("Changed keys: %s" % changed_keys)
if len(changed_keys) > 0:
ic = IncidentContext(sessionKey, contents['incident_id'])
if "owner" in changed_keys:
eh.handleEvent(alert=incident[0]["alert"], event="incident_assigned", incident=incident[0], context=ic.getContext())
elif "status" in changed_keys and contents["status"] == "resolved":
eh.handleEvent(alert=incident[0]["alert"], event="incident_resolved", incident=incident[0], context=ic.getContext())
else:
eh.handleEvent(alert=incident[0]["alert"], event="incident_changed", incident=incident[0], context=ic.getContext())
if contents['comment'] != "":
event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"' % (now, event_id, user, incident[0]['incident_id'], contents['comment'])
logger.debug("Comment event will be: %s" % event)
event = event.encode('utf8')
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
return 'Done'
if incident_suppressed:
logSuppressEvent(alert, incident_id, job_id, result_id, rule_names)
# Write results to collection
writeResultToCollection(results)
log.info("Alert results for job_id=%s incident_id=%s result_id=%s written to collection incident_results" % (job_id, incident_id, str(result_id)))
# Write metadata to index
writeAlertMetadataToIndex(job, incident_id, result_id)
# Fire incident_created or incident_suppressed event
ic = IncidentContext(sessionKey, incident_id)
if incident_suppressed == False:
log.info("Firing incident_created event for incident=%s" % incident_id)
eh.handleEvent(alert=alert, event="incident_created", incident={"owner": config['default_owner']}, context=ic.getContext())
else:
log.info("Firing incident_suppressed event for incident=%s" % incident_id)
eh.handleEvent(alert=alert, event="incident_suppressed", incident={"owner": config['default_owner']}, context=ic.getContext())
# Handle auto-assign
if incident_config['auto_assign'] and incident_config['auto_assign_owner'] != 'unassigned' and incident_suppressed == False:
log.info("auto_assign is active for %s. Starting to handle it." % alert)
assignIncident(incident_key, incident_id, incident_config['auto_assign_owner'])
logAutoAssignEvent(incident_id, job_id, result_id, incident_config['auto_assign_owner'])
eh.handleEvent(alert=alert, event="incident_auto_assigned", incident={"owner": incident_config["auto_assign_owner"]}, context=ic.getContext())
# Auto Previous Resolve - run only once
if incident_config['auto_previous_resolve'] and incident_suppressed == False:
log.info("auto_previous_resolve is active for %s. Starting to handle it." % alert)
autoPreviousResolve(alert, job_id, job['title'])
def setIncidentAutoSubsequentResolved(context, index, sessionKey):
if not context.get('title'):
query = '{ "alert": "'+ context.get('name') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
else:
log.debug("Using title '%s' to search for incidents to auto subsequent resolve." % context.get('title'))
query = '{ "title": "'+ context.get('title') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(query)
prev_incidents = getRestData(uri, sessionKey, output_mode = 'default')
if len(prev_incidents) > 0:
prev_incident = prev_incidents[0]
log.info("Found '%s' as pre-existing incident" % prev_incident['incident_id'])
# Set status of current incident and fire event
setStatus(context.get('_key'), context.get('incident_id'), 'auto_subsequent_resolved', sessionKey)
event = 'severity=INFO origin="alert_handler" user="******" action="auto_subsequent_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (context.get('status'), context.get('incident_id'), context.get('job_id'))
createIncidentChangeEvent(event, context.get('job_id'), index)
ic = IncidentContext(sessionKey, incident_id)
eh.handleEvent(alert=context.get('name'), event="incident_auto_subsequent_resolved", incident={"owner": context.get("owner")}, context=ic.getContext())
# Update history of pre-existing incident and fire event
event = 'severity=INFO origin="alert_handler" user="******" action="new_subsequent_incident" incident_id="%s" new_incident_id="%s"' % (prev_incident['incident_id'], context.get('incident_id'))
createIncidentChangeEvent(context.get('event'), context.get('job_id'), index)
ic = IncidentContext(sessionKey, prev_incident['incident_id'])
eh.handleEvent(alert=context.get('name'), event="incident_new_subsequent_incident", incident=prev_incident, context=ic.getContext())
return True
else:
log.info("No pre-existing incidents with matching criteria for auto_subsequent_resolve found, keep this one open.")
return False
log.info("Incident %s (%s) should be resolved. alert_time=%s ttl=%s now=%s" % (incident['incident_id'], incident['_key'], incident['alert_time'], incident['ttl'], time.time()))
old_status = incident['status']
incident['status'] = 'auto_ttl_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
incidentStr = json.dumps(incident)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=incidentStr)
now = datetime.datetime.now().isoformat()
event_id = hashlib.md5(incident['incident_id'] + now).hexdigest()
log.debug("event_id=%s now=%s" % (event_id, now))
event = 'time=%s severity=INFO origin="alert_manager_scheduler" event_id="%s" user="******" action="auto_ttl_resolve" previous_status="%s" status="auto_ttl_resolved" incident_id="%s"' % (now, event_id, old_status, incident['incident_id'])
log.debug("Event will be: %s" % event)
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'alert_manager_scheduler.py', index = config['index'])
ic = IncidentContext(sessionKey, incident["incident_id"])
eh.handleEvent(alert=alert["name"], event="incident_auto_ttl_resolved", incident={"owner": incident["owner"]}, context=ic.getContext())
else:
log.info("Incident %s has not ttl reached yet." % incident['incident_id'])
else:
log.info("No incidents of alert %s to check for reached ttl." % alert['name'])
log.debug('Alert "%s" is not configured for auto_ttl_resolve, skipping...' % alert['name'])
#
# Look for auto_suppress_resolve incidents
#
query = {}
query['auto_suppress_resolve'] = True
log.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_settings?query=%s' % urllib.quote(json.dumps(query))
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
try:
def _update_incident(self, sessionKey, user, post_data):
logger.debug("START _update_incident()")
required = ['incident_data']
missing = [r for r in required if r not in post_data]
if missing:
return self.response("Missing required arguments: %s" % missing, httplib.BAD_REQUEST)
incident_data = post_data.pop('incident_data')
splunk.setDefault('sessionKey', sessionKey)
eh = EventHandler(sessionKey = sessionKey)
config = {}
config['index'] = 'main'
restconfig = entity.getEntities('configs/alert_manager', count=-1, sessionKey=sessionKey)
if len(restconfig) > 0:
if 'index' in restconfig['settings']:
config['index'] = restconfig['settings']['index']
logger.debug("Global settings: %s" % config)
# Parse the JSON
incident_data = json.loads(incident_data)
# Get key
query = {}
query['incident_id'] = incident_data['incident_id']
logger.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(json.dumps(query))
serverResponse, incident = rest.simpleRequest(uri, sessionKey=sessionKey)
logger.debug("Settings for incident: %s" % incident)
incident = json.loads(incident)
# Update incident
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/' + incident[0]['_key']
logger.debug("URI for incident update: %s" % uri )
# Prepared new entry
now = datetime.datetime.now().isoformat()
changed_keys = []
for key in incident[0].keys():
if (key in incident_data) and (incident[0][key] != incident_data[key]):
changed_keys.append(key)
logger.info("%s for incident %s changed. Writing change event to index %s." % (key, incident[0]['incident_id'], config['index']))
event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"' % (now, event_id, user, incident[0]['incident_id'], key, incident_data[key], key, incident[0][key])
logger.debug("Change event will be: %s" % event)
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
incident[0][key] = incident_data[key]
else:
logger.info("%s for incident %s didn't change." % (key, incident[0]['incident_id']))
del incident[0]['_key']
contentsStr = json.dumps(incident[0])
logger.debug("content for update: %s" % contentsStr)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=contentsStr)
logger.debug("Response from update incident entry was %s " % serverResponse)
logger.debug("Changed keys: %s" % changed_keys)
if len(changed_keys) > 0:
ic = IncidentContext(sessionKey, incident_data['incident_id'])
if "owner" in changed_keys:
eh.handleEvent(alert=incident[0]["alert"], event="incident_assigned", incident=incident[0], context=ic.getContext())
elif "status" in changed_keys and incident_data["status"] == "resolved":
eh.handleEvent(alert=incident[0]["alert"], event="incident_resolved", incident=incident[0], context=ic.getContext())
else:
eh.handleEvent(alert=incident[0]["alert"], event="incident_changed", incident=incident[0], context=ic.getContext())
if incident_data['comment'] != "":
incident_data['comment'] = incident_data['comment'].replace('\n', '<br />').replace('\r', '')
event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"' % (now, event_id, user, incident[0]['incident_id'], incident_data['comment'])
logger.debug("Comment event will be: %s" % event)
event = event.encode('utf8')
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
ic = IncidentContext(sessionKey, incident_data['incident_id'])
eh.handleEvent(alert=incident[0]["alert"], event="incident_commented", incident=incident[0], context=ic.getContext())
return self.response('Successfully updated incident.', httplib.OK)
event = 'severity=INFO origin="alert_handler" user="******" action="suppress" alert="%s" incident_id="%s" job_id="%s" result_id="%s" %s' % ('splunk-system-user', search_name, incident_id, job_id, result_id, rules)
createIncidentChangeEvent(event, metadata['job_id'], settings.get('index'))
# Write results to collection
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_results'
response = getRestData(uri, sessionKey, json.dumps(results))
log.info("Results for incident_id=%s written to collection." % (incident_id))
# Write metadata to index
createMetadataEvent(metadata, settings.get('index'), sessionKey)
# Fire incident_created or incident_suppressed event
ic = IncidentContext(sessionKey, incident_id)
if incident_suppressed == False:
log.info("Firing incident_created event for incident=%s" % incident_id)
eh.handleEvent(alert=search_name, event="incident_created", incident={"owner": settings.get('default_owner')}, context=ic.getContext())
else:
log.info("Firing incident_suppressed event for incident=%s" % incident_id)
eh.handleEvent(alert=search_name, event="incident_suppressed", incident={"owner": settings.get('default_owner')}, context=ic.getContext())
# Handle auto-assign
if config['auto_assign_owner'] != '' and config['auto_assign_owner'] != 'unassigned' and incident_suppressed == False:
log.debug("auto_assign is active for %s. Starting to handle it." % search_name)
setOwner(incident_key, incident_id, config['auto_assign_owner'], sessionKey)
setStatus(incident_key, incident_id, 'auto_assigned', sessionKey)
ic.update("owner", config['auto_assign_owner'])
event = 'severity=INFO origin="alert_handler" user="******" action="change" incident_id="%s" job_id="%s" result_id="%s" owner="%s" previous_owner="unassigned"' % (incident_id, job_id, result_id, config['auto_assign_owner'])
createIncidentChangeEvent(event, metadata['job_id'], settings.get('index'))
event = 'severity=INFO origin="alert_handler" user="******" action="change" incident_id="%s" job_id="%s" result_id="%s" status="auto_assigned" previous_status="new"' % (incident_id, job_id, result_id)
def autoPreviousResolve(alert, job_id, title):
# Auto Previous resolve
log.info("auto_previous_resolve is active for alert %s, searching for incidents to resolve..." % alert)
if title == "":
query = '{ "alert": "'+ alert +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ job_id +'"} }'
else:
log.debug("Using title (%s) to search for incidents to auto previous resolve." % title)
query = '{ "title": "'+ title +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ job_id +'"} }'
log.debug("Filter for auto_previous_resolve: %s" % query)
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(query)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
incidents = json.loads(serverContent)
if len(incidents) > 0:
log.info("Got %s incidents to auto-resolve" % len(incidents))
for incident in incidents:
log.info("Auto-resolving incident with key=%s" % incident['_key'])
previous_status = incident["status"]
previous_job_id = incident["job_id"]
previous_incident_id = incident["incident_id"]
previous_owner = incident["owner"]
incident['status'] = 'auto_previous_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
incident = json.dumps(incident)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=incident)
now = datetime.datetime.now().isoformat()
event_id = hashlib.md5(job_id + now).hexdigest()
log.debug("event_id=%s now=%s incident=%s" % (event_id, now, incident))
event = 'time=%s severity=INFO origin="alert_handler" event_id="%s" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (now, event_id, previous_status, previous_incident_id, previous_job_id)
log.debug("Resolve event will be: %s" % event)
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'alert_handler.py', index = config['index'])
ic = IncidentContext(sessionKey, previous_incident_id)
eh.handleEvent(alert=alert, event="incident_auto_previous_resolved", incident={"owner": previous_owner}, context=ic.getContext())
else:
log.info("No incidents with matching criteria for auto_previous_resolve found.")
setIncidentsAutoPreviousResolved(ic, settings.get('index'), sessionKey)
elif config['auto_subsequent_resolve'] and incident_suppressed == False:
log.debug("auto_subsequent_resolve is active for %s. Starting to handle it." % search_name)
is_subsequent_resolved = setIncidentAutoSubsequentResolved(ic, settings.get('index'), sessionKey)
# Fire incident_created or incident_suppressed event
# only if it was not deemed a duplicate
if is_subsequent_resolved:
log.info("Skipping firing of incident_created event for incident=%s because it is a duplicate." % incident_id)
else:
if incident_suppressed == False:
log.info("Firing incident_created event for incident=%s" % incident_id)
eh.handleEvent(alert=search_name, event="incident_created", incident={"owner": settings.get('default_owner')}, context=ic.getContext())
else:
log.info("Firing incident_suppressed event for incident=%s" % incident_id)
eh.handleEvent(alert=search_name, event="incident_suppressed", incident={"owner": settings.get('default_owner')}, context=ic.getContext())
# If the incident was not resolved already, auto resolved is enabled, and priority is informational - resolve it.
auto_info_resolved = False
if is_subsequent_resolved == False:
# This automatic resolution is optional.
try:
if normalize_bool(settings.get('auto_close_info')) and metadata['priority'] == 'informational':
log.debug('Auto close informational is on')
setIncidentAutoInfoResolved(ic, settings.get('index'), sessionKey, settings.get('auto_close_info_status'))
auto_info_resolved = True
except:
log.info("Incident %s (%s) should be resolved. alert_time=%s ttl=%s now=%s" % (incident['incident_id'], incident['_key'], incident['alert_time'], incident['ttl'], time.time()))
old_status = incident['status']
incident['status'] = 'auto_ttl_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
incidentStr = json.dumps(incident)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=incidentStr)
now = datetime.datetime.now().isoformat()
event_id = hashlib.md5(incident['incident_id'] + now).hexdigest()
log.debug("event_id=%s now=%s" % (event_id, now))
event = 'time=%s severity=INFO origin="alert_manager_scheduler" event_id="%s" user="******" action="auto_ttl_resolve" previous_status="%s" status="auto_ttl_resolved" incident_id="%s"' % (now, event_id, old_status, incident['incident_id'])
log.debug("Event will be: %s" % event)
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'alert_manager_scheduler.py', index = config['index'])
ic = IncidentContext(sessionKey, incident["incident_id"])
eh.handleEvent(alert=alert["name"], event="incident_auto_ttl_resolved", incident={"owner": incident["owner"]}, context=ic.getContext())
else:
log.info("Incident %s has not ttl reached yet." % incident['incident_id'])
else:
log.info("No incidents of alert %s to check for reached ttl." % alert['name'])
log.debug('Alert "%s" is not configured for auto_ttl_resolve, skipping...' % alert['name'])
# Look for auto_suppress_resolve incidents
query = {}
query['auto_suppress_resolve'] = True
log.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_settings?query=%s' % urllib.quote(json.dumps(query))
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
alerts = json.loads(serverContent)
if len(alerts) >0:
for alert in alerts:
def setIncidentsAutoPreviousResolved(context, index, sessionKey):
if not context.get('title'):
query = '{ "alert": "'+ context.get('name') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
else:
log.debug("Using title '%s' to search for incidents to auto previous resolve." % context.get('title'))
query = '{ "title": "'+ context.get('title') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(query)
incidents = getRestData(uri, sessionKey, output_mode = 'default')
if len(incidents) > 0:
log.info("Got %s incidents to auto-resolve" % len(incidents))
for incident in incidents:
log.info("Auto-resolving incident with key=%s" % incident['_key'])
previous_status = incident["status"]
previous_job_id = incident["job_id"]
previous_incident_id = incident["incident_id"]
previous_owner = incident["owner"]
incident['status'] = 'auto_previous_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
getRestData(uri, sessionKey, json.dumps(incident))
event = 'severity=INFO origin="alert_handler" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s" resolving_incident="%s"' % (previous_status, previous_incident_id, previous_job_id, context.get('incident_id'))
createIncidentChangeEvent(event, previous_job_id, index)
ic = IncidentContext(sessionKey, previous_incident_id)
eh.handleEvent(alert=context.get('name'), event="incident_auto_previous_resolved", incident={"owner": previous_owner}, context=ic.getContext())
else:
log.info("No incidents with matching criteria for auto_previous_resolve found.")
uri = "/servicesNS/nobody/alert_manager/storage/collections/data/incident_results"
response = getRestData(uri, sessionKey, json.dumps(results))
log.info("Results for incident_id=%s written to collection." % (incident_id))
# Write metadata to index
createMetadataEvent(metadata, settings.get("index"), sessionKey)
# Fire incident_created or incident_suppressed event
ic = IncidentContext(sessionKey, incident_id)
if incident_suppressed == False:
log.info("Firing incident_created event for incident=%s" % incident_id)
eh.handleEvent(
alert=search_name,
event="incident_created",
incident={"owner": settings.get("default_owner")},
context=ic.getContext(),
)
else:
log.info("Firing incident_suppressed event for incident=%s" % incident_id)
eh.handleEvent(
alert=search_name,
event="incident_suppressed",
incident={"owner": settings.get("default_owner")},
context=ic.getContext(),
)
# Handle auto-assign
if (
config["auto_assign_owner"] != ""
and config["auto_assign_owner"] != "unassigned"
and incident_suppressed == False
def setIncidentAutoInfoResolved(context, index, sessionKey, statusval):
log.info('Resolving incident %s per settings.' % context.get('incident_id'))
# set the status of the incident to the configured resolution status
setStatus(context.get('_key'), context.get('incident_id'), statusval, sessionKey)
# create and index a change event
event = 'severity=INFO origin="alert_handler" user="******" action="auto_informational_resolve" previous_status="%s" status="%s" incident_id="%s" job_id="%s"' % (context.get('status'), statusval, context.get('incident_id'), context.get('job_id'))
createIncidentChangeEvent(event, context.get('job_id'), index)
# create a context run the event handler
ic = IncidentContext(sessionKey, incident_id)
eh.handleEvent(alert=context.get('name'), event="auto_informational_resolve", incident={"owner": context.get("owner")}, context=ic.getContext())
def _update_incident(self, sessionKey, user, post_data):
logger.debug("START _update_incident()")
required = ['incident_data']
missing = [r for r in required if r not in post_data]
if missing:
return self.response("Missing required arguments: %s" % missing,
httplib.BAD_REQUEST)
incident_data = post_data.pop('incident_data')
splunk.setDefault('sessionKey', sessionKey)
eh = EventHandler(sessionKey=sessionKey)
config = {}
config['index'] = 'main'
restconfig = entity.getEntities('configs/alert_manager',
count=-1,
sessionKey=sessionKey)
if len(restconfig) > 0:
if 'index' in restconfig['settings']:
config['index'] = restconfig['settings']['index']
logger.debug("Global settings: %s" % config)
# Parse the JSON
incident_data = json.loads(incident_data)
# Get key
query = {}
query['incident_id'] = incident_data['incident_id']
logger.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
json.dumps(query))
serverResponse, incident = rest.simpleRequest(uri,
sessionKey=sessionKey)
logger.debug("Settings for incident: %s" % incident)
incident = json.loads(incident)
# Update incident
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/' + incident[
0]['_key']
logger.debug("URI for incident update: %s" % uri)
# Prepared new entry
now = datetime.datetime.now().isoformat()
changed_keys = []
for key in incident[0].keys():
if (key in incident_data) and (incident[0][key] !=
incident_data[key]):
changed_keys.append(key)
logger.info(
"%s for incident %s changed. Writing change event to index %s."
% (key, incident[0]['incident_id'], config['index']))
event_id = hashlib.md5(incident[0]['incident_id'] +
now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"' % (
now, event_id, user, incident[0]['incident_id'], key,
incident_data[key], key, incident[0][key])
logger.debug("Change event will be: %s" % event)
input.submit(event,
hostname=socket.gethostname(),
sourcetype='incident_change',
source='incident_settings.py',
index=config['index'])
incident[0][key] = incident_data[key]
else:
logger.info("%s for incident %s didn't change." %
(key, incident[0]['incident_id']))
del incident[0]['_key']
contentsStr = json.dumps(incident[0])
logger.debug("content for update: %s" % contentsStr)
serverResponse, serverContent = rest.simpleRequest(
uri, sessionKey=sessionKey, jsonargs=contentsStr)
logger.debug("Response from update incident entry was %s " %
serverResponse)
logger.debug("Changed keys: %s" % changed_keys)
if len(changed_keys) > 0:
ic = IncidentContext(sessionKey, incident_data['incident_id'])
if "owner" in changed_keys:
eh.handleEvent(alert=incident[0]["alert"],
event="incident_assigned",
incident=incident[0],
context=ic.getContext())
elif "status" in changed_keys and incident_data[
"status"] == "resolved":
eh.handleEvent(alert=incident[0]["alert"],
event="incident_resolved",
incident=incident[0],
context=ic.getContext())
else:
eh.handleEvent(alert=incident[0]["alert"],
event="incident_changed",
incident=incident[0],
context=ic.getContext())
if incident_data['comment'] != "":
incident_data['comment'] = incident_data['comment'].replace(
'\n', '<br />').replace('\r', '')
event_id = hashlib.md5(incident[0]['incident_id'] +
now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"' % (
now, event_id, user, incident[0]['incident_id'],
incident_data['comment'])
logger.debug("Comment event will be: %s" % event)
event = event.encode('utf8')
input.submit(event,
hostname=socket.gethostname(),
sourcetype='incident_change',
source='incident_settings.py',
index=config['index'])
ic = IncidentContext(sessionKey, incident_data['incident_id'])
eh.handleEvent(alert=incident[0]["alert"],
event="incident_commented",
incident=incident[0],
context=ic.getContext())
return self.response('Successfully updated incident.', httplib.OK)