def setIncidentAutoSubsequentResolved(context, index, sessionKey):
if not context.get('title'):
query = '{ "alert": "'+ context.get('name') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
else:
log.debug("Using title '%s' to search for incidents to auto subsequent resolve." % context.get('title'))
query = '{ "title": "'+ context.get('title') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" }, { "status": "assigned" }, { "status": "work_in_progress" }, { "status": "on_hold" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(query)
prev_incidents = getRestData(uri, sessionKey, output_mode = 'default')
if len(prev_incidents) > 0:
prev_incident = prev_incidents[0]
log.info("Found '%s' as pre-existing incident" % prev_incident['incident_id'])
# Set status of current incident and fire event
setStatus(context.get('_key'), context.get('incident_id'), 'auto_subsequent_resolved', sessionKey)
event = 'severity=INFO origin="alert_handler" user="******" action="auto_subsequent_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (context.get('status'), context.get('incident_id'), context.get('job_id'))
createIncidentChangeEvent(event, context.get('job_id'), index)
ic = IncidentContext(sessionKey, incident_id)
eh.handleEvent(alert=context.get('name'), event="incident_auto_subsequent_resolved", incident={"owner": context.get("owner")}, context=ic.getContext())
# Update history of pre-existing incident and fire event
event = 'severity=INFO origin="alert_handler" user="******" action="new_subsequent_incident" incident_id="%s" new_incident_id="%s"' % (prev_incident['incident_id'], context.get('incident_id'))
createIncidentChangeEvent(context.get('event'), context.get('job_id'), index)
ic = IncidentContext(sessionKey, prev_incident['incident_id'])
eh.handleEvent(alert=context.get('name'), event="incident_new_subsequent_incident", incident=prev_incident, context=ic.getContext())
else:
log.info("No pre-existing incidents with matching criteria for auto_subsequent_resolve found, keep this one open.")
def autoPreviousResolve(alert, job_id, title):
# Auto Previous resolve
log.info(
"auto_previous_resolve is active for alert %s, searching for incidents to resolve..."
% alert)
if title == "":
query = '{ "alert": "' + alert + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + job_id + '"} }'
else:
log.debug(
"Using title (%s) to search for incidents to auto previous resolve."
% title)
query = '{ "title": "' + title + '", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "' + job_id + '"} }'
log.debug("Filter for auto_previous_resolve: %s" % query)
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
query)
serverResponse, serverContent = rest.simpleRequest(uri,
sessionKey=sessionKey)
incidents = json.loads(serverContent)
if len(incidents) > 0:
log.info("Got %s incidents to auto-resolve" % len(incidents))
for incident in incidents:
log.info("Auto-resolving incident with key=%s" % incident['_key'])
previous_status = incident["status"]
previous_job_id = incident["job_id"]
previous_incident_id = incident["incident_id"]
previous_owner = incident["owner"]
incident['status'] = 'auto_previous_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident[
'_key']
incident = json.dumps(incident)
serverResponse, serverContent = rest.simpleRequest(
uri, sessionKey=sessionKey, jsonargs=incident)
now = datetime.datetime.now().isoformat()
event_id = hashlib.md5(job_id + now).hexdigest()
log.debug("event_id=%s now=%s incident=%s" %
(event_id, now, incident))
event = 'time=%s severity=INFO origin="alert_handler" event_id="%s" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (
now, event_id, previous_status, previous_incident_id,
previous_job_id)
log.debug("Resolve event will be: %s" % event)
input.submit(event,
hostname=socket.gethostname(),
sourcetype='incident_change',
source='alert_handler.py',
index=config['index'])
ic = IncidentContext(sessionKey, previous_incident_id)
eh.handleEvent(alert=alert,
event="incident_auto_previous_resolved",
incident={"owner": previous_owner},
context=ic.getContext())
else:
log.info(
"No incidents with matching criteria for auto_previous_resolve found."
)
def setIncidentsAutoPreviousResolved(context, index, sessionKey):
if not context.get('title'):
query = '{ "alert": "'+ context.get('name') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
else:
log.debug("Using title '%s' to search for incidents to auto previous resolve." % context.get('title'))
query = '{ "title": "'+ context.get('title') +'", "$or": [ { "status": "auto_assigned" } , { "status": "new" } ], "job_id": { "$ne": "'+ context.get('job_id') +'"} }'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(query)
incidents = getRestData(uri, sessionKey, output_mode = 'default')
if len(incidents) > 0:
log.info("Got %s incidents to auto-resolve" % len(incidents))
for incident in incidents:
log.info("Auto-resolving incident with key=%s" % incident['_key'])
previous_status = incident["status"]
previous_job_id = incident["job_id"]
previous_incident_id = incident["incident_id"]
previous_owner = incident["owner"]
incident['status'] = 'auto_previous_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
getRestData(uri, sessionKey, json.dumps(incident))
event = 'severity=INFO origin="alert_handler" user="******" action="auto_previous_resolve" previous_status="%s" status="auto_previous_resolved" incident_id="%s" job_id="%s"' % (previous_status, previous_incident_id, previous_job_id)
createIncidentChangeEvent(event, previous_job_id, index)
ic = IncidentContext(sessionKey, previous_incident_id)
eh.handleEvent(alert=context.get('name'), event="incident_auto_previous_resolved", incident={"owner": previous_owner}, context=ic.getContext())
else:
log.info("No incidents with matching criteria for auto_previous_resolve found.")
def setIncidentAutoInfoResolved(context, index, sessionKey, statusval):
log.info('Resolving incident %s per settings.' %
context.get('incident_id'))
# set the status of the incident to the configured resolution status
setStatus(context.get('_key'), context.get('incident_id'), statusval,
sessionKey)
# create and index a change event
event = 'severity=INFO origin="alert_handler" user="******" action="auto_informational_resolve" previous_status="%s" status="%s" incident_id="%s" job_id="%s"' % (
context.get('status'), statusval, context.get('incident_id'),
context.get('job_id'))
createIncidentChangeEvent(event, context.get('job_id'), index)
# create a context run the event handler
ic = IncidentContext(sessionKey, incident_id)
eh.handleEvent(alert=context.get('name'),
event="auto_informational_resolve",
incident={"owner": context.get("owner")},
context=ic.getContext())
% (job_id, incident_id, incident_key))
if incident_suppressed:
logSuppressEvent(alert, incident_id, job_id, result_id, rule_names)
# Write results to collection
writeResultToCollection(results)
log.info(
"Alert results for job_id=%s incident_id=%s result_id=%s written to collection incident_results"
% (job_id, incident_id, str(result_id)))
# Write metadata to index
writeAlertMetadataToIndex(job, incident_id, result_id)
# Fire incident_created or incident_suppressed event
ic = IncidentContext(sessionKey, incident_id)
if incident_suppressed == False:
log.info("Firing incident_created event for incident=%s" % incident_id)
eh.handleEvent(alert=alert,
event="incident_created",
incident={"owner": config['default_owner']},
context=ic.getContext())
else:
log.info("Firing incident_suppressed event for incident=%s" % incident_id)
eh.handleEvent(alert=alert,
event="incident_suppressed",
incident={"owner": config['default_owner']},
context=ic.getContext())
# Handle auto-assign
if incident_config['auto_assign'] and incident_config[
def save(self, contents, **kwargs):
logger.info("Saving incident settings contents...")
user = cherrypy.session['user']['name']
sessionKey = cherrypy.session.get('sessionKey')
splunk.setDefault('sessionKey', sessionKey)
eh = EventHandler(sessionKey = sessionKey)
config = {}
config['index'] = 'alerts'
restconfig = entity.getEntities('configs/alert_manager', count=-1, sessionKey=sessionKey)
if len(restconfig) > 0:
if 'index' in restconfig['settings']:
config['index'] = restconfig['settings']['index']
logger.debug("Global settings: %s" % config)
# Parse the JSON
contents = json.loads(contents)
logger.debug("Contents: %s" % json.dumps(contents))
# Get key
query = {}
query['incident_id'] = contents['incident_id']
logger.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(json.dumps(query))
serverResponse, incident = rest.simpleRequest(uri, sessionKey=sessionKey)
logger.debug("Settings for incident: %s" % incident)
incident = json.loads(incident)
# Update incident
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/' + incident[0]['_key']
logger.debug("URI for incident update: %s" % uri )
# Prepared new entry
now = datetime.datetime.now().isoformat()
changed_keys = []
for key in incident[0].keys():
if (key in contents) and (incident[0][key] != contents[key]):
changed_keys.append(key)
logger.info("%s for incident %s changed. Writing change event to index %s." % (key, incident[0]['incident_id'], config['index']))
event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"' % (now, event_id, user, incident[0]['incident_id'], key, contents[key], key, incident[0][key])
logger.debug("Change event will be: %s" % event)
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
incident[0][key] = contents[key]
else:
logger.info("%s for incident %s didn't change." % (key, incident[0]['incident_id']))
del incident[0]['_key']
contentsStr = json.dumps(incident[0])
logger.debug("content for update: %s" % contentsStr)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=contentsStr)
logger.debug("Response from update incident entry was %s " % serverResponse)
logger.debug("Changed keys: %s" % changed_keys)
if len(changed_keys) > 0:
ic = IncidentContext(sessionKey, contents['incident_id'])
if "owner" in changed_keys:
eh.handleEvent(alert=incident[0]["alert"], event="incident_assigned", incident=incident[0], context=ic.getContext())
elif "status" in changed_keys and contents["status"] == "resolved":
eh.handleEvent(alert=incident[0]["alert"], event="incident_resolved", incident=incident[0], context=ic.getContext())
else:
eh.handleEvent(alert=incident[0]["alert"], event="incident_changed", incident=incident[0], context=ic.getContext())
if contents['comment'] != "":
event_id = hashlib.md5(incident[0]['incident_id'] + now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"' % (now, event_id, user, incident[0]['incident_id'], contents['comment'])
logger.debug("Comment event will be: %s" % event)
event = event.encode('utf8')
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'incident_settings.py', index = config['index'])
return 'Done'
if (incident['alert_time'] + incident['ttl']) <= time.time():
log.info("Incident %s (%s) should be resolved. alert_time=%s ttl=%s now=%s" % (incident['incident_id'], incident['_key'], incident['alert_time'], incident['ttl'], time.time()))
old_status = incident['status']
incident['status'] = 'auto_ttl_resolved'
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/%s' % incident['_key']
incidentStr = json.dumps(incident)
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey, jsonargs=incidentStr)
now = datetime.datetime.now().isoformat()
event_id = hashlib.md5(incident['incident_id'] + now).hexdigest()
log.debug("event_id=%s now=%s" % (event_id, now))
event = 'time=%s severity=INFO origin="alert_manager_scheduler" event_id="%s" user="******" action="auto_ttl_resolve" previous_status="%s" status="auto_ttl_resolved" incident_id="%s"' % (now, event_id, old_status, incident['incident_id'])
log.debug("Event will be: %s" % event)
input.submit(event, hostname = socket.gethostname(), sourcetype = 'incident_change', source = 'alert_manager_scheduler.py', index = config['index'])
ic = IncidentContext(sessionKey, incident["incident_id"])
eh.handleEvent(alert=alert["name"], event="incident_auto_ttl_resolved", incident={"owner": incident["owner"]}, context=ic.getContext())
else:
log.info("Incident %s has not ttl reached yet." % incident['incident_id'])
else:
log.info("No incidents of alert %s to check for reached ttl." % alert['name'])
log.debug('Alert "%s" is not configured for auto_ttl_resolve, skipping...' % alert['name'])
#
# Look for auto_suppress_resolve incidents
#
query = {}
query['auto_suppress_resolve'] = True
log.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_settings?query=%s' % urllib.quote(json.dumps(query))
serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
def _update_incident(self, sessionKey, user, post_data):
logger.debug("START _update_incident()")
required = ['incident_data']
missing = [r for r in required if r not in post_data]
if missing:
return self.response("Missing required arguments: %s" % missing,
httplib.BAD_REQUEST)
incident_data = post_data.pop('incident_data')
splunk.setDefault('sessionKey', sessionKey)
eh = EventHandler(sessionKey=sessionKey)
config = {}
config['index'] = 'main'
restconfig = entity.getEntities('configs/alert_manager',
count=-1,
sessionKey=sessionKey)
if len(restconfig) > 0:
if 'index' in restconfig['settings']:
config['index'] = restconfig['settings']['index']
logger.debug("Global settings: %s" % config)
# Parse the JSON
incident_data = json.loads(incident_data)
# Get key
query = {}
query['incident_id'] = incident_data['incident_id']
logger.debug("Filter: %s" % json.dumps(query))
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query=%s' % urllib.quote(
json.dumps(query))
serverResponse, incident = rest.simpleRequest(uri,
sessionKey=sessionKey)
logger.debug("Settings for incident: %s" % incident)
incident = json.loads(incident)
# Update incident
uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents/' + incident[
0]['_key']
logger.debug("URI for incident update: %s" % uri)
# Prepared new entry
now = datetime.datetime.now().isoformat()
changed_keys = []
for key in incident[0].keys():
if (key in incident_data) and (incident[0][key] !=
incident_data[key]):
changed_keys.append(key)
logger.info(
"%s for incident %s changed. Writing change event to index %s."
% (key, incident[0]['incident_id'], config['index']))
event_id = hashlib.md5(incident[0]['incident_id'] +
now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="change" incident_id="%s" %s="%s" previous_%s="%s"' % (
now, event_id, user, incident[0]['incident_id'], key,
incident_data[key], key, incident[0][key])
logger.debug("Change event will be: %s" % event)
input.submit(event,
hostname=socket.gethostname(),
sourcetype='incident_change',
source='incident_settings.py',
index=config['index'])
incident[0][key] = incident_data[key]
else:
logger.info("%s for incident %s didn't change." %
(key, incident[0]['incident_id']))
del incident[0]['_key']
contentsStr = json.dumps(incident[0])
logger.debug("content for update: %s" % contentsStr)
serverResponse, serverContent = rest.simpleRequest(
uri, sessionKey=sessionKey, jsonargs=contentsStr)
logger.debug("Response from update incident entry was %s " %
serverResponse)
logger.debug("Changed keys: %s" % changed_keys)
if len(changed_keys) > 0:
ic = IncidentContext(sessionKey, incident_data['incident_id'])
if "owner" in changed_keys:
eh.handleEvent(alert=incident[0]["alert"],
event="incident_assigned",
incident=incident[0],
context=ic.getContext())
elif "status" in changed_keys and incident_data[
"status"] == "resolved":
eh.handleEvent(alert=incident[0]["alert"],
event="incident_resolved",
incident=incident[0],
context=ic.getContext())
else:
eh.handleEvent(alert=incident[0]["alert"],
event="incident_changed",
incident=incident[0],
context=ic.getContext())
if incident_data['comment'] != "":
incident_data['comment'] = incident_data['comment'].replace(
'\n', '<br />').replace('\r', '')
event_id = hashlib.md5(incident[0]['incident_id'] +
now).hexdigest()
event = 'time=%s severity=INFO origin="incident_posture" event_id="%s" user="******" action="comment" incident_id="%s" comment="%s"' % (
now, event_id, user, incident[0]['incident_id'],
incident_data['comment'])
logger.debug("Comment event will be: %s" % event)
event = event.encode('utf8')
input.submit(event,
hostname=socket.gethostname(),
sourcetype='incident_change',
source='incident_settings.py',
index=config['index'])
ic = IncidentContext(sessionKey, incident_data['incident_id'])
eh.handleEvent(alert=incident[0]["alert"],
event="incident_commented",
incident=incident[0],
context=ic.getContext())
return self.response('Successfully updated incident.', httplib.OK)
