def getEventCorrelationTimeline(eventCorrelationID):
while True:
eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0]
if eventCorrelation.mergedID != "":
eventCorrelationID = eventCorrelation.mergedID
else:
break
eventIDS = []
for eventID in eventCorrelation.ids:
eventIDS.append(jimi.db.ObjectId(eventID))
events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS } })["results"]
timeline = []
for sourceEvent in events:
try:
label = sourceEvent["eventTitle"]
if label == "":
label = sourceEvent["uid"]
except KeyError:
label = sourceEvent["uid"]
formatted_date = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(sourceEvent["eventRaiseTime"]))
timeline.append({ "id" : len(timeline), "content" : label, "start" : formatted_date })
return { "results" : timeline }, 200
def run(self,data,persistentData,actionResult):
correlationID = helpers.evalString(self.correlationID,{"data" : data})
correlatedRelationship = event._eventCorrelation().query(id=correlationID)["results"][0]
actionResult["result"] = True
actionResult["rc"] = 0
actionResult["correlation"] = correlatedRelationship
return actionResult
def check(self):
correlationName = self.correlationName
expiryTime = int(time.time())
if self.includeInactive:
expiryTime = 0
events = event._eventCorrelation().query(
query={
"correlationName": correlationName,
"score": {
"$gt": self.minScore
},
"expiryTime": {
"$gt": expiryTime
}
})["results"]
if self.excludeSingleTypes:
events = [
x for x in events
if len(x["types"]) > 1 or len(x["subTypes"]) > 1
]
if self.summaryOnly:
events = [{
"_id": x["_id"],
"score": x["score"],
"types": x["types"],
"subTypes": x["subTypes"],
"correlations": x["correlations"]
} for x in events]
if self.idsOnly:
events = [x["_id"] for x in events]
self.result["events"] = events
def activeCorrelationTable(action):
findActiveCorrelations = event._eventCorrelation().aggregate(
sessionData=api.g.sessionData,
aggregateStatement=[{
"$project": {
"_id": 1,
"expiryTime": 1,
"types": 1,
"subTypes": 1,
"score": 1,
"idsSize": {
"$cond": {
"if": {
"$isArray": "$ids"
},
"then": {
"$size": "$ids"
},
"else": 0
}
}
}
}, {
"$match": {
"expiryTime": {
"$gt": time.time()
},
"idsSize": {
"$gt": 1
}
}
}])
total = len(findActiveCorrelations)
columns = ["id", "Types", "Sub Types", "Score"]
table = ui.table(columns, total, total)
if action == "build":
return table.getColumns(), 200
elif action == "poll":
# Custom table data so it can be vertical
data = []
for activeCorrelation in findActiveCorrelations:
data.append([
"<a href='/plugin/event/eventCorrelations/{0}/'>{0}</a>".
format(activeCorrelation["_id"]),
ui.dictTable(activeCorrelation["types"]),
ui.dictTable(activeCorrelation["subTypes"]),
ui.dictTable(activeCorrelation["score"])
])
table.data = data
return {
"draw": int(jimi.api.request.args.get('draw')),
"recordsTable": 0,
"recordsFiltered": 0,
"recordsTotal": 0,
"data": data
}, 200
def getEvent(eventCorrelationID):
while True:
eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0]
if eventCorrelation.mergedID != "":
eventCorrelationID = eventCorrelation.mergedID
else:
break
eventCorrelation.expiryTime = 0
eventCorrelation.update(["expiryTime"],sessionData=api.g.sessionData)
return { }, 200
def getEventCorrelation(eventCorrelationID):
while True:
eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0]
if eventCorrelation.mergedID != "":
eventCorrelationID = eventCorrelation.mergedID
else:
break
eventIDS = []
for eventID in eventCorrelation.ids:
eventIDS.append(jimi.db.ObjectId(eventID))
events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS } })["results"]
nodesDict = {}
edgesDict = {}
for sourceEvent in events:
try:
label = sourceEvent["eventTitle"]
if label == "":
label = sourceEvent["uid"]
except KeyError:
label = sourceEvent["uid"]
if label not in nodesDict:
nodesDict[label] = { "id" : sourceEvent["uid"], "label" : label, "value" : 1, "color" : { "background" : "#C72F1E", "border" : "#C72F1E" , "highlight" : { "background" : "#000", "border" : "#FFF" } } }
else:
nodesDict[label]["value"] += 1
nodeUID = nodesDict[label]["id"]
for field, fieldValue in sourceEvent["eventValues"].items():
if type(fieldValue) is list:
for fieldValueItem in fieldValue:
uid = "{0}={1}".format(field,fieldValueItem)
if uid not in nodesDict:
nodesDict[uid] = { "id" : uid, "label" : uid, "value" : 1 }
else:
nodesDict[uid]["value"] += 1
key = "{0}-{1}".format(nodeUID,uid)
edgesDict[key] = { "id" : key, "from" : nodeUID, "to" : uid }
else:
uid = "{0}={1}".format(field,fieldValue)
if uid not in nodesDict:
nodesDict[uid] = { "id" : uid, "label" : uid, "value" : 1 }
else:
nodesDict[uid]["value"] += 1
key = "{0}-{1}".format(nodeUID,uid)
edgesDict[key] = { "id" : key, "from" : nodeUID, "to" : uid }
nodes = [ x for x in nodesDict.values() ]
edges = [ x for x in edgesDict.values() ]
return { "nodes" : nodes, "edges" : edges }, 200
def closeCorrelation(eventCorrelationID,eventUID):
while True:
eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0]
if eventCorrelation.mergedID != "":
eventCorrelationID = eventCorrelation.mergedID
else:
break
eventIDS = []
for eventID in eventCorrelation.ids:
eventIDS.append(jimi.db.ObjectId(eventID))
events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS }, "uid" : eventUID })
return events, 200
def run(self, data, persistentData, actionResult):
correlationName = helpers.evalString(self.correlationName,
{"data": data})
expiryTime = int(time.time())
if self.includeInactive:
expiryTime = 0
correlatedRelationships = event._eventCorrelation().query(
query={
"correlationName": correlationName,
"score": {
"$gt": self.minScore
},
"expiryTime": {
"$gt": expiryTime
}
})["results"]
if self.excludeSingleTypes:
correlatedRelationships = [
x for x in correlatedRelationships
if len(x["types"]) > 1 or len(x["subTypes"]) > 1
]
if self.summaryOnly:
correlatedRelationships = [{
"_id": x["_id"],
"score": x["score"],
"types": x["types"],
"subTypes": x["subTypes"],
"correlations": x["correlations"]
} for x in correlatedRelationships]
if self.idsOnly:
correlatedRelationships = [
x["_id"] for x in correlatedRelationships
]
if self.multiTypeMultiplier > 1:
for correlatedRelationship in correlatedRelationships:
multiplier = ((len(correlatedRelationship["types"]) - 1 +
len(correlatedRelationship["subTypes"]) - 1) *
self.multiTypeMultiplier)
if multiplier > 0:
correlatedRelationship[
"score"] = correlatedRelationship["score"] * multiplier
actionResult["result"] = True
actionResult["rc"] = 0
actionResult["correlations"] = correlatedRelationships
return actionResult
def mainPage():
eventCount = event._event().count(
sessionData=api.g.sessionData,
query={"expiryTime": {
"$gt": time.time()
}})["results"][0]["count"]
correlationCount = event._eventCorrelation().aggregate(
sessionData=api.g.sessionData,
aggregateStatement=[{
"$project": {
"_id": 1,
"expiryTime": 1,
"idsSize": {
"$cond": {
"if": {
"$isArray": "$ids"
},
"then": {
"$size": "$ids"
},
"else": 0
}
}
}
}, {
"$match": {
"expiryTime": {
"$gt": time.time()
},
"idsSize": {
"$gt": 1
}
}
}, {
"$count": "count"
}])
if len(correlationCount) > 0:
correlationCount = correlationCount[0]["count"]
else:
correlationCount = 0
return render_template("home.html",
activeEvents=eventCount,
activeCorrelations=correlationCount)
def eventCorrelationsPage():
findActiveEvents = event._eventCorrelation().aggregate(
sessionData=api.g.sessionData,
aggregateStatement=[{
"$project": {
"_id": 1,
"expiryTime": 1,
"types": 1,
"subTypes": 1,
"score": 1,
"correlationName": 1,
"ids": 1,
"correlations": 1,
"relationships": 1,
"correlationLastUpdate": 1,
"mergedID": 1,
"idsSize": {
"$cond": {
"if": {
"$isArray": "$ids"
},
"then": {
"$size": "$ids"
},
"else": 0
}
}
}
}, {
"$match": {
"expiryTime": {
"$gt": time.time()
},
"idsSize": {
"$gt": 1
}
}
}])
return render_template("eventCorrelations.html",
eventCorrelations=findActiveEvents)
def run(self,data,persistentData,actionResult):
correlationName = helpers.evalString(self.correlationName,{"data" : data})
excludeCorrelationValues = helpers.evalDict(self.excludeCorrelationValues,{"data" : data})
expiryTime = time.time() + self.expiryTime
correlatedRelationships = event._eventCorrelation().getAsClass(query={ "correlationName" : correlationName, "expiryTime" : { "$gt" : int(time.time()) } })
eventsAfterTime = int(time.time()) - self.oldestEvent
if not self.alwaysProcessEvents:
ids = event._eventCorrelation()._dbCollection.distinct("ids",{ "$or" : [ { "expiryTime" : { "$gt" : int(time.time()) } }, { "lastUpdateTime" : { "$gt" : eventsAfterTime } } ] })
objectIds = []
for idItem in ids:
objectIds.append(db.ObjectId(idItem))
events = event._event().getAsClass(query={ "_id" : { "$nin" : objectIds }, "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } })
else:
events = event._event().getAsClass(query={ "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } })
# Build correlation field hash table
fields = {}
for field in self.correlationFields:
fields[field] = {}
if field not in excludeCorrelationValues:
excludeCorrelationValues[field] = []
for correlatedRelationshipItem in correlatedRelationships:
for field in self.correlationFields:
try:
if field not in excludeCorrelationValues:
excludeCorrelationValues[field] = []
for value in correlatedRelationshipItem.correlations[field]:
fields[field][value] = correlatedRelationshipItem
except KeyError:
pass
correlatedRelationshipsCreated = []
correlatedRelationshipsUpdated = []
correlatedRelationshipsDeleted = []
# Initial Pass Loop
for eventItem in events:
foundCorrelatedRelationship = None
correlations = {}
processNew = False
for eventField in eventItem.eventValues:
try:
if type(eventItem.eventValues[eventField]) is list:
correlations[eventField] = [ x for x in eventItem.eventValues[eventField] if eventField in self.correlationFields ]
matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x in fields[eventField] and x not in excludeCorrelationValues[eventField] ]
if len(matchFound) > 0:
foundCorrelatedRelationship = matchFound[0]
else:
matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x and x not in excludeCorrelationValues[eventField] ]
if len(matchFound) > 0:
processNew = True
else:
correlations[eventField] = [eventItem.eventValues[eventField]]
if eventField in self.correlationFields and eventItem.eventValues[eventField] in fields[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]:
foundCorrelatedRelationship = fields[eventField][eventItem.eventValues[eventField]]
else:
if eventField in self.correlationFields and eventItem.eventValues[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]:
processNew = True
except KeyError:
pass
# Create new
if processNew == True:
newEventCorrelation = event._eventCorrelation()
newEventCorrelation.bulkNew(self.bulkClass, self.acl, correlationName,expiryTime,[eventItem._id],[eventItem.eventType],[eventItem.eventSubType],correlations,eventItem.score)
correlatedRelationshipsCreated.append(newEventCorrelation)
correlatedRelationships.append(newEventCorrelation)
for eventField in eventItem.eventValues:
try:
for eventValue in correlations[eventField]:
try:
fields[eventField][eventValue] = newEventCorrelation
except KeyError:
fields[eventField] = { eventValue : newEventCorrelation }
except KeyError:
pass
# Merge existing
elif foundCorrelatedRelationship != None:
for eventField in correlations:
try:
foundCorrelatedRelationship.correlations[eventField] += correlations[eventField]
foundCorrelatedRelationship.correlations[eventField] = list(set(foundCorrelatedRelationship.correlations[eventField]))
except KeyError:
foundCorrelatedRelationship.correlations[eventField] = correlations[eventField]
if eventItem._id not in foundCorrelatedRelationship.ids:
foundCorrelatedRelationship.ids.append(eventItem._id)
foundCorrelatedRelationship.score += eventItem.score
if eventItem.eventType not in foundCorrelatedRelationship.types:
foundCorrelatedRelationship.types.append(eventItem.eventType)
if eventItem.eventSubType not in foundCorrelatedRelationship.subTypes:
foundCorrelatedRelationship.subTypes.append(eventItem.eventSubType)
foundCorrelatedRelationship.correlationLastUpdate = int(time.time())
foundCorrelatedRelationship.expiryTime = expiryTime
if foundCorrelatedRelationship not in correlatedRelationshipsCreated and foundCorrelatedRelationship not in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.append(foundCorrelatedRelationship)
# Process bulk creation if needed before merging
self.bulkClass.bulkOperatonProcessing()
# Reduction Loop
loop = 1
maxLoops = 5
while loop > 0 and maxLoops > 0:
correlatedFieldsHash = {}
for correlatedRelationship in correlatedRelationships:
for eventField, eventValue in ((eventField, eventValue) for eventField in correlatedRelationship.correlations for eventValue in correlatedRelationship.correlations[eventField] ):
if eventField in self.correlationFields and eventValue not in excludeCorrelationValues[eventField]:
try:
if eventValue not in correlatedFieldsHash[eventField]:
correlatedFieldsHash[eventField][eventValue] = correlatedRelationship
else:
currentCorrelation = correlatedFieldsHash[eventField][eventValue]
for eventField in correlatedRelationship.correlations:
try:
currentCorrelation.correlations[eventField] += correlatedRelationship.correlations[eventField]
currentCorrelation.correlations[eventField] = list(set(currentCorrelation.correlations[eventField]))
except KeyError:
currentCorrelation.correlations[eventField] = correlatedRelationship.correlations[eventField]
for mergeKey in ["ids","types","subTypes"]:
for value in getattr(correlatedRelationship,mergeKey):
if value not in getattr(currentCorrelation,mergeKey):
getattr(currentCorrelation,mergeKey).append(value)
currentCorrelation.score += correlatedRelationship.score
currentCorrelation.correlationLastUpdate = int(time.time())
currentCorrelation.expiryTime = expiryTime
if currentCorrelation not in correlatedRelationshipsCreated and correlatedRelationship not in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.append(currentCorrelation)
# Deleting the eventCorrelation it was merged with
if correlatedRelationship not in correlatedRelationshipsDeleted:
correlatedRelationshipsDeleted.append(correlatedRelationship)
if correlatedRelationship in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.remove(correlatedRelationship)
if correlatedRelationship in correlatedRelationshipsCreated:
correlatedRelationshipsCreated.remove(correlatedRelationship)
correlatedRelationship.bulkMerge(currentCorrelation._id,self.bulkClass)
correlatedRelationships.remove(correlatedRelationship)
loop+=1
break
except KeyError:
correlatedFieldsHash[eventField] = { eventValue : correlatedRelationship }
maxLoops -= 1
loop -= 1
created = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsCreated ]
updated = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsUpdated ]
deleted = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsDeleted ]
for correlatedRelationshipUpdated in correlatedRelationshipsUpdated:
correlatedRelationshipUpdated.bulkUpdate(["expiryTime","ids","types","subTypes","correlations","score"],self.bulkClass)
updated.append(helpers.classToJson(correlatedRelationshipUpdated,hidden=True))
self.bulkClass.bulkOperatonProcessing()
actionResult["result"] = True
actionResult["rc"] = 0
actionResult["correlatedEvents"] = { "created" : created, "updated" : updated, "deleted" : deleted }
return actionResult
def eventCorrelationsPage():
findActiveEvents = event._eventCorrelation().query(sessionData=api.g.sessionData,query={"expiryTime" : { "$gt" : time.time() } })["results"]
return render_template("eventCorrelations.html", eventCorrelations=findActiveEvents)
