def activeEventsTable(action):
findActiveEvents = event._event().getAsClass(
sessionData=api.g.sessionData,
query={"expiryTime": {
"$gt": time.time()
}})
total = len(findActiveEvents)
columns = ["id", "Title", "Score"]
table = ui.table(columns, total, total)
if action == "build":
return table.getColumns(), 200
elif action == "poll":
# Custom table data so it can be vertical
data = []
for activeEvent in findActiveEvents:
data.append([
ui.safe(activeEvent._id),
ui.dictTable(activeEvent.eventTitle),
ui.dictTable(activeEvent.score)
])
table.data = data
return {
"draw": int(jimi.api.request.args.get('draw')),
"recordsTable": 0,
"recordsFiltered": 0,
"recordsTotal": 0,
"data": data
}, 200
def getEventCorrelationTimeline(eventCorrelationID):
while True:
eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0]
if eventCorrelation.mergedID != "":
eventCorrelationID = eventCorrelation.mergedID
else:
break
eventIDS = []
for eventID in eventCorrelation.ids:
eventIDS.append(jimi.db.ObjectId(eventID))
events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS } })["results"]
timeline = []
for sourceEvent in events:
try:
label = sourceEvent["eventTitle"]
if label == "":
label = sourceEvent["uid"]
except KeyError:
label = sourceEvent["uid"]
formatted_date = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(sourceEvent["eventRaiseTime"]))
timeline.append({ "id" : len(timeline), "content" : label, "start" : formatted_date })
return { "results" : timeline }, 200
def eventsPage():
findActiveEvents = event._event().query(
sessionData=api.g.sessionData,
query={"expiryTime": {
"$gt": time.time()
}})["results"]
return render_template("event.html", events=findActiveEvents)
def getEventCorrelation(eventCorrelationID):
while True:
eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0]
if eventCorrelation.mergedID != "":
eventCorrelationID = eventCorrelation.mergedID
else:
break
eventIDS = []
for eventID in eventCorrelation.ids:
eventIDS.append(jimi.db.ObjectId(eventID))
events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS } })["results"]
nodesDict = {}
edgesDict = {}
for sourceEvent in events:
try:
label = sourceEvent["eventTitle"]
if label == "":
label = sourceEvent["uid"]
except KeyError:
label = sourceEvent["uid"]
if label not in nodesDict:
nodesDict[label] = { "id" : sourceEvent["uid"], "label" : label, "value" : 1, "color" : { "background" : "#C72F1E", "border" : "#C72F1E" , "highlight" : { "background" : "#000", "border" : "#FFF" } } }
else:
nodesDict[label]["value"] += 1
nodeUID = nodesDict[label]["id"]
for field, fieldValue in sourceEvent["eventValues"].items():
if type(fieldValue) is list:
for fieldValueItem in fieldValue:
uid = "{0}={1}".format(field,fieldValueItem)
if uid not in nodesDict:
nodesDict[uid] = { "id" : uid, "label" : uid, "value" : 1 }
else:
nodesDict[uid]["value"] += 1
key = "{0}-{1}".format(nodeUID,uid)
edgesDict[key] = { "id" : key, "from" : nodeUID, "to" : uid }
else:
uid = "{0}={1}".format(field,fieldValue)
if uid not in nodesDict:
nodesDict[uid] = { "id" : uid, "label" : uid, "value" : 1 }
else:
nodesDict[uid]["value"] += 1
key = "{0}-{1}".format(nodeUID,uid)
edgesDict[key] = { "id" : key, "from" : nodeUID, "to" : uid }
nodes = [ x for x in nodesDict.values() ]
edges = [ x for x in edgesDict.values() ]
return { "nodes" : nodes, "edges" : edges }, 200
def closeCorrelation(eventCorrelationID,eventUID):
while True:
eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0]
if eventCorrelation.mergedID != "":
eventCorrelationID = eventCorrelation.mergedID
else:
break
eventIDS = []
for eventID in eventCorrelation.ids:
eventIDS.append(jimi.db.ObjectId(eventID))
events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS }, "uid" : eventUID })
return events, 200
def getEvent(cacheUID, sessionData, conductID, flowID, uid, eventType,
eventSubType):
results = event._event().getAsClass(
query={
"conductID": conductID,
"flowID": flowID,
"uid": uid,
"eventType": eventType,
"eventSubType": eventSubType,
"expiryTime": {
"$gt": time.time()
}
})
if len(results) > 0:
return results[0]
return None
def mainPage():
eventCount = event._event().count(
sessionData=api.g.sessionData,
query={"expiryTime": {
"$gt": time.time()
}})["results"][0]["count"]
correlationCount = event._eventCorrelation().aggregate(
sessionData=api.g.sessionData,
aggregateStatement=[{
"$project": {
"_id": 1,
"expiryTime": 1,
"idsSize": {
"$cond": {
"if": {
"$isArray": "$ids"
},
"then": {
"$size": "$ids"
},
"else": 0
}
}
}
}, {
"$match": {
"expiryTime": {
"$gt": time.time()
},
"idsSize": {
"$gt": 1
}
}
}, {
"$count": "count"
}])
if len(correlationCount) > 0:
correlationCount = correlationCount[0]["count"]
else:
correlationCount = 0
return render_template("home.html",
activeEvents=eventCount,
activeCorrelations=correlationCount)
def run(self,data,persistentData,actionResult):
eventTitle = helpers.evalString(self.eventTitle,{"data" : data})
eventType = helpers.evalString(self.eventType,{"data" : data})
eventSubType = helpers.evalString(self.eventSubType,{"data" : data})
layer = self.layer
accuracy = self.accuracy
impact = self.impact
benign = self.benign
timeToLive = self.timeToLive
uid = helpers.evalString(self.uid,{"data" : data})
eventValues = helpers.evalDict(self.eventValues,{"data" : data})
uid = "{0}-{1}-{2}-{3}".format(self._id,eventType,eventSubType,uid)
data["var"]["event"] = {}
data["var"]["event"]["type"] = eventType
data["var"]["event"]["eventSubType"] = eventSubType
data["var"]["event"]["layer"] = layer
data["var"]["event"]["accuracy"] = accuracy
data["var"]["event"]["impact"] = impact
data["var"]["event"]["benign"] = benign
try:
score = ((accuracy*(impact*layer))/benign)
except ZeroDivisionError:
score = 0
data["var"]["event"]["score"] = score
cacheUID = "{0}-{1}-{2}".format(data["conductID"],data["flow_id"],uid)
foundEvent = cache.globalCache.get("eventCache",cacheUID,getEvent,data["conductID"],data["flow_id"],uid,eventType,eventSubType,extendCacheTime=True,customCacheTime=timeToLive,nullUpdate=True)
if foundEvent != None:
try:
persistentData["plugin"]["event"].append(foundEvent)
except:
persistentData["plugin"]["event"] = [foundEvent]
arrayIndex = len(persistentData["plugin"]["event"])-1
actionResult["eventIndex"] = arrayIndex
if foundEvent._id != "":
if foundEvent.expiryTime > time.time():
changes = False
for key,value in eventValues.items():
if key in foundEvent.eventValues:
if value != foundEvent.eventValues[key]:
changes = True
break
else:
changes = True
break
if changes:
foundEvent.updateRecord(self.bulkClass,eventValues,accuracy,impact,layer,benign,score,int( time.time() + timeToLive ),self.history)
actionResult["result"] = True
actionResult["rc"] = 202
return actionResult
else:
foundEvent.expiryTime = int(time.time() + timeToLive)
foundEvent.bulkUpdate(["expiryTime"],self.bulkClass)
actionResult["result"] = True
actionResult["rc"] = 302
return actionResult
else:
cache.globalCache.delete("eventCache",cacheUID)
else:
logging.debug("Event Update Failed - NO ID, actionID='{0}'".format(self._id),7)
actionResult["result"] = False
actionResult["rc"] = 500
return actionResult
eventObject = event._event().bulkNew(self.bulkClass,self.acl,data["conductID"],data["flow_id"],eventType,eventSubType,int( time.time() + timeToLive ),eventValues,uid,accuracy,impact,layer,benign,score,data,eventTitle)
cache.globalCache.insert("eventCache",cacheUID,eventObject,customCacheTime=timeToLive)
try:
persistentData["plugin"]["event"].append(eventObject)
except:
persistentData["plugin"]["event"] = [eventObject]
arrayIndex = len(persistentData["plugin"]["event"])-1
actionResult["eventIndex"] = arrayIndex
actionResult["result"] = True
actionResult["rc"] = 201
return actionResult
def run(self,data,persistentData,actionResult):
correlationName = helpers.evalString(self.correlationName,{"data" : data})
excludeCorrelationValues = helpers.evalDict(self.excludeCorrelationValues,{"data" : data})
expiryTime = time.time() + self.expiryTime
correlatedRelationships = event._eventCorrelation().getAsClass(query={ "correlationName" : correlationName, "expiryTime" : { "$gt" : int(time.time()) } })
eventsAfterTime = int(time.time()) - self.oldestEvent
if not self.alwaysProcessEvents:
ids = event._eventCorrelation()._dbCollection.distinct("ids",{ "$or" : [ { "expiryTime" : { "$gt" : int(time.time()) } }, { "lastUpdateTime" : { "$gt" : eventsAfterTime } } ] })
objectIds = []
for idItem in ids:
objectIds.append(db.ObjectId(idItem))
events = event._event().getAsClass(query={ "_id" : { "$nin" : objectIds }, "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } })
else:
events = event._event().getAsClass(query={ "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } })
# Build correlation field hash table
fields = {}
for field in self.correlationFields:
fields[field] = {}
if field not in excludeCorrelationValues:
excludeCorrelationValues[field] = []
for correlatedRelationshipItem in correlatedRelationships:
for field in self.correlationFields:
try:
if field not in excludeCorrelationValues:
excludeCorrelationValues[field] = []
for value in correlatedRelationshipItem.correlations[field]:
fields[field][value] = correlatedRelationshipItem
except KeyError:
pass
correlatedRelationshipsCreated = []
correlatedRelationshipsUpdated = []
correlatedRelationshipsDeleted = []
# Initial Pass Loop
for eventItem in events:
foundCorrelatedRelationship = None
correlations = {}
processNew = False
for eventField in eventItem.eventValues:
try:
if type(eventItem.eventValues[eventField]) is list:
correlations[eventField] = [ x for x in eventItem.eventValues[eventField] if eventField in self.correlationFields ]
matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x in fields[eventField] and x not in excludeCorrelationValues[eventField] ]
if len(matchFound) > 0:
foundCorrelatedRelationship = matchFound[0]
else:
matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x and x not in excludeCorrelationValues[eventField] ]
if len(matchFound) > 0:
processNew = True
else:
correlations[eventField] = [eventItem.eventValues[eventField]]
if eventField in self.correlationFields and eventItem.eventValues[eventField] in fields[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]:
foundCorrelatedRelationship = fields[eventField][eventItem.eventValues[eventField]]
else:
if eventField in self.correlationFields and eventItem.eventValues[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]:
processNew = True
except KeyError:
pass
# Create new
if processNew == True:
newEventCorrelation = event._eventCorrelation()
newEventCorrelation.bulkNew(self.bulkClass, self.acl, correlationName,expiryTime,[eventItem._id],[eventItem.eventType],[eventItem.eventSubType],correlations,eventItem.score)
correlatedRelationshipsCreated.append(newEventCorrelation)
correlatedRelationships.append(newEventCorrelation)
for eventField in eventItem.eventValues:
try:
for eventValue in correlations[eventField]:
try:
fields[eventField][eventValue] = newEventCorrelation
except KeyError:
fields[eventField] = { eventValue : newEventCorrelation }
except KeyError:
pass
# Merge existing
elif foundCorrelatedRelationship != None:
for eventField in correlations:
try:
foundCorrelatedRelationship.correlations[eventField] += correlations[eventField]
foundCorrelatedRelationship.correlations[eventField] = list(set(foundCorrelatedRelationship.correlations[eventField]))
except KeyError:
foundCorrelatedRelationship.correlations[eventField] = correlations[eventField]
if eventItem._id not in foundCorrelatedRelationship.ids:
foundCorrelatedRelationship.ids.append(eventItem._id)
foundCorrelatedRelationship.score += eventItem.score
if eventItem.eventType not in foundCorrelatedRelationship.types:
foundCorrelatedRelationship.types.append(eventItem.eventType)
if eventItem.eventSubType not in foundCorrelatedRelationship.subTypes:
foundCorrelatedRelationship.subTypes.append(eventItem.eventSubType)
foundCorrelatedRelationship.correlationLastUpdate = int(time.time())
foundCorrelatedRelationship.expiryTime = expiryTime
if foundCorrelatedRelationship not in correlatedRelationshipsCreated and foundCorrelatedRelationship not in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.append(foundCorrelatedRelationship)
# Process bulk creation if needed before merging
self.bulkClass.bulkOperatonProcessing()
# Reduction Loop
loop = 1
maxLoops = 5
while loop > 0 and maxLoops > 0:
correlatedFieldsHash = {}
for correlatedRelationship in correlatedRelationships:
for eventField, eventValue in ((eventField, eventValue) for eventField in correlatedRelationship.correlations for eventValue in correlatedRelationship.correlations[eventField] ):
if eventField in self.correlationFields and eventValue not in excludeCorrelationValues[eventField]:
try:
if eventValue not in correlatedFieldsHash[eventField]:
correlatedFieldsHash[eventField][eventValue] = correlatedRelationship
else:
currentCorrelation = correlatedFieldsHash[eventField][eventValue]
for eventField in correlatedRelationship.correlations:
try:
currentCorrelation.correlations[eventField] += correlatedRelationship.correlations[eventField]
currentCorrelation.correlations[eventField] = list(set(currentCorrelation.correlations[eventField]))
except KeyError:
currentCorrelation.correlations[eventField] = correlatedRelationship.correlations[eventField]
for mergeKey in ["ids","types","subTypes"]:
for value in getattr(correlatedRelationship,mergeKey):
if value not in getattr(currentCorrelation,mergeKey):
getattr(currentCorrelation,mergeKey).append(value)
currentCorrelation.score += correlatedRelationship.score
currentCorrelation.correlationLastUpdate = int(time.time())
currentCorrelation.expiryTime = expiryTime
if currentCorrelation not in correlatedRelationshipsCreated and correlatedRelationship not in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.append(currentCorrelation)
# Deleting the eventCorrelation it was merged with
if correlatedRelationship not in correlatedRelationshipsDeleted:
correlatedRelationshipsDeleted.append(correlatedRelationship)
if correlatedRelationship in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.remove(correlatedRelationship)
if correlatedRelationship in correlatedRelationshipsCreated:
correlatedRelationshipsCreated.remove(correlatedRelationship)
correlatedRelationship.bulkMerge(currentCorrelation._id,self.bulkClass)
correlatedRelationships.remove(correlatedRelationship)
loop+=1
break
except KeyError:
correlatedFieldsHash[eventField] = { eventValue : correlatedRelationship }
maxLoops -= 1
loop -= 1
created = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsCreated ]
updated = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsUpdated ]
deleted = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsDeleted ]
for correlatedRelationshipUpdated in correlatedRelationshipsUpdated:
correlatedRelationshipUpdated.bulkUpdate(["expiryTime","ids","types","subTypes","correlations","score"],self.bulkClass)
updated.append(helpers.classToJson(correlatedRelationshipUpdated,hidden=True))
self.bulkClass.bulkOperatonProcessing()
actionResult["result"] = True
actionResult["rc"] = 0
actionResult["correlatedEvents"] = { "created" : created, "updated" : updated, "deleted" : deleted }
return actionResult
