def getEventCorrelationTimeline(eventCorrelationID): while True: eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0] if eventCorrelation.mergedID != "": eventCorrelationID = eventCorrelation.mergedID else: break eventIDS = [] for eventID in eventCorrelation.ids: eventIDS.append(jimi.db.ObjectId(eventID)) events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS } })["results"] timeline = [] for sourceEvent in events: try: label = sourceEvent["eventTitle"] if label == "": label = sourceEvent["uid"] except KeyError: label = sourceEvent["uid"] formatted_date = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(sourceEvent["eventRaiseTime"])) timeline.append({ "id" : len(timeline), "content" : label, "start" : formatted_date }) return { "results" : timeline }, 200
def run(self,data,persistentData,actionResult): correlationID = helpers.evalString(self.correlationID,{"data" : data}) correlatedRelationship = event._eventCorrelation().query(id=correlationID)["results"][0] actionResult["result"] = True actionResult["rc"] = 0 actionResult["correlation"] = correlatedRelationship return actionResult
def check(self): correlationName = self.correlationName expiryTime = int(time.time()) if self.includeInactive: expiryTime = 0 events = event._eventCorrelation().query( query={ "correlationName": correlationName, "score": { "$gt": self.minScore }, "expiryTime": { "$gt": expiryTime } })["results"] if self.excludeSingleTypes: events = [ x for x in events if len(x["types"]) > 1 or len(x["subTypes"]) > 1 ] if self.summaryOnly: events = [{ "_id": x["_id"], "score": x["score"], "types": x["types"], "subTypes": x["subTypes"], "correlations": x["correlations"] } for x in events] if self.idsOnly: events = [x["_id"] for x in events] self.result["events"] = events
def activeCorrelationTable(action): findActiveCorrelations = event._eventCorrelation().aggregate( sessionData=api.g.sessionData, aggregateStatement=[{ "$project": { "_id": 1, "expiryTime": 1, "types": 1, "subTypes": 1, "score": 1, "idsSize": { "$cond": { "if": { "$isArray": "$ids" }, "then": { "$size": "$ids" }, "else": 0 } } } }, { "$match": { "expiryTime": { "$gt": time.time() }, "idsSize": { "$gt": 1 } } }]) total = len(findActiveCorrelations) columns = ["id", "Types", "Sub Types", "Score"] table = ui.table(columns, total, total) if action == "build": return table.getColumns(), 200 elif action == "poll": # Custom table data so it can be vertical data = [] for activeCorrelation in findActiveCorrelations: data.append([ "<a href='/plugin/event/eventCorrelations/{0}/'>{0}</a>". format(activeCorrelation["_id"]), ui.dictTable(activeCorrelation["types"]), ui.dictTable(activeCorrelation["subTypes"]), ui.dictTable(activeCorrelation["score"]) ]) table.data = data return { "draw": int(jimi.api.request.args.get('draw')), "recordsTable": 0, "recordsFiltered": 0, "recordsTotal": 0, "data": data }, 200
def getEvent(eventCorrelationID): while True: eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0] if eventCorrelation.mergedID != "": eventCorrelationID = eventCorrelation.mergedID else: break eventCorrelation.expiryTime = 0 eventCorrelation.update(["expiryTime"],sessionData=api.g.sessionData) return { }, 200
def getEventCorrelation(eventCorrelationID): while True: eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0] if eventCorrelation.mergedID != "": eventCorrelationID = eventCorrelation.mergedID else: break eventIDS = [] for eventID in eventCorrelation.ids: eventIDS.append(jimi.db.ObjectId(eventID)) events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS } })["results"] nodesDict = {} edgesDict = {} for sourceEvent in events: try: label = sourceEvent["eventTitle"] if label == "": label = sourceEvent["uid"] except KeyError: label = sourceEvent["uid"] if label not in nodesDict: nodesDict[label] = { "id" : sourceEvent["uid"], "label" : label, "value" : 1, "color" : { "background" : "#C72F1E", "border" : "#C72F1E" , "highlight" : { "background" : "#000", "border" : "#FFF" } } } else: nodesDict[label]["value"] += 1 nodeUID = nodesDict[label]["id"] for field, fieldValue in sourceEvent["eventValues"].items(): if type(fieldValue) is list: for fieldValueItem in fieldValue: uid = "{0}={1}".format(field,fieldValueItem) if uid not in nodesDict: nodesDict[uid] = { "id" : uid, "label" : uid, "value" : 1 } else: nodesDict[uid]["value"] += 1 key = "{0}-{1}".format(nodeUID,uid) edgesDict[key] = { "id" : key, "from" : nodeUID, "to" : uid } else: uid = "{0}={1}".format(field,fieldValue) if uid not in nodesDict: nodesDict[uid] = { "id" : uid, "label" : uid, "value" : 1 } else: nodesDict[uid]["value"] += 1 key = "{0}-{1}".format(nodeUID,uid) edgesDict[key] = { "id" : key, "from" : nodeUID, "to" : uid } nodes = [ x for x in nodesDict.values() ] edges = [ x for x in edgesDict.values() ] return { "nodes" : nodes, "edges" : edges }, 200
def closeCorrelation(eventCorrelationID,eventUID): while True: eventCorrelation = event._eventCorrelation().getAsClass(sessionData=api.g.sessionData,id=eventCorrelationID)[0] if eventCorrelation.mergedID != "": eventCorrelationID = eventCorrelation.mergedID else: break eventIDS = [] for eventID in eventCorrelation.ids: eventIDS.append(jimi.db.ObjectId(eventID)) events = event._event().query(sessionData=api.g.sessionData,query={ "_id" : { "$in" : eventIDS }, "uid" : eventUID }) return events, 200
def run(self, data, persistentData, actionResult): correlationName = helpers.evalString(self.correlationName, {"data": data}) expiryTime = int(time.time()) if self.includeInactive: expiryTime = 0 correlatedRelationships = event._eventCorrelation().query( query={ "correlationName": correlationName, "score": { "$gt": self.minScore }, "expiryTime": { "$gt": expiryTime } })["results"] if self.excludeSingleTypes: correlatedRelationships = [ x for x in correlatedRelationships if len(x["types"]) > 1 or len(x["subTypes"]) > 1 ] if self.summaryOnly: correlatedRelationships = [{ "_id": x["_id"], "score": x["score"], "types": x["types"], "subTypes": x["subTypes"], "correlations": x["correlations"] } for x in correlatedRelationships] if self.idsOnly: correlatedRelationships = [ x["_id"] for x in correlatedRelationships ] if self.multiTypeMultiplier > 1: for correlatedRelationship in correlatedRelationships: multiplier = ((len(correlatedRelationship["types"]) - 1 + len(correlatedRelationship["subTypes"]) - 1) * self.multiTypeMultiplier) if multiplier > 0: correlatedRelationship[ "score"] = correlatedRelationship["score"] * multiplier actionResult["result"] = True actionResult["rc"] = 0 actionResult["correlations"] = correlatedRelationships return actionResult
def mainPage(): eventCount = event._event().count( sessionData=api.g.sessionData, query={"expiryTime": { "$gt": time.time() }})["results"][0]["count"] correlationCount = event._eventCorrelation().aggregate( sessionData=api.g.sessionData, aggregateStatement=[{ "$project": { "_id": 1, "expiryTime": 1, "idsSize": { "$cond": { "if": { "$isArray": "$ids" }, "then": { "$size": "$ids" }, "else": 0 } } } }, { "$match": { "expiryTime": { "$gt": time.time() }, "idsSize": { "$gt": 1 } } }, { "$count": "count" }]) if len(correlationCount) > 0: correlationCount = correlationCount[0]["count"] else: correlationCount = 0 return render_template("home.html", activeEvents=eventCount, activeCorrelations=correlationCount)
def eventCorrelationsPage(): findActiveEvents = event._eventCorrelation().aggregate( sessionData=api.g.sessionData, aggregateStatement=[{ "$project": { "_id": 1, "expiryTime": 1, "types": 1, "subTypes": 1, "score": 1, "correlationName": 1, "ids": 1, "correlations": 1, "relationships": 1, "correlationLastUpdate": 1, "mergedID": 1, "idsSize": { "$cond": { "if": { "$isArray": "$ids" }, "then": { "$size": "$ids" }, "else": 0 } } } }, { "$match": { "expiryTime": { "$gt": time.time() }, "idsSize": { "$gt": 1 } } }]) return render_template("eventCorrelations.html", eventCorrelations=findActiveEvents)
def run(self,data,persistentData,actionResult): correlationName = helpers.evalString(self.correlationName,{"data" : data}) excludeCorrelationValues = helpers.evalDict(self.excludeCorrelationValues,{"data" : data}) expiryTime = time.time() + self.expiryTime correlatedRelationships = event._eventCorrelation().getAsClass(query={ "correlationName" : correlationName, "expiryTime" : { "$gt" : int(time.time()) } }) eventsAfterTime = int(time.time()) - self.oldestEvent if not self.alwaysProcessEvents: ids = event._eventCorrelation()._dbCollection.distinct("ids",{ "$or" : [ { "expiryTime" : { "$gt" : int(time.time()) } }, { "lastUpdateTime" : { "$gt" : eventsAfterTime } } ] }) objectIds = [] for idItem in ids: objectIds.append(db.ObjectId(idItem)) events = event._event().getAsClass(query={ "_id" : { "$nin" : objectIds }, "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } }) else: events = event._event().getAsClass(query={ "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } }) # Build correlation field hash table fields = {} for field in self.correlationFields: fields[field] = {} if field not in excludeCorrelationValues: excludeCorrelationValues[field] = [] for correlatedRelationshipItem in correlatedRelationships: for field in self.correlationFields: try: if field not in excludeCorrelationValues: excludeCorrelationValues[field] = [] for value in correlatedRelationshipItem.correlations[field]: fields[field][value] = correlatedRelationshipItem except KeyError: pass correlatedRelationshipsCreated = [] correlatedRelationshipsUpdated = [] correlatedRelationshipsDeleted = [] # Initial Pass Loop for eventItem in events: foundCorrelatedRelationship = None correlations = {} processNew = False for eventField in eventItem.eventValues: try: if type(eventItem.eventValues[eventField]) is list: correlations[eventField] = [ x for x in eventItem.eventValues[eventField] if eventField in self.correlationFields ] matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x in fields[eventField] and x not in excludeCorrelationValues[eventField] ] if len(matchFound) > 0: foundCorrelatedRelationship = matchFound[0] else: matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x and x not in excludeCorrelationValues[eventField] ] if len(matchFound) > 0: processNew = True else: correlations[eventField] = [eventItem.eventValues[eventField]] if eventField in self.correlationFields and eventItem.eventValues[eventField] in fields[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]: foundCorrelatedRelationship = fields[eventField][eventItem.eventValues[eventField]] else: if eventField in self.correlationFields and eventItem.eventValues[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]: processNew = True except KeyError: pass # Create new if processNew == True: newEventCorrelation = event._eventCorrelation() newEventCorrelation.bulkNew(self.bulkClass, self.acl, correlationName,expiryTime,[eventItem._id],[eventItem.eventType],[eventItem.eventSubType],correlations,eventItem.score) correlatedRelationshipsCreated.append(newEventCorrelation) correlatedRelationships.append(newEventCorrelation) for eventField in eventItem.eventValues: try: for eventValue in correlations[eventField]: try: fields[eventField][eventValue] = newEventCorrelation except KeyError: fields[eventField] = { eventValue : newEventCorrelation } except KeyError: pass # Merge existing elif foundCorrelatedRelationship != None: for eventField in correlations: try: foundCorrelatedRelationship.correlations[eventField] += correlations[eventField] foundCorrelatedRelationship.correlations[eventField] = list(set(foundCorrelatedRelationship.correlations[eventField])) except KeyError: foundCorrelatedRelationship.correlations[eventField] = correlations[eventField] if eventItem._id not in foundCorrelatedRelationship.ids: foundCorrelatedRelationship.ids.append(eventItem._id) foundCorrelatedRelationship.score += eventItem.score if eventItem.eventType not in foundCorrelatedRelationship.types: foundCorrelatedRelationship.types.append(eventItem.eventType) if eventItem.eventSubType not in foundCorrelatedRelationship.subTypes: foundCorrelatedRelationship.subTypes.append(eventItem.eventSubType) foundCorrelatedRelationship.correlationLastUpdate = int(time.time()) foundCorrelatedRelationship.expiryTime = expiryTime if foundCorrelatedRelationship not in correlatedRelationshipsCreated and foundCorrelatedRelationship not in correlatedRelationshipsUpdated: correlatedRelationshipsUpdated.append(foundCorrelatedRelationship) # Process bulk creation if needed before merging self.bulkClass.bulkOperatonProcessing() # Reduction Loop loop = 1 maxLoops = 5 while loop > 0 and maxLoops > 0: correlatedFieldsHash = {} for correlatedRelationship in correlatedRelationships: for eventField, eventValue in ((eventField, eventValue) for eventField in correlatedRelationship.correlations for eventValue in correlatedRelationship.correlations[eventField] ): if eventField in self.correlationFields and eventValue not in excludeCorrelationValues[eventField]: try: if eventValue not in correlatedFieldsHash[eventField]: correlatedFieldsHash[eventField][eventValue] = correlatedRelationship else: currentCorrelation = correlatedFieldsHash[eventField][eventValue] for eventField in correlatedRelationship.correlations: try: currentCorrelation.correlations[eventField] += correlatedRelationship.correlations[eventField] currentCorrelation.correlations[eventField] = list(set(currentCorrelation.correlations[eventField])) except KeyError: currentCorrelation.correlations[eventField] = correlatedRelationship.correlations[eventField] for mergeKey in ["ids","types","subTypes"]: for value in getattr(correlatedRelationship,mergeKey): if value not in getattr(currentCorrelation,mergeKey): getattr(currentCorrelation,mergeKey).append(value) currentCorrelation.score += correlatedRelationship.score currentCorrelation.correlationLastUpdate = int(time.time()) currentCorrelation.expiryTime = expiryTime if currentCorrelation not in correlatedRelationshipsCreated and correlatedRelationship not in correlatedRelationshipsUpdated: correlatedRelationshipsUpdated.append(currentCorrelation) # Deleting the eventCorrelation it was merged with if correlatedRelationship not in correlatedRelationshipsDeleted: correlatedRelationshipsDeleted.append(correlatedRelationship) if correlatedRelationship in correlatedRelationshipsUpdated: correlatedRelationshipsUpdated.remove(correlatedRelationship) if correlatedRelationship in correlatedRelationshipsCreated: correlatedRelationshipsCreated.remove(correlatedRelationship) correlatedRelationship.bulkMerge(currentCorrelation._id,self.bulkClass) correlatedRelationships.remove(correlatedRelationship) loop+=1 break except KeyError: correlatedFieldsHash[eventField] = { eventValue : correlatedRelationship } maxLoops -= 1 loop -= 1 created = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsCreated ] updated = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsUpdated ] deleted = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsDeleted ] for correlatedRelationshipUpdated in correlatedRelationshipsUpdated: correlatedRelationshipUpdated.bulkUpdate(["expiryTime","ids","types","subTypes","correlations","score"],self.bulkClass) updated.append(helpers.classToJson(correlatedRelationshipUpdated,hidden=True)) self.bulkClass.bulkOperatonProcessing() actionResult["result"] = True actionResult["rc"] = 0 actionResult["correlatedEvents"] = { "created" : created, "updated" : updated, "deleted" : deleted } return actionResult
def eventCorrelationsPage(): findActiveEvents = event._eventCorrelation().query(sessionData=api.g.sessionData,query={"expiryTime" : { "$gt" : time.time() } })["results"] return render_template("eventCorrelations.html", eventCorrelations=findActiveEvents)