def run(self,data,persistentData,actionResult): headers = helpers.evalDict(self.headers,{"data" : data}) url = helpers.evalString(self.url,{"data" : data}) method = helpers.evalString(self.method,{"data" : data}).upper() postData = helpers.evalString(self.postData,{"data" : data}) kwargs={} kwargs["headers"] = headers if self.timeout == 0: kwargs["timeout"] = 5 else: kwargs["timeout"] = self.timeout if self.ca: kwargs["verify"] = Path(self.ca) if self.proxy: kwargs["proxies"] = self.proxy else: kwargs["proxies"] = { "http": None, "https": None } if self.cookies: cookies = helpers.evalDict(self.cookies,{"data" : data}) kwargs["cookies"] = cookies try: if method == "GET": response = requests.get(url,**kwargs) elif method == "POST": kwargs["data"] = postData response = requests.post(url,**kwargs) elif method == "PUT": kwargs["data"] = postData response = requests.put(url,**kwargs) elif method == "DELETE": response = requests.delete(url,**kwargs) text = response.text if self.returnsJson: try: text = response.json() except: text = response.text actionResult["result"] = True actionResult["rc"] = response.status_code actionResult["data"] = { "headers" : dict(response.headers), "text" : text } except: actionResult["result"] = False actionResult["rc"] = 404 return actionResult
def doAction(self, data): activityData = helpers.evalDict(self.activityData, {"data": data["flowData"]}) conversationId = helpers.evalString(self.conversationId, {"data": data["flowData"]}) activityId = helpers.evalString(self.activityId, {"data": data["flowData"]}) if not self.client_id: azureBotTrigger = data["persistentData"]["system"]["trigger"] if not hasattr(azureBotTrigger, "azurebotservice"): azureBotTrigger.azurebotservice = azurebotservice._azurebotservice( clientId=azureBotTrigger.client_id, clientSecret=jimi.auth.getPasswordFromENC( azureBotTrigger.client_secret), serviceUrl=data["flowData"]["event"]["serviceUrl"]) else: azureBotTrigger.azurebotservice = azurebotservice._azurebotservice( clientId=self.client_id, clientSecret=jimi.auth.getPasswordFromENC(self.client_secret), serviceUrl=self.service_url) result = azureBotTrigger.azurebotservice.updateActivity( activityData, conversationId, activityId) if result: return {"result": True, "rc": 0, "msg": "Activity updated."} else: return { "result": False, "rc": 400, "msg": "Invalid response from web service, check your url." }
def run(self, data, persistentData, actionResult): relationshipData = helpers.evalDict(self.relationshipData, {"data": data}) events = helpers.evalString(self.eventField, {"data": data}) timespan = 3600 if self.timespan > 0: timespan = self.timespan timespan = helpers.roundTime(roundTo=timespan).timestamp() matches = {} for event in events: matches["{0}->{1}".format(event[self.fromAssetField], event[self.toAssetField])] = event results = _assetRelationship().query(query={"timespan": timespan}, fields=["fromAsset", "toAsset"])["results"] for result in results: match = "{0}->{1}".format(result["fromAsset"], result["toAsset"]) if match in matches: del matches[match] for match in matches: _assetRelationship().bulkNew(self.acl, timespan, matches[match][self.fromAssetField], matches[match][self.toAssetField], relationshipData, self.bulkClass) actionResult["result"] = True actionResult["rc"] = 0 return actionResult
def run(self,data,persistentData,actionResult): matches = helpers.evalDict(self.matches,{ "data" : data }) lastSeenWithin = self.lastSeenWithin if lastSeenWithin == 0: lastSeenWithin = 86400 ors = {} finds = {} for matchItem, matchValue in matches.items(): if type(matchValue) is list: ors[matchItem] = matchValue else: finds[matchItem] = matchValue search = {} search["$or"] = [] for matchItem, matchValue in matches.items(): if type(matchValue) is list: for matchValueItem in matchValue: if matchValueItem != None: search["$or"].append( { matchItem : matchValueItem }) else: if matchValue != None: search["$or"].append( { matchItem : matchValue }) search["lastSeenTimestamp"] = { "$gt" : time.time() - lastSeenWithin } assetsList = asset._asset().query(query=search)["results"] actionResult["assets"] = assetsList actionResult["result"] = True actionResult["rc"] = 0 return actionResult
def run(self, data, persistentData, actionResult): relationshipData = helpers.evalDict(self.relationshipData, {"data": data}) fromAsset = helpers.evalString(self.fromAsset, {"data": data}) toAsset = helpers.evalString(self.toAsset, {"data": data}) timespan = 3600 if self.timespan > 0: timespan = self.timespan timespan = helpers.roundTime(roundTo=timespan).timestamp() match = "{0}={1}->{2}".format(timespan, fromAsset, toAsset) # This will cause a memory leak overtime as globalCache is not cleared down --- this needs to be fixed relationship = cache.globalCache.get("assetRelationshipCache", match, getAssetRelationshipObject, timespan, fromAsset, toAsset, extendCacheTime=True) if not relationship: relationshipItem = _assetRelationship().bulkNew( self.acl, timespan, fromAsset, toAsset, relationshipData, self.bulkClass) if relationshipItem: cache.globalCache.insert("assetRelationshipCache", match, [relationshipItem]) actionResult["result"] = True actionResult["rc"] = 0 return actionResult
def check(self): latestTime = 0 search = helpers.evalDict(self.search, {"data": {}}) match = "" for key, value in search.items(): match += key if type(value) == str: match += value if not self.cache: # cache does not support nested dicts as these will always be seen as the same assetList = cache.globalCache.get("assetSearchCache", match, getSearchObject, search, self.fields) else: assetList = asset._asset().query(query=search, fields=self.fields)["results"] if assetList is not None: for assetItem in assetList: if len(self.flattenFields) > 0: for flattenField in self.flattenFields: if flattenField in assetItem["fields"]: assetItem[flattenField] = assetItem["fields"][ flattenField] del assetItem["fields"][flattenField] if self.return_one: if "event" in self.result: if self.result["event"]["lastUpdateTime"] > latestTime: self.result["event"] = assetItem else: self.result["event"] = asset else: self.result["events"].append(asset)
def doAction(self, data): actionResult = {} scanName = helpers.evalString(self.scanName, {"data": data["flowData"]}) customSearch = helpers.evalDict(self.customSearch, {"data": data["flowData"]}) if scanName != "": search = {"scanName": scanName} else: search = {} if customSearch: for key, value in customSearch.items(): search[key] = value actionResult["result"] = True actionResult["rc"] = 0 if self.limit > 0: actionResult["events"] = inga._inga().query( query=search, limit=self.limit, sort=[("ports.scanDetails.lastPortScan", 1)])["results"] else: actionResult["events"] = inga._inga().query( query=search, sort=[("ports.scanDetails.lastPortScan", 1)])["results"] return actionResult
def run(self, data, persistentData, actionResult): eventIndex = helpers.evalString(self.eventIndex, {"data": data}) eventValues = helpers.evalDict(self.eventValues, {"data": data}) try: currentEvent = persistentData["plugin"]["event"][eventIndex] if self.updateMode == 0: for key, value in eventValues.items(): currentEvent.eventValues[key] = value elif self.updateMode == 1: currentEvent.eventValues = eventValues elif self.updateMode == 2: for key, value in eventValues.items(): if value: if key in currentEvent.eventValues: if type(currentEvent.eventValues[key]) != list: currentEvent.eventValues[key] = [ currentEvent.eventValues[key] ] if type(value) is list: currentEvent.eventValues[key] += value else: currentEvent.eventValues[key].append(value) else: currentEvent.eventValues[key] = value currentEvent.eventFields = list(currentEvent.eventValues.keys()) currentEvent.update(["eventValues", "eventFields"]) actionResult["result"] = True actionResult["rc"] = 0 except KeyError: actionResult["result"] = False actionResult["rc"] = 404 actionResult["msg"] = "No event found within current flow" return actionResult
def varEval(varDict,currentVarDict,dicts={}): functionSafeList = function.systemFunctions for key, value in varDict.items(): logicResult = True if "if" in value: logicResult = logic.ifEval(value["if"],dicts) if logicResult: # Checking that supplied dictionary contains key value if "value" in value: if type(value["value"]) is str: currentVarDict[key] = helpers.evalString(value["value"],dicts,functionSafeList) elif type(value["value"]) is dict: if key in currentVarDict: currentVarDict[key].update(helpers.evalDict(value["value"],dicts,functionSafeList)) else: currentVarDict[key] = helpers.evalDict(value["value"],dicts,functionSafeList) else: currentVarDict[key] = value["value"] return currentVarDict
def doAction(self, data): try: if "skip" in data["flowData"]["plugin"]["azurebotservice"]: conversationData = [] currentConversation = conversation._azurebotserviceconversation( ).getAsClass( query={ "conversationId": data["flowData"]["event"]["conversation"]["id"], "conductId": data["flowData"]["conductID"] }) if len(currentConversation) > 0: currentConversation = currentConversation[0] conversationData = currentConversation.conversationData currentConversation.conductId = None currentConversation.flowId = None currentConversation.update(["conductId", "flowId"]) del data["flowData"]["plugin"]["azurebotservice"]["skip"] return { "result": True, "rc": 0, "msg": "Conversation resumed.", "conversationData": conversationData } except KeyError: pass preserveData = helpers.evalDict(self.preserveData, {"data": data["flowData"]}) currentConversation = conversation._azurebotserviceconversation( ).getAsClass( query={ "conversationId": data["flowData"]["event"]["conversation"] ["id"], "conductId": data["flowData"]["conductID"] }) if len(currentConversation) > 0: currentConversation = currentConversation[0] currentConversation.updateConversation( data["flowData"]["conductID"], data["flowData"]["flow_id"], self.waitFor, preserveData) else: conversation._azurebotserviceconversation().new( self.acl, data["flowData"]["event"]["conversation"]["id"], data["flowData"]["conductID"], data["flowData"]["flow_id"], self.waitFor, preserveData) return {"result": False, "rc": 200, "msg": "Conversation saved."}
def run(self,data,persistentData,actionResult): eventIndex = helpers.evalString(self.eventIndex,{"data" : data}) eventValues = helpers.evalDict(self.eventValues,{"data" : data}) try: currentEvent = persistentData["plugin"]["event"][eventIndex] if self.updateMode == 0: for key, value in eventValues.items(): currentEvent.eventValues[key] = value elif self.updateMode == 1: currentEvent.eventValues = eventValues currentEvent.update(["eventValues"]) actionResult["result"] = True actionResult["rc"] = 0 except KeyError: actionResult["result"] = False actionResult["rc"] = 404 actionResult["msg"] = "No event found within current flow" return actionResult
def run(self,data,persistentData,actionResult): latestTime = 0 search = helpers.evalDict(self.search,{"data" : data}) order_by = helpers.evalString(self.order_by,{"data" : data}) match = "" for key, value in search.items(): match += key if type(value) == str: match += value sort = None if order_by != "": if self.ascending: sort = [(order_by, 1)] else: sort = [(order_by, -1)] if not self.cache: # cache does not support nested dicts as these will always be seen as the same assetList = cache.globalCache.get("assetSearchCache",match,getSearchObject,search,self.fields,sort) else: assetList = asset._asset().query(query=search,fields=self.fields,sort=sort)["results"] if self.return_one: actionResult["event"] = "" else: actionResult["events"] = [] if assetList is not None: for assetItem in assetList: if len(self.flattenFields) > 0: for flattenField in self.flattenFields: if flattenField in assetItem["fields"]: assetItem[flattenField] = assetItem["fields"][flattenField] del assetItem["fields"][flattenField] if self.return_one: if assetItem["lastUpdateTime"] > latestTime: actionResult["event"] = assetItem else: actionResult["events"].append(assetItem) actionResult["result"] = True actionResult["rc"] = 0 return actionResult
def run(self, data, persistentData, actionResult): import pyodbc host = helpers.evalString(self.host, {"data": data}) username = helpers.evalString(self.username, {"data": data}) connectionDetails = helpers.evalDict(self.connectionDetails, {"data": data}) dbType = helpers.evalString(self.dbType, {"data": data}) search = helpers.evalString(self.search, {"data": data}) password = auth.getPasswordFromENC(self.password) timeout = 30 if self.timeout != 0: timeout = self.timeout actionResult["result"] = False actionResult["rc"] = 404 actionResult["msg"] = "could not connect" dbControl = database.control(dbType, host, username, password, connectionDetails, timeout) if dbControl.isConnected(): rc, results = dbControl.query(search, self.limit) if rc: if self.limit == 1: if self.count: actionResult["count"] = 1 else: actionResult["row"] = results[0] else: if self.count: actionResult["count"] = len(results) else: actionResult["rows"] = results actionResult["rc"] = 0 actionResult["msg"] = "success" actionResult["result"] = True else: actionResult["msg"] = results else: actionResult["msg"] = dbControl.message return actionResult
def run(self, data, persistentData, actionResult): occurrenceIndex = helpers.evalString(self.occurrenceIndex, {"data": data}) occurrenceVarData = helpers.evalDict(self.occurrenceVarData, {"data": data}) try: currentOccurrence = persistentData["plugin"]["occurrence"][ occurrenceIndex] if self.updateMode == 0: for key, value in occurrenceVarData.items(): currentOccurrence.data["var"][key] = value elif self.updateMode == 1: currentOccurrence.data["var"] = occurrenceVarData currentOccurrence.update(["data"]) actionResult["result"] = True actionResult["rc"] = 0 except KeyError: actionResult["result"] = False actionResult["rc"] = 404 actionResult["msg"] = "No occurrence found within current flow" return actionResult
def run(self, data, persistentData, actionResult): if not hasattr(self, "plain_humioAPIToken"): self.plain_humio_ingest_token = auth.getPasswordFromENC( self.humio_ingest_token) # Get data dict if len(self.custom_data) > 0: dataToSend = helpers.evalDict(self.custom_data, {"data": data}) else: if len(self.field) > 0: dataToSend = helpers.getDictValue(self.field[0], {"data": data}) else: dataToSend = data # Apply flatten if self.flatten_field: for key, value in data[self.flatten_field].items(): dataToSend[key] = value del dataToSend[self.flatten_field] # Send events if type(dataToSend) is list: events = [] for entry in dataToSend: events.append(self.buildEvents(entry)) if not self.shippingHandlerBulk(events): actionResult["result"] = False actionResult["rc"] = 1 return actionResult elif type(dataToSend) is dict: if not self.shippingHandler(dataToSend): actionResult["result"] = False actionResult["rc"] = 2 return actionResult actionResult["result"] = True actionResult["rc"] = 0 return actionResult
def run(self, data, persistentData, actionResult): assetName = helpers.evalString(self.assetName, {"data": data}) assetType = helpers.evalString(self.assetType, {"data": data}) assetEntity = helpers.evalString(self.assetEntity, {"data": data}) updateSource = helpers.evalString(self.updateSource, {"data": data}) updateTime = helpers.evalString(self.updateTime, {"data": data}) assetFields = helpers.evalDict(self.assetFields, {"data": data}) if not assetName or not assetType or not updateSource or not assetFields: actionResult["result"] = False actionResult["msg"] = "Missing required properties" actionResult["rc"] = 403 return actionResult match = "{0}-{1}-{2}".format(assetName, assetType, assetEntity) assetItem = cache.globalCache.get("assetCache", match, getAssetObject, assetName, assetType, assetEntity) if not assetItem: assetItem = asset._asset().bulkNew(self.acl, assetName, assetEntity, assetType, updateSource, assetFields, updateTime, self.sourcePriority, self.bulkClass) cache.globalCache.insert("assetCache", match, [assetItem]) actionResult["result"] = True actionResult["msg"] = "Created new asset" actionResult["rc"] = 201 return actionResult assetChanged = False # Removing entires if more than one result is found if len(assetItem) > 1: newestItem = assetItem[0] for singleAssetItem in assetItem: if newestItem.lastUpdateTime < singleAssetItem.lastUpdateTime: singleAssetItem.delete() else: newestItem.delete() newestItem = singleAssetItem assetItem = newestItem cache.globalCache.update("assetCache", match, [assetItem]) assetChanged = True else: assetItem = assetItem[0] if assetItem._id == "": actionResult["result"] = False actionResult["msg"] = "Asset not yet added" actionResult["rc"] = 404 return actionResult lastSeen = None for source in assetItem.lastSeen: if source["source"] == updateSource: if "lastUpdate" not in lastSeen: lastSeen["lastUpdate"] = 0 lastSeen = source break if not lastSeen: assetItem.lastSeen.append({ "source": updateSource, "lastUpdate": 0 }) lastSeen = assetItem.lastSeen[-1] # Converting millsecond int epoch into epoch floats currentTimestamp = lastSeen["lastUpdate"] if len(str(currentTimestamp).split(".")[0]) == 13: currentTimestamp = currentTimestamp / 1000 if len(str(updateTime).split(".")[0]) == 13: updateTime = updateTime / 1000 newTimestamp = None if updateTime: try: if updateTime < currentTimestamp: newTimestamp = False else: newTimestamp = updateTime except (KeyError, ValueError): pass if newTimestamp == None: newTimestamp = time.time() if newTimestamp != False: try: if (time.time() - currentTimestamp) < self.delayedUpdate: actionResult["result"] = False actionResult["msg"] = "Delay time not met" actionResult["rc"] = 300 return actionResult except KeyError: pass assetChanged = True if newTimestamp > assetItem.lastSeenTimestamp: assetItem.lastSeenTimestamp = newTimestamp if self.replaceExisting: lastSeen = assetFields else: for key, value in assetFields.items(): lastSeen[key] = value lastSeen["priority"] = self.sourcePriority lastSeen["lastUpdate"] = newTimestamp # Working out priority and define fields if assetChanged: foundValues = {} now = time.time() blacklist = ["lastUpdate", "priority"] for sourceValue in assetItem.lastSeen: for key, value in sourceValue.items(): if key not in blacklist: if key not in foundValues: foundValues[key] = { "value": value, "priority": sourceValue["priority"] } else: if sourceValue["priority"] < foundValues[key][ "priority"] and ( sourceValue["lastUpdate"] + self.sourcePriorityMaxAge) > now: foundValues[key] = { "value": value, "priority": sourceValue["priority"] } assetItem.fields = {} for key, value in foundValues.items(): assetItem.fields[key] = value["value"] assetItem.bulkUpdate(["lastSeen", "fields"], self.bulkClass) actionResult["result"] = True actionResult["msg"] = "Updated asset" actionResult["rc"] = 302 return actionResult actionResult["result"] = True actionResult["msg"] = "Nothing to do" actionResult["rc"] = 304 return actionResult
def run(self,data,persistentData,actionResult): eventTitle = helpers.evalString(self.eventTitle,{"data" : data}) eventType = helpers.evalString(self.eventType,{"data" : data}) eventSubType = helpers.evalString(self.eventSubType,{"data" : data}) layer = self.layer accuracy = self.accuracy impact = self.impact benign = self.benign timeToLive = self.timeToLive uid = helpers.evalString(self.uid,{"data" : data}) eventValues = helpers.evalDict(self.eventValues,{"data" : data}) uid = "{0}-{1}-{2}-{3}".format(self._id,eventType,eventSubType,uid) data["var"]["event"] = {} data["var"]["event"]["type"] = eventType data["var"]["event"]["eventSubType"] = eventSubType data["var"]["event"]["layer"] = layer data["var"]["event"]["accuracy"] = accuracy data["var"]["event"]["impact"] = impact data["var"]["event"]["benign"] = benign try: score = ((accuracy*(impact*layer))/benign) except ZeroDivisionError: score = 0 data["var"]["event"]["score"] = score cacheUID = "{0}-{1}-{2}".format(data["conductID"],data["flow_id"],uid) foundEvent = cache.globalCache.get("eventCache",cacheUID,getEvent,data["conductID"],data["flow_id"],uid,eventType,eventSubType,extendCacheTime=True,customCacheTime=timeToLive,nullUpdate=True) if foundEvent != None: try: persistentData["plugin"]["event"].append(foundEvent) except: persistentData["plugin"]["event"] = [foundEvent] arrayIndex = len(persistentData["plugin"]["event"])-1 actionResult["eventIndex"] = arrayIndex if foundEvent._id != "": if foundEvent.expiryTime > time.time(): changes = False for key,value in eventValues.items(): if key in foundEvent.eventValues: if value != foundEvent.eventValues[key]: changes = True break else: changes = True break if changes: foundEvent.updateRecord(self.bulkClass,eventValues,accuracy,impact,layer,benign,score,int( time.time() + timeToLive ),self.history) actionResult["result"] = True actionResult["rc"] = 202 return actionResult else: foundEvent.expiryTime = int(time.time() + timeToLive) foundEvent.bulkUpdate(["expiryTime"],self.bulkClass) actionResult["result"] = True actionResult["rc"] = 302 return actionResult else: cache.globalCache.delete("eventCache",cacheUID) else: logging.debug("Event Update Failed - NO ID, actionID='{0}'".format(self._id),7) actionResult["result"] = False actionResult["rc"] = 500 return actionResult eventObject = event._event().bulkNew(self.bulkClass,self.acl,data["conductID"],data["flow_id"],eventType,eventSubType,int( time.time() + timeToLive ),eventValues,uid,accuracy,impact,layer,benign,score,data,eventTitle) cache.globalCache.insert("eventCache",cacheUID,eventObject,customCacheTime=timeToLive) try: persistentData["plugin"]["event"].append(eventObject) except: persistentData["plugin"]["event"] = [eventObject] arrayIndex = len(persistentData["plugin"]["event"])-1 actionResult["eventIndex"] = arrayIndex actionResult["result"] = True actionResult["rc"] = 201 return actionResult
def run(self,data,persistentData,actionResult): correlationName = helpers.evalString(self.correlationName,{"data" : data}) excludeCorrelationValues = helpers.evalDict(self.excludeCorrelationValues,{"data" : data}) expiryTime = time.time() + self.expiryTime correlatedRelationships = event._eventCorrelation().getAsClass(query={ "correlationName" : correlationName, "expiryTime" : { "$gt" : int(time.time()) } }) eventsAfterTime = int(time.time()) - self.oldestEvent if not self.alwaysProcessEvents: ids = event._eventCorrelation()._dbCollection.distinct("ids",{ "$or" : [ { "expiryTime" : { "$gt" : int(time.time()) } }, { "lastUpdateTime" : { "$gt" : eventsAfterTime } } ] }) objectIds = [] for idItem in ids: objectIds.append(db.ObjectId(idItem)) events = event._event().getAsClass(query={ "_id" : { "$nin" : objectIds }, "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } }) else: events = event._event().getAsClass(query={ "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } }) # Build correlation field hash table fields = {} for field in self.correlationFields: fields[field] = {} if field not in excludeCorrelationValues: excludeCorrelationValues[field] = [] for correlatedRelationshipItem in correlatedRelationships: for field in self.correlationFields: try: if field not in excludeCorrelationValues: excludeCorrelationValues[field] = [] for value in correlatedRelationshipItem.correlations[field]: fields[field][value] = correlatedRelationshipItem except KeyError: pass correlatedRelationshipsCreated = [] correlatedRelationshipsUpdated = [] correlatedRelationshipsDeleted = [] # Initial Pass Loop for eventItem in events: foundCorrelatedRelationship = None correlations = {} processNew = False for eventField in eventItem.eventValues: try: if type(eventItem.eventValues[eventField]) is list: correlations[eventField] = [ x for x in eventItem.eventValues[eventField] if eventField in self.correlationFields ] matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x in fields[eventField] and x not in excludeCorrelationValues[eventField] ] if len(matchFound) > 0: foundCorrelatedRelationship = matchFound[0] else: matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x and x not in excludeCorrelationValues[eventField] ] if len(matchFound) > 0: processNew = True else: correlations[eventField] = [eventItem.eventValues[eventField]] if eventField in self.correlationFields and eventItem.eventValues[eventField] in fields[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]: foundCorrelatedRelationship = fields[eventField][eventItem.eventValues[eventField]] else: if eventField in self.correlationFields and eventItem.eventValues[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]: processNew = True except KeyError: pass # Create new if processNew == True: newEventCorrelation = event._eventCorrelation() newEventCorrelation.bulkNew(self.bulkClass, self.acl, correlationName,expiryTime,[eventItem._id],[eventItem.eventType],[eventItem.eventSubType],correlations,eventItem.score) correlatedRelationshipsCreated.append(newEventCorrelation) correlatedRelationships.append(newEventCorrelation) for eventField in eventItem.eventValues: try: for eventValue in correlations[eventField]: try: fields[eventField][eventValue] = newEventCorrelation except KeyError: fields[eventField] = { eventValue : newEventCorrelation } except KeyError: pass # Merge existing elif foundCorrelatedRelationship != None: for eventField in correlations: try: foundCorrelatedRelationship.correlations[eventField] += correlations[eventField] foundCorrelatedRelationship.correlations[eventField] = list(set(foundCorrelatedRelationship.correlations[eventField])) except KeyError: foundCorrelatedRelationship.correlations[eventField] = correlations[eventField] if eventItem._id not in foundCorrelatedRelationship.ids: foundCorrelatedRelationship.ids.append(eventItem._id) foundCorrelatedRelationship.score += eventItem.score if eventItem.eventType not in foundCorrelatedRelationship.types: foundCorrelatedRelationship.types.append(eventItem.eventType) if eventItem.eventSubType not in foundCorrelatedRelationship.subTypes: foundCorrelatedRelationship.subTypes.append(eventItem.eventSubType) foundCorrelatedRelationship.correlationLastUpdate = int(time.time()) foundCorrelatedRelationship.expiryTime = expiryTime if foundCorrelatedRelationship not in correlatedRelationshipsCreated and foundCorrelatedRelationship not in correlatedRelationshipsUpdated: correlatedRelationshipsUpdated.append(foundCorrelatedRelationship) # Process bulk creation if needed before merging self.bulkClass.bulkOperatonProcessing() # Reduction Loop loop = 1 maxLoops = 5 while loop > 0 and maxLoops > 0: correlatedFieldsHash = {} for correlatedRelationship in correlatedRelationships: for eventField, eventValue in ((eventField, eventValue) for eventField in correlatedRelationship.correlations for eventValue in correlatedRelationship.correlations[eventField] ): if eventField in self.correlationFields and eventValue not in excludeCorrelationValues[eventField]: try: if eventValue not in correlatedFieldsHash[eventField]: correlatedFieldsHash[eventField][eventValue] = correlatedRelationship else: currentCorrelation = correlatedFieldsHash[eventField][eventValue] for eventField in correlatedRelationship.correlations: try: currentCorrelation.correlations[eventField] += correlatedRelationship.correlations[eventField] currentCorrelation.correlations[eventField] = list(set(currentCorrelation.correlations[eventField])) except KeyError: currentCorrelation.correlations[eventField] = correlatedRelationship.correlations[eventField] for mergeKey in ["ids","types","subTypes"]: for value in getattr(correlatedRelationship,mergeKey): if value not in getattr(currentCorrelation,mergeKey): getattr(currentCorrelation,mergeKey).append(value) currentCorrelation.score += correlatedRelationship.score currentCorrelation.correlationLastUpdate = int(time.time()) currentCorrelation.expiryTime = expiryTime if currentCorrelation not in correlatedRelationshipsCreated and correlatedRelationship not in correlatedRelationshipsUpdated: correlatedRelationshipsUpdated.append(currentCorrelation) # Deleting the eventCorrelation it was merged with if correlatedRelationship not in correlatedRelationshipsDeleted: correlatedRelationshipsDeleted.append(correlatedRelationship) if correlatedRelationship in correlatedRelationshipsUpdated: correlatedRelationshipsUpdated.remove(correlatedRelationship) if correlatedRelationship in correlatedRelationshipsCreated: correlatedRelationshipsCreated.remove(correlatedRelationship) correlatedRelationship.bulkMerge(currentCorrelation._id,self.bulkClass) correlatedRelationships.remove(correlatedRelationship) loop+=1 break except KeyError: correlatedFieldsHash[eventField] = { eventValue : correlatedRelationship } maxLoops -= 1 loop -= 1 created = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsCreated ] updated = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsUpdated ] deleted = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsDeleted ] for correlatedRelationshipUpdated in correlatedRelationshipsUpdated: correlatedRelationshipUpdated.bulkUpdate(["expiryTime","ids","types","subTypes","correlations","score"],self.bulkClass) updated.append(helpers.classToJson(correlatedRelationshipUpdated,hidden=True)) self.bulkClass.bulkOperatonProcessing() actionResult["result"] = True actionResult["rc"] = 0 actionResult["correlatedEvents"] = { "created" : created, "updated" : updated, "deleted" : deleted } return actionResult
def run(self, data, persistentData, actionResult): assetType = helpers.evalString(self.assetType, {"data": data}) assetEntity = helpers.evalString(self.assetEntity, {"data": data}) updateSource = helpers.evalString(self.updateSource, {"data": data}) updateTime = helpers.evalString(self.updateTime, {"data": data}) assetData = helpers.evalDict(self.assetData, {"data": data}) existingAssets = asset._asset().getAsClass( query={ "name": { "$in": list(assetData.keys()) }, "assetType": assetType, "entity": assetEntity }) # Updating existing for assetItem in existingAssets: lastSeen = None for source in assetItem.lastSeen: if source["source"] == updateSource: if "lastUpdate" not in lastSeen: lastSeen["lastUpdate"] = 0 lastSeen = source break if not lastSeen: assetItem.lastSeen.append({ "source": updateSource, "lastUpdate": 0 }) lastSeen = assetItem.lastSeen[-1] # Converting millsecond int epoch into epoch floats currentTimestamp = lastSeen["lastUpdate"] if len(str(currentTimestamp).split(".")[0]) == 13: currentTimestamp = currentTimestamp / 1000 updateTime = assetData[assetItem.name]["lastUpdate"] if len(str(updateTime).split(".")[0]) == 13: updateTime = updateTime / 1000 newTimestamp = None if updateTime: try: if updateTime < currentTimestamp: newTimestamp = False else: newTimestamp = updateTime except (KeyError, ValueError): pass if newTimestamp == None: newTimestamp = time.time() assetChanged = False if newTimestamp != False: try: if (time.time() - currentTimestamp) < self.delayedUpdate: continue except KeyError: pass assetChanged = True if newTimestamp > assetItem.lastSeenTimestamp: assetItem.lastSeenTimestamp = newTimestamp if self.replaceExisting: lastSeen = assetData[assetItem.name] else: for key, value in assetData[assetItem.name].items(): lastSeen[key] = value lastSeen["priority"] = self.sourcePriority lastSeen["lastUpdate"] = newTimestamp # Working out priority and define fields if assetChanged: foundValues = {} now = time.time() blacklist = ["lastUpdate", "priority"] for sourceValue in assetItem.lastSeen: for key, value in sourceValue.items(): if key not in blacklist: if key not in foundValues: foundValues[key] = { "value": value, "priority": sourceValue["priority"] } else: if sourceValue["priority"] < foundValues[key][ "priority"] and ( sourceValue["lastUpdate"] + self.sourcePriorityMaxAge) > now: foundValues[key] = { "value": value, "priority": sourceValue["priority"] } assetItem.fields = {} for key, value in foundValues.items(): assetItem.fields[key] = value["value"] assetItem.bulkUpdate(["lastSeen", "fields"], self.bulkClass) del assetData[assetItem.name] # Adding new for assetName, assetFields in assetData.items(): assetItem = asset._asset().bulkNew(self.acl, assetName, assetEntity, assetType, updateSource, assetFields, updateTime, self.sourcePriority, self.bulkClass) actionResult["result"] = True actionResult["rc"] = 0 return actionResult