def run(self,data,persistentData,actionResult):
headers = helpers.evalDict(self.headers,{"data" : data})
url = helpers.evalString(self.url,{"data" : data})
method = helpers.evalString(self.method,{"data" : data}).upper()
postData = helpers.evalString(self.postData,{"data" : data})
kwargs={}
kwargs["headers"] = headers
if self.timeout == 0:
kwargs["timeout"] = 5
else:
kwargs["timeout"] = self.timeout
if self.ca:
kwargs["verify"] = Path(self.ca)
if self.proxy:
kwargs["proxies"] = self.proxy
else:
kwargs["proxies"] = { "http": None, "https": None }
if self.cookies:
cookies = helpers.evalDict(self.cookies,{"data" : data})
kwargs["cookies"] = cookies
try:
if method == "GET":
response = requests.get(url,**kwargs)
elif method == "POST":
kwargs["data"] = postData
response = requests.post(url,**kwargs)
elif method == "PUT":
kwargs["data"] = postData
response = requests.put(url,**kwargs)
elif method == "DELETE":
response = requests.delete(url,**kwargs)
text = response.text
if self.returnsJson:
try:
text = response.json()
except:
text = response.text
actionResult["result"] = True
actionResult["rc"] = response.status_code
actionResult["data"] = { "headers" : dict(response.headers), "text" : text }
except:
actionResult["result"] = False
actionResult["rc"] = 404
return actionResult
def doAction(self, data):
activityData = helpers.evalDict(self.activityData,
{"data": data["flowData"]})
conversationId = helpers.evalString(self.conversationId,
{"data": data["flowData"]})
activityId = helpers.evalString(self.activityId,
{"data": data["flowData"]})
if not self.client_id:
azureBotTrigger = data["persistentData"]["system"]["trigger"]
if not hasattr(azureBotTrigger, "azurebotservice"):
azureBotTrigger.azurebotservice = azurebotservice._azurebotservice(
clientId=azureBotTrigger.client_id,
clientSecret=jimi.auth.getPasswordFromENC(
azureBotTrigger.client_secret),
serviceUrl=data["flowData"]["event"]["serviceUrl"])
else:
azureBotTrigger.azurebotservice = azurebotservice._azurebotservice(
clientId=self.client_id,
clientSecret=jimi.auth.getPasswordFromENC(self.client_secret),
serviceUrl=self.service_url)
result = azureBotTrigger.azurebotservice.updateActivity(
activityData, conversationId, activityId)
if result:
return {"result": True, "rc": 0, "msg": "Activity updated."}
else:
return {
"result": False,
"rc": 400,
"msg": "Invalid response from web service, check your url."
}
def run(self, data, persistentData, actionResult):
relationshipData = helpers.evalDict(self.relationshipData,
{"data": data})
events = helpers.evalString(self.eventField, {"data": data})
timespan = 3600
if self.timespan > 0:
timespan = self.timespan
timespan = helpers.roundTime(roundTo=timespan).timestamp()
matches = {}
for event in events:
matches["{0}->{1}".format(event[self.fromAssetField],
event[self.toAssetField])] = event
results = _assetRelationship().query(query={"timespan": timespan},
fields=["fromAsset",
"toAsset"])["results"]
for result in results:
match = "{0}->{1}".format(result["fromAsset"], result["toAsset"])
if match in matches:
del matches[match]
for match in matches:
_assetRelationship().bulkNew(self.acl, timespan,
matches[match][self.fromAssetField],
matches[match][self.toAssetField],
relationshipData, self.bulkClass)
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
def run(self,data,persistentData,actionResult):
matches = helpers.evalDict(self.matches,{ "data" : data })
lastSeenWithin = self.lastSeenWithin
if lastSeenWithin == 0:
lastSeenWithin = 86400
ors = {}
finds = {}
for matchItem, matchValue in matches.items():
if type(matchValue) is list:
ors[matchItem] = matchValue
else:
finds[matchItem] = matchValue
search = {}
search["$or"] = []
for matchItem, matchValue in matches.items():
if type(matchValue) is list:
for matchValueItem in matchValue:
if matchValueItem != None:
search["$or"].append( { matchItem : matchValueItem })
else:
if matchValue != None:
search["$or"].append( { matchItem : matchValue })
search["lastSeenTimestamp"] = { "$gt" : time.time() - lastSeenWithin }
assetsList = asset._asset().query(query=search)["results"]
actionResult["assets"] = assetsList
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
def run(self, data, persistentData, actionResult):
relationshipData = helpers.evalDict(self.relationshipData,
{"data": data})
fromAsset = helpers.evalString(self.fromAsset, {"data": data})
toAsset = helpers.evalString(self.toAsset, {"data": data})
timespan = 3600
if self.timespan > 0:
timespan = self.timespan
timespan = helpers.roundTime(roundTo=timespan).timestamp()
match = "{0}={1}->{2}".format(timespan, fromAsset, toAsset)
# This will cause a memory leak overtime as globalCache is not cleared down --- this needs to be fixed
relationship = cache.globalCache.get("assetRelationshipCache",
match,
getAssetRelationshipObject,
timespan,
fromAsset,
toAsset,
extendCacheTime=True)
if not relationship:
relationshipItem = _assetRelationship().bulkNew(
self.acl, timespan, fromAsset, toAsset, relationshipData,
self.bulkClass)
if relationshipItem:
cache.globalCache.insert("assetRelationshipCache", match,
[relationshipItem])
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
def check(self):
latestTime = 0
search = helpers.evalDict(self.search, {"data": {}})
match = ""
for key, value in search.items():
match += key
if type(value) == str:
match += value
if not self.cache:
# cache does not support nested dicts as these will always be seen as the same
assetList = cache.globalCache.get("assetSearchCache", match,
getSearchObject, search,
self.fields)
else:
assetList = asset._asset().query(query=search,
fields=self.fields)["results"]
if assetList is not None:
for assetItem in assetList:
if len(self.flattenFields) > 0:
for flattenField in self.flattenFields:
if flattenField in assetItem["fields"]:
assetItem[flattenField] = assetItem["fields"][
flattenField]
del assetItem["fields"][flattenField]
if self.return_one:
if "event" in self.result:
if self.result["event"]["lastUpdateTime"] > latestTime:
self.result["event"] = assetItem
else:
self.result["event"] = asset
else:
self.result["events"].append(asset)
def doAction(self, data):
actionResult = {}
scanName = helpers.evalString(self.scanName,
{"data": data["flowData"]})
customSearch = helpers.evalDict(self.customSearch,
{"data": data["flowData"]})
if scanName != "":
search = {"scanName": scanName}
else:
search = {}
if customSearch:
for key, value in customSearch.items():
search[key] = value
actionResult["result"] = True
actionResult["rc"] = 0
if self.limit > 0:
actionResult["events"] = inga._inga().query(
query=search,
limit=self.limit,
sort=[("ports.scanDetails.lastPortScan", 1)])["results"]
else:
actionResult["events"] = inga._inga().query(
query=search,
sort=[("ports.scanDetails.lastPortScan", 1)])["results"]
return actionResult
def run(self, data, persistentData, actionResult):
eventIndex = helpers.evalString(self.eventIndex, {"data": data})
eventValues = helpers.evalDict(self.eventValues, {"data": data})
try:
currentEvent = persistentData["plugin"]["event"][eventIndex]
if self.updateMode == 0:
for key, value in eventValues.items():
currentEvent.eventValues[key] = value
elif self.updateMode == 1:
currentEvent.eventValues = eventValues
elif self.updateMode == 2:
for key, value in eventValues.items():
if value:
if key in currentEvent.eventValues:
if type(currentEvent.eventValues[key]) != list:
currentEvent.eventValues[key] = [
currentEvent.eventValues[key]
]
if type(value) is list:
currentEvent.eventValues[key] += value
else:
currentEvent.eventValues[key].append(value)
else:
currentEvent.eventValues[key] = value
currentEvent.eventFields = list(currentEvent.eventValues.keys())
currentEvent.update(["eventValues", "eventFields"])
actionResult["result"] = True
actionResult["rc"] = 0
except KeyError:
actionResult["result"] = False
actionResult["rc"] = 404
actionResult["msg"] = "No event found within current flow"
return actionResult
def varEval(varDict,currentVarDict,dicts={}):
functionSafeList = function.systemFunctions
for key, value in varDict.items():
logicResult = True
if "if" in value:
logicResult = logic.ifEval(value["if"],dicts)
if logicResult:
# Checking that supplied dictionary contains key value
if "value" in value:
if type(value["value"]) is str:
currentVarDict[key] = helpers.evalString(value["value"],dicts,functionSafeList)
elif type(value["value"]) is dict:
if key in currentVarDict:
currentVarDict[key].update(helpers.evalDict(value["value"],dicts,functionSafeList))
else:
currentVarDict[key] = helpers.evalDict(value["value"],dicts,functionSafeList)
else:
currentVarDict[key] = value["value"]
return currentVarDict
def doAction(self, data):
try:
if "skip" in data["flowData"]["plugin"]["azurebotservice"]:
conversationData = []
currentConversation = conversation._azurebotserviceconversation(
).getAsClass(
query={
"conversationId":
data["flowData"]["event"]["conversation"]["id"],
"conductId":
data["flowData"]["conductID"]
})
if len(currentConversation) > 0:
currentConversation = currentConversation[0]
conversationData = currentConversation.conversationData
currentConversation.conductId = None
currentConversation.flowId = None
currentConversation.update(["conductId", "flowId"])
del data["flowData"]["plugin"]["azurebotservice"]["skip"]
return {
"result": True,
"rc": 0,
"msg": "Conversation resumed.",
"conversationData": conversationData
}
except KeyError:
pass
preserveData = helpers.evalDict(self.preserveData,
{"data": data["flowData"]})
currentConversation = conversation._azurebotserviceconversation(
).getAsClass(
query={
"conversationId": data["flowData"]["event"]["conversation"]
["id"],
"conductId": data["flowData"]["conductID"]
})
if len(currentConversation) > 0:
currentConversation = currentConversation[0]
currentConversation.updateConversation(
data["flowData"]["conductID"], data["flowData"]["flow_id"],
self.waitFor, preserveData)
else:
conversation._azurebotserviceconversation().new(
self.acl, data["flowData"]["event"]["conversation"]["id"],
data["flowData"]["conductID"], data["flowData"]["flow_id"],
self.waitFor, preserveData)
return {"result": False, "rc": 200, "msg": "Conversation saved."}
def run(self,data,persistentData,actionResult):
eventIndex = helpers.evalString(self.eventIndex,{"data" : data})
eventValues = helpers.evalDict(self.eventValues,{"data" : data})
try:
currentEvent = persistentData["plugin"]["event"][eventIndex]
if self.updateMode == 0:
for key, value in eventValues.items():
currentEvent.eventValues[key] = value
elif self.updateMode == 1:
currentEvent.eventValues = eventValues
currentEvent.update(["eventValues"])
actionResult["result"] = True
actionResult["rc"] = 0
except KeyError:
actionResult["result"] = False
actionResult["rc"] = 404
actionResult["msg"] = "No event found within current flow"
return actionResult
def run(self,data,persistentData,actionResult):
latestTime = 0
search = helpers.evalDict(self.search,{"data" : data})
order_by = helpers.evalString(self.order_by,{"data" : data})
match = ""
for key, value in search.items():
match += key
if type(value) == str:
match += value
sort = None
if order_by != "":
if self.ascending:
sort = [(order_by, 1)]
else:
sort = [(order_by, -1)]
if not self.cache:
# cache does not support nested dicts as these will always be seen as the same
assetList = cache.globalCache.get("assetSearchCache",match,getSearchObject,search,self.fields,sort)
else:
assetList = asset._asset().query(query=search,fields=self.fields,sort=sort)["results"]
if self.return_one:
actionResult["event"] = ""
else:
actionResult["events"] = []
if assetList is not None:
for assetItem in assetList:
if len(self.flattenFields) > 0:
for flattenField in self.flattenFields:
if flattenField in assetItem["fields"]:
assetItem[flattenField] = assetItem["fields"][flattenField]
del assetItem["fields"][flattenField]
if self.return_one:
if assetItem["lastUpdateTime"] > latestTime:
actionResult["event"] = assetItem
else:
actionResult["events"].append(assetItem)
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
def run(self, data, persistentData, actionResult):
import pyodbc
host = helpers.evalString(self.host, {"data": data})
username = helpers.evalString(self.username, {"data": data})
connectionDetails = helpers.evalDict(self.connectionDetails,
{"data": data})
dbType = helpers.evalString(self.dbType, {"data": data})
search = helpers.evalString(self.search, {"data": data})
password = auth.getPasswordFromENC(self.password)
timeout = 30
if self.timeout != 0:
timeout = self.timeout
actionResult["result"] = False
actionResult["rc"] = 404
actionResult["msg"] = "could not connect"
dbControl = database.control(dbType, host, username, password,
connectionDetails, timeout)
if dbControl.isConnected():
rc, results = dbControl.query(search, self.limit)
if rc:
if self.limit == 1:
if self.count:
actionResult["count"] = 1
else:
actionResult["row"] = results[0]
else:
if self.count:
actionResult["count"] = len(results)
else:
actionResult["rows"] = results
actionResult["rc"] = 0
actionResult["msg"] = "success"
actionResult["result"] = True
else:
actionResult["msg"] = results
else:
actionResult["msg"] = dbControl.message
return actionResult
def run(self, data, persistentData, actionResult):
occurrenceIndex = helpers.evalString(self.occurrenceIndex,
{"data": data})
occurrenceVarData = helpers.evalDict(self.occurrenceVarData,
{"data": data})
try:
currentOccurrence = persistentData["plugin"]["occurrence"][
occurrenceIndex]
if self.updateMode == 0:
for key, value in occurrenceVarData.items():
currentOccurrence.data["var"][key] = value
elif self.updateMode == 1:
currentOccurrence.data["var"] = occurrenceVarData
currentOccurrence.update(["data"])
actionResult["result"] = True
actionResult["rc"] = 0
except KeyError:
actionResult["result"] = False
actionResult["rc"] = 404
actionResult["msg"] = "No occurrence found within current flow"
return actionResult
def run(self, data, persistentData, actionResult):
if not hasattr(self, "plain_humioAPIToken"):
self.plain_humio_ingest_token = auth.getPasswordFromENC(
self.humio_ingest_token)
# Get data dict
if len(self.custom_data) > 0:
dataToSend = helpers.evalDict(self.custom_data, {"data": data})
else:
if len(self.field) > 0:
dataToSend = helpers.getDictValue(self.field[0],
{"data": data})
else:
dataToSend = data
# Apply flatten
if self.flatten_field:
for key, value in data[self.flatten_field].items():
dataToSend[key] = value
del dataToSend[self.flatten_field]
# Send events
if type(dataToSend) is list:
events = []
for entry in dataToSend:
events.append(self.buildEvents(entry))
if not self.shippingHandlerBulk(events):
actionResult["result"] = False
actionResult["rc"] = 1
return actionResult
elif type(dataToSend) is dict:
if not self.shippingHandler(dataToSend):
actionResult["result"] = False
actionResult["rc"] = 2
return actionResult
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
def run(self, data, persistentData, actionResult):
assetName = helpers.evalString(self.assetName, {"data": data})
assetType = helpers.evalString(self.assetType, {"data": data})
assetEntity = helpers.evalString(self.assetEntity, {"data": data})
updateSource = helpers.evalString(self.updateSource, {"data": data})
updateTime = helpers.evalString(self.updateTime, {"data": data})
assetFields = helpers.evalDict(self.assetFields, {"data": data})
if not assetName or not assetType or not updateSource or not assetFields:
actionResult["result"] = False
actionResult["msg"] = "Missing required properties"
actionResult["rc"] = 403
return actionResult
match = "{0}-{1}-{2}".format(assetName, assetType, assetEntity)
assetItem = cache.globalCache.get("assetCache", match, getAssetObject,
assetName, assetType, assetEntity)
if not assetItem:
assetItem = asset._asset().bulkNew(self.acl, assetName,
assetEntity, assetType,
updateSource, assetFields,
updateTime, self.sourcePriority,
self.bulkClass)
cache.globalCache.insert("assetCache", match, [assetItem])
actionResult["result"] = True
actionResult["msg"] = "Created new asset"
actionResult["rc"] = 201
return actionResult
assetChanged = False
# Removing entires if more than one result is found
if len(assetItem) > 1:
newestItem = assetItem[0]
for singleAssetItem in assetItem:
if newestItem.lastUpdateTime < singleAssetItem.lastUpdateTime:
singleAssetItem.delete()
else:
newestItem.delete()
newestItem = singleAssetItem
assetItem = newestItem
cache.globalCache.update("assetCache", match, [assetItem])
assetChanged = True
else:
assetItem = assetItem[0]
if assetItem._id == "":
actionResult["result"] = False
actionResult["msg"] = "Asset not yet added"
actionResult["rc"] = 404
return actionResult
lastSeen = None
for source in assetItem.lastSeen:
if source["source"] == updateSource:
if "lastUpdate" not in lastSeen:
lastSeen["lastUpdate"] = 0
lastSeen = source
break
if not lastSeen:
assetItem.lastSeen.append({
"source": updateSource,
"lastUpdate": 0
})
lastSeen = assetItem.lastSeen[-1]
# Converting millsecond int epoch into epoch floats
currentTimestamp = lastSeen["lastUpdate"]
if len(str(currentTimestamp).split(".")[0]) == 13:
currentTimestamp = currentTimestamp / 1000
if len(str(updateTime).split(".")[0]) == 13:
updateTime = updateTime / 1000
newTimestamp = None
if updateTime:
try:
if updateTime < currentTimestamp:
newTimestamp = False
else:
newTimestamp = updateTime
except (KeyError, ValueError):
pass
if newTimestamp == None:
newTimestamp = time.time()
if newTimestamp != False:
try:
if (time.time() - currentTimestamp) < self.delayedUpdate:
actionResult["result"] = False
actionResult["msg"] = "Delay time not met"
actionResult["rc"] = 300
return actionResult
except KeyError:
pass
assetChanged = True
if newTimestamp > assetItem.lastSeenTimestamp:
assetItem.lastSeenTimestamp = newTimestamp
if self.replaceExisting:
lastSeen = assetFields
else:
for key, value in assetFields.items():
lastSeen[key] = value
lastSeen["priority"] = self.sourcePriority
lastSeen["lastUpdate"] = newTimestamp
# Working out priority and define fields
if assetChanged:
foundValues = {}
now = time.time()
blacklist = ["lastUpdate", "priority"]
for sourceValue in assetItem.lastSeen:
for key, value in sourceValue.items():
if key not in blacklist:
if key not in foundValues:
foundValues[key] = {
"value": value,
"priority": sourceValue["priority"]
}
else:
if sourceValue["priority"] < foundValues[key][
"priority"] and (
sourceValue["lastUpdate"] +
self.sourcePriorityMaxAge) > now:
foundValues[key] = {
"value": value,
"priority": sourceValue["priority"]
}
assetItem.fields = {}
for key, value in foundValues.items():
assetItem.fields[key] = value["value"]
assetItem.bulkUpdate(["lastSeen", "fields"], self.bulkClass)
actionResult["result"] = True
actionResult["msg"] = "Updated asset"
actionResult["rc"] = 302
return actionResult
actionResult["result"] = True
actionResult["msg"] = "Nothing to do"
actionResult["rc"] = 304
return actionResult
def run(self,data,persistentData,actionResult):
eventTitle = helpers.evalString(self.eventTitle,{"data" : data})
eventType = helpers.evalString(self.eventType,{"data" : data})
eventSubType = helpers.evalString(self.eventSubType,{"data" : data})
layer = self.layer
accuracy = self.accuracy
impact = self.impact
benign = self.benign
timeToLive = self.timeToLive
uid = helpers.evalString(self.uid,{"data" : data})
eventValues = helpers.evalDict(self.eventValues,{"data" : data})
uid = "{0}-{1}-{2}-{3}".format(self._id,eventType,eventSubType,uid)
data["var"]["event"] = {}
data["var"]["event"]["type"] = eventType
data["var"]["event"]["eventSubType"] = eventSubType
data["var"]["event"]["layer"] = layer
data["var"]["event"]["accuracy"] = accuracy
data["var"]["event"]["impact"] = impact
data["var"]["event"]["benign"] = benign
try:
score = ((accuracy*(impact*layer))/benign)
except ZeroDivisionError:
score = 0
data["var"]["event"]["score"] = score
cacheUID = "{0}-{1}-{2}".format(data["conductID"],data["flow_id"],uid)
foundEvent = cache.globalCache.get("eventCache",cacheUID,getEvent,data["conductID"],data["flow_id"],uid,eventType,eventSubType,extendCacheTime=True,customCacheTime=timeToLive,nullUpdate=True)
if foundEvent != None:
try:
persistentData["plugin"]["event"].append(foundEvent)
except:
persistentData["plugin"]["event"] = [foundEvent]
arrayIndex = len(persistentData["plugin"]["event"])-1
actionResult["eventIndex"] = arrayIndex
if foundEvent._id != "":
if foundEvent.expiryTime > time.time():
changes = False
for key,value in eventValues.items():
if key in foundEvent.eventValues:
if value != foundEvent.eventValues[key]:
changes = True
break
else:
changes = True
break
if changes:
foundEvent.updateRecord(self.bulkClass,eventValues,accuracy,impact,layer,benign,score,int( time.time() + timeToLive ),self.history)
actionResult["result"] = True
actionResult["rc"] = 202
return actionResult
else:
foundEvent.expiryTime = int(time.time() + timeToLive)
foundEvent.bulkUpdate(["expiryTime"],self.bulkClass)
actionResult["result"] = True
actionResult["rc"] = 302
return actionResult
else:
cache.globalCache.delete("eventCache",cacheUID)
else:
logging.debug("Event Update Failed - NO ID, actionID='{0}'".format(self._id),7)
actionResult["result"] = False
actionResult["rc"] = 500
return actionResult
eventObject = event._event().bulkNew(self.bulkClass,self.acl,data["conductID"],data["flow_id"],eventType,eventSubType,int( time.time() + timeToLive ),eventValues,uid,accuracy,impact,layer,benign,score,data,eventTitle)
cache.globalCache.insert("eventCache",cacheUID,eventObject,customCacheTime=timeToLive)
try:
persistentData["plugin"]["event"].append(eventObject)
except:
persistentData["plugin"]["event"] = [eventObject]
arrayIndex = len(persistentData["plugin"]["event"])-1
actionResult["eventIndex"] = arrayIndex
actionResult["result"] = True
actionResult["rc"] = 201
return actionResult
def run(self,data,persistentData,actionResult):
correlationName = helpers.evalString(self.correlationName,{"data" : data})
excludeCorrelationValues = helpers.evalDict(self.excludeCorrelationValues,{"data" : data})
expiryTime = time.time() + self.expiryTime
correlatedRelationships = event._eventCorrelation().getAsClass(query={ "correlationName" : correlationName, "expiryTime" : { "$gt" : int(time.time()) } })
eventsAfterTime = int(time.time()) - self.oldestEvent
if not self.alwaysProcessEvents:
ids = event._eventCorrelation()._dbCollection.distinct("ids",{ "$or" : [ { "expiryTime" : { "$gt" : int(time.time()) } }, { "lastUpdateTime" : { "$gt" : eventsAfterTime } } ] })
objectIds = []
for idItem in ids:
objectIds.append(db.ObjectId(idItem))
events = event._event().getAsClass(query={ "_id" : { "$nin" : objectIds }, "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } })
else:
events = event._event().getAsClass(query={ "expiryTime" : { "$gt" : eventsAfterTime }, "eventFields" : { "$in" : self.correlationFields } })
# Build correlation field hash table
fields = {}
for field in self.correlationFields:
fields[field] = {}
if field not in excludeCorrelationValues:
excludeCorrelationValues[field] = []
for correlatedRelationshipItem in correlatedRelationships:
for field in self.correlationFields:
try:
if field not in excludeCorrelationValues:
excludeCorrelationValues[field] = []
for value in correlatedRelationshipItem.correlations[field]:
fields[field][value] = correlatedRelationshipItem
except KeyError:
pass
correlatedRelationshipsCreated = []
correlatedRelationshipsUpdated = []
correlatedRelationshipsDeleted = []
# Initial Pass Loop
for eventItem in events:
foundCorrelatedRelationship = None
correlations = {}
processNew = False
for eventField in eventItem.eventValues:
try:
if type(eventItem.eventValues[eventField]) is list:
correlations[eventField] = [ x for x in eventItem.eventValues[eventField] if eventField in self.correlationFields ]
matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x in fields[eventField] and x not in excludeCorrelationValues[eventField] ]
if len(matchFound) > 0:
foundCorrelatedRelationship = matchFound[0]
else:
matchFound = [ fields[eventField][x] for x in eventItem.eventValues[eventField] if eventField in self.correlationFields and x and x not in excludeCorrelationValues[eventField] ]
if len(matchFound) > 0:
processNew = True
else:
correlations[eventField] = [eventItem.eventValues[eventField]]
if eventField in self.correlationFields and eventItem.eventValues[eventField] in fields[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]:
foundCorrelatedRelationship = fields[eventField][eventItem.eventValues[eventField]]
else:
if eventField in self.correlationFields and eventItem.eventValues[eventField] and eventItem.eventValues[eventField] not in excludeCorrelationValues[eventField]:
processNew = True
except KeyError:
pass
# Create new
if processNew == True:
newEventCorrelation = event._eventCorrelation()
newEventCorrelation.bulkNew(self.bulkClass, self.acl, correlationName,expiryTime,[eventItem._id],[eventItem.eventType],[eventItem.eventSubType],correlations,eventItem.score)
correlatedRelationshipsCreated.append(newEventCorrelation)
correlatedRelationships.append(newEventCorrelation)
for eventField in eventItem.eventValues:
try:
for eventValue in correlations[eventField]:
try:
fields[eventField][eventValue] = newEventCorrelation
except KeyError:
fields[eventField] = { eventValue : newEventCorrelation }
except KeyError:
pass
# Merge existing
elif foundCorrelatedRelationship != None:
for eventField in correlations:
try:
foundCorrelatedRelationship.correlations[eventField] += correlations[eventField]
foundCorrelatedRelationship.correlations[eventField] = list(set(foundCorrelatedRelationship.correlations[eventField]))
except KeyError:
foundCorrelatedRelationship.correlations[eventField] = correlations[eventField]
if eventItem._id not in foundCorrelatedRelationship.ids:
foundCorrelatedRelationship.ids.append(eventItem._id)
foundCorrelatedRelationship.score += eventItem.score
if eventItem.eventType not in foundCorrelatedRelationship.types:
foundCorrelatedRelationship.types.append(eventItem.eventType)
if eventItem.eventSubType not in foundCorrelatedRelationship.subTypes:
foundCorrelatedRelationship.subTypes.append(eventItem.eventSubType)
foundCorrelatedRelationship.correlationLastUpdate = int(time.time())
foundCorrelatedRelationship.expiryTime = expiryTime
if foundCorrelatedRelationship not in correlatedRelationshipsCreated and foundCorrelatedRelationship not in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.append(foundCorrelatedRelationship)
# Process bulk creation if needed before merging
self.bulkClass.bulkOperatonProcessing()
# Reduction Loop
loop = 1
maxLoops = 5
while loop > 0 and maxLoops > 0:
correlatedFieldsHash = {}
for correlatedRelationship in correlatedRelationships:
for eventField, eventValue in ((eventField, eventValue) for eventField in correlatedRelationship.correlations for eventValue in correlatedRelationship.correlations[eventField] ):
if eventField in self.correlationFields and eventValue not in excludeCorrelationValues[eventField]:
try:
if eventValue not in correlatedFieldsHash[eventField]:
correlatedFieldsHash[eventField][eventValue] = correlatedRelationship
else:
currentCorrelation = correlatedFieldsHash[eventField][eventValue]
for eventField in correlatedRelationship.correlations:
try:
currentCorrelation.correlations[eventField] += correlatedRelationship.correlations[eventField]
currentCorrelation.correlations[eventField] = list(set(currentCorrelation.correlations[eventField]))
except KeyError:
currentCorrelation.correlations[eventField] = correlatedRelationship.correlations[eventField]
for mergeKey in ["ids","types","subTypes"]:
for value in getattr(correlatedRelationship,mergeKey):
if value not in getattr(currentCorrelation,mergeKey):
getattr(currentCorrelation,mergeKey).append(value)
currentCorrelation.score += correlatedRelationship.score
currentCorrelation.correlationLastUpdate = int(time.time())
currentCorrelation.expiryTime = expiryTime
if currentCorrelation not in correlatedRelationshipsCreated and correlatedRelationship not in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.append(currentCorrelation)
# Deleting the eventCorrelation it was merged with
if correlatedRelationship not in correlatedRelationshipsDeleted:
correlatedRelationshipsDeleted.append(correlatedRelationship)
if correlatedRelationship in correlatedRelationshipsUpdated:
correlatedRelationshipsUpdated.remove(correlatedRelationship)
if correlatedRelationship in correlatedRelationshipsCreated:
correlatedRelationshipsCreated.remove(correlatedRelationship)
correlatedRelationship.bulkMerge(currentCorrelation._id,self.bulkClass)
correlatedRelationships.remove(correlatedRelationship)
loop+=1
break
except KeyError:
correlatedFieldsHash[eventField] = { eventValue : correlatedRelationship }
maxLoops -= 1
loop -= 1
created = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsCreated ]
updated = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsUpdated ]
deleted = [ helpers.classToJson(x,hidden=True) for x in correlatedRelationshipsDeleted ]
for correlatedRelationshipUpdated in correlatedRelationshipsUpdated:
correlatedRelationshipUpdated.bulkUpdate(["expiryTime","ids","types","subTypes","correlations","score"],self.bulkClass)
updated.append(helpers.classToJson(correlatedRelationshipUpdated,hidden=True))
self.bulkClass.bulkOperatonProcessing()
actionResult["result"] = True
actionResult["rc"] = 0
actionResult["correlatedEvents"] = { "created" : created, "updated" : updated, "deleted" : deleted }
return actionResult
def run(self, data, persistentData, actionResult):
assetType = helpers.evalString(self.assetType, {"data": data})
assetEntity = helpers.evalString(self.assetEntity, {"data": data})
updateSource = helpers.evalString(self.updateSource, {"data": data})
updateTime = helpers.evalString(self.updateTime, {"data": data})
assetData = helpers.evalDict(self.assetData, {"data": data})
existingAssets = asset._asset().getAsClass(
query={
"name": {
"$in": list(assetData.keys())
},
"assetType": assetType,
"entity": assetEntity
})
# Updating existing
for assetItem in existingAssets:
lastSeen = None
for source in assetItem.lastSeen:
if source["source"] == updateSource:
if "lastUpdate" not in lastSeen:
lastSeen["lastUpdate"] = 0
lastSeen = source
break
if not lastSeen:
assetItem.lastSeen.append({
"source": updateSource,
"lastUpdate": 0
})
lastSeen = assetItem.lastSeen[-1]
# Converting millsecond int epoch into epoch floats
currentTimestamp = lastSeen["lastUpdate"]
if len(str(currentTimestamp).split(".")[0]) == 13:
currentTimestamp = currentTimestamp / 1000
updateTime = assetData[assetItem.name]["lastUpdate"]
if len(str(updateTime).split(".")[0]) == 13:
updateTime = updateTime / 1000
newTimestamp = None
if updateTime:
try:
if updateTime < currentTimestamp:
newTimestamp = False
else:
newTimestamp = updateTime
except (KeyError, ValueError):
pass
if newTimestamp == None:
newTimestamp = time.time()
assetChanged = False
if newTimestamp != False:
try:
if (time.time() - currentTimestamp) < self.delayedUpdate:
continue
except KeyError:
pass
assetChanged = True
if newTimestamp > assetItem.lastSeenTimestamp:
assetItem.lastSeenTimestamp = newTimestamp
if self.replaceExisting:
lastSeen = assetData[assetItem.name]
else:
for key, value in assetData[assetItem.name].items():
lastSeen[key] = value
lastSeen["priority"] = self.sourcePriority
lastSeen["lastUpdate"] = newTimestamp
# Working out priority and define fields
if assetChanged:
foundValues = {}
now = time.time()
blacklist = ["lastUpdate", "priority"]
for sourceValue in assetItem.lastSeen:
for key, value in sourceValue.items():
if key not in blacklist:
if key not in foundValues:
foundValues[key] = {
"value": value,
"priority": sourceValue["priority"]
}
else:
if sourceValue["priority"] < foundValues[key][
"priority"] and (
sourceValue["lastUpdate"] +
self.sourcePriorityMaxAge) > now:
foundValues[key] = {
"value": value,
"priority": sourceValue["priority"]
}
assetItem.fields = {}
for key, value in foundValues.items():
assetItem.fields[key] = value["value"]
assetItem.bulkUpdate(["lastSeen", "fields"], self.bulkClass)
del assetData[assetItem.name]
# Adding new
for assetName, assetFields in assetData.items():
assetItem = asset._asset().bulkNew(self.acl, assetName,
assetEntity, assetType,
updateSource, assetFields,
updateTime, self.sourcePriority,
self.bulkClass)
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
