def check(self):
latestTime = 0
search = helpers.evalDict(self.search, {"data": {}})
match = ""
for key, value in search.items():
match += key
if type(value) == str:
match += value
if not self.cache:
# cache does not support nested dicts as these will always be seen as the same
assetList = cache.globalCache.get("assetSearchCache", match,
getSearchObject, search,
self.fields)
else:
assetList = asset._asset().query(query=search,
fields=self.fields)["results"]
if assetList is not None:
for assetItem in assetList:
if len(self.flattenFields) > 0:
for flattenField in self.flattenFields:
if flattenField in assetItem["fields"]:
assetItem[flattenField] = assetItem["fields"][
flattenField]
del assetItem["fields"][flattenField]
if self.return_one:
if "event" in self.result:
if self.result["event"]["lastUpdateTime"] > latestTime:
self.result["event"] = assetItem
else:
self.result["event"] = asset
else:
self.result["events"].append(asset)
def run(self,data,persistentData,actionResult):
matches = helpers.evalDict(self.matches,{ "data" : data })
lastSeenWithin = self.lastSeenWithin
if lastSeenWithin == 0:
lastSeenWithin = 86400
ors = {}
finds = {}
for matchItem, matchValue in matches.items():
if type(matchValue) is list:
ors[matchItem] = matchValue
else:
finds[matchItem] = matchValue
search = {}
search["$or"] = []
for matchItem, matchValue in matches.items():
if type(matchValue) is list:
for matchValueItem in matchValue:
if matchValueItem != None:
search["$or"].append( { matchItem : matchValueItem })
else:
if matchValue != None:
search["$or"].append( { matchItem : matchValue })
search["lastSeenTimestamp"] = { "$gt" : time.time() - lastSeenWithin }
assetsList = asset._asset().query(query=search)["results"]
actionResult["assets"] = assetsList
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
def doughnutEntity():
doughnut = ui.doughnut()
entities = asset._asset().groupby(sessionData=api.g.sessionData,
field="entity")
data = []
for entity in entities:
doughnut.addLabel(entity["_id"])
data.append(entity["_count"])
doughnut.addDataset("Assets", data)
data = json.loads(jimi.api.request.data)
return doughnut.generate(data), 200
def singleAsset(assetID):
assetObject = asset._asset().getAsClass(sessionData=api.g.sessionData,
id=assetID)[0]
assetSources = []
for source in assetObject.lastSeen:
assetSources.append(source["source"])
assetSources = ", ".join(assetSources)
return render_template("assetItem.html",
CSRF=jimi.api.g.sessionData["CSRF"],
assetObject=assetObject,
assetSources=assetSources)
def radarSources():
radar = ui.radar()
sources = asset._asset().groupby(sessionData=api.g.sessionData,
field="lastSeen.source")
data = {}
for source in sources:
for _id in source["_id"]:
if _id not in data:
data[_id] = 0
data[_id] += source["_count"]
dataPoints = []
for key, value in data.items():
radar.addLabel(key)
dataPoints.append(value)
radar.addDataset("Assets", dataPoints)
data = json.loads(jimi.api.request.data)
return radar.generate(data), 200
def run(self,data,persistentData,actionResult):
latestTime = 0
search = helpers.evalDict(self.search,{"data" : data})
order_by = helpers.evalString(self.order_by,{"data" : data})
match = ""
for key, value in search.items():
match += key
if type(value) == str:
match += value
sort = None
if order_by != "":
if self.ascending:
sort = [(order_by, 1)]
else:
sort = [(order_by, -1)]
if not self.cache:
# cache does not support nested dicts as these will always be seen as the same
assetList = cache.globalCache.get("assetSearchCache",match,getSearchObject,search,self.fields,sort)
else:
assetList = asset._asset().query(query=search,fields=self.fields,sort=sort)["results"]
if self.return_one:
actionResult["event"] = ""
else:
actionResult["events"] = []
if assetList is not None:
for assetItem in assetList:
if len(self.flattenFields) > 0:
for flattenField in self.flattenFields:
if flattenField in assetItem["fields"]:
assetItem[flattenField] = assetItem["fields"][flattenField]
del assetItem["fields"][flattenField]
if self.return_one:
if assetItem["lastUpdateTime"] > latestTime:
actionResult["event"] = assetItem
else:
actionResult["events"].append(assetItem)
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
def singleAssetTableFieldsSources(assetID, action):
assetObject = asset._asset().getAsClass(sessionData=api.g.sessionData,
id=assetID)[0]
total = len(assetObject.fields)
columns = ["Source", "Fields"]
table = ui.table(columns, total, total)
if action == "build":
return table.getColumns(), 200
elif action == "poll":
# Custom table data so it can be vertical
data = []
for source in assetObject.lastSeen:
data.append([ui.safe(source["source"]), ui.dictTable(source)])
table.data = data
return {
"draw": int(jimi.api.request.args.get('draw')),
"recordsTable": 0,
"recordsFiltered": 0,
"recordsTotal": 0,
"data": data
}, 200
def lineCreated():
line = ui.line()
aggregateStatement = [{
"$match": {
"creationTime": {
"$gt": time.time() - (86400 * 365)
}
}
}, {
"$project": {
"_id": 0,
"date": {
"$toDate": "$_id"
}
}
}, {
"$group": {
"_id": {
"$month": "$date"
},
"_count": {
"$sum": 1
}
}
}, {
"$sort": {
"_id": 1
}
}]
timeline = asset._asset().aggregate(sessionData=api.g.sessionData,
aggregateStatement=aggregateStatement)
data = []
for item in timeline:
line.addLabel(item["_id"])
data.append(item["_count"])
line.addDataset("Assets", data)
data = json.loads(jimi.api.request.data)
return line.generate(data), 200
def singleAssetNetworkRelationships(assetID):
assetObject = asset._asset().getAsClass(sessionData=api.g.sessionData,
id=assetID)[0]
timespan = helpers.roundTime(roundTo=int("86400")).timestamp()
relationships = relationship._assetRelationship().getAsClass(
sessionData=api.g.sessionData,
query={
"timespan":
timespan,
"$or": [{
"fromAsset": assetObject.fields["ip"]
}, {
"toAsset": assetObject.fields["ip"]
}]
})
nodesDict = {}
edgesDict = {}
for relationshipItem in relationships:
if relationshipItem.fromAsset not in nodesDict:
nodesDict[relationshipItem.fromAsset] = {
"id": relationshipItem.fromAsset,
"label": relationshipItem.fromAsset,
"shape": "image",
"image": "/static/img/computer.svg",
"value": 1,
"color": {
"background": "#C72F1E",
"border": "#C72F1E",
"highlight": {
"background": "#000",
"border": "#FFF"
}
}
}
if relationshipItem.toAsset not in nodesDict:
nodesDict[relationshipItem.toAsset] = {
"id": relationshipItem.toAsset,
"label": relationshipItem.toAsset,
"shape": "image",
"image": "/static/img/computer.svg",
"value": 1,
"color": {
"background": "#C72F1E",
"border": "#C72F1E",
"highlight": {
"background": "#000",
"border": "#FFF"
}
}
}
key = "{0}-{1}".format(relationshipItem.fromAsset,
relationshipItem.toAsset)
if key not in edgesDict:
edgesDict[key] = {
"id": key,
"from": relationshipItem.fromAsset,
"to": relationshipItem.toAsset
}
nodes = [x for x in nodesDict.values()]
edges = [x for x in edgesDict.values()]
return {"nodes": nodes, "edges": edges}, 200
def getAssetObject(match, sessionData, assetName, assetType, assetEntity):
return asset._asset().getAsClass(query={
"name": assetName,
"assetType": assetType,
"entity": assetEntity
})
def run(self, data, persistentData, actionResult):
assetType = helpers.evalString(self.assetType, {"data": data})
assetEntity = helpers.evalString(self.assetEntity, {"data": data})
updateSource = helpers.evalString(self.updateSource, {"data": data})
updateTime = helpers.evalString(self.updateTime, {"data": data})
assetData = helpers.evalDict(self.assetData, {"data": data})
existingAssets = asset._asset().getAsClass(
query={
"name": {
"$in": list(assetData.keys())
},
"assetType": assetType,
"entity": assetEntity
})
# Updating existing
for assetItem in existingAssets:
lastSeen = None
for source in assetItem.lastSeen:
if source["source"] == updateSource:
if "lastUpdate" not in lastSeen:
lastSeen["lastUpdate"] = 0
lastSeen = source
break
if not lastSeen:
assetItem.lastSeen.append({
"source": updateSource,
"lastUpdate": 0
})
lastSeen = assetItem.lastSeen[-1]
# Converting millsecond int epoch into epoch floats
currentTimestamp = lastSeen["lastUpdate"]
if len(str(currentTimestamp).split(".")[0]) == 13:
currentTimestamp = currentTimestamp / 1000
updateTime = assetData[assetItem.name]["lastUpdate"]
if len(str(updateTime).split(".")[0]) == 13:
updateTime = updateTime / 1000
newTimestamp = None
if updateTime:
try:
if updateTime < currentTimestamp:
newTimestamp = False
else:
newTimestamp = updateTime
except (KeyError, ValueError):
pass
if newTimestamp == None:
newTimestamp = time.time()
assetChanged = False
if newTimestamp != False:
try:
if (time.time() - currentTimestamp) < self.delayedUpdate:
continue
except KeyError:
pass
assetChanged = True
if newTimestamp > assetItem.lastSeenTimestamp:
assetItem.lastSeenTimestamp = newTimestamp
if self.replaceExisting:
lastSeen = assetData[assetItem.name]
else:
for key, value in assetData[assetItem.name].items():
lastSeen[key] = value
lastSeen["priority"] = self.sourcePriority
lastSeen["lastUpdate"] = newTimestamp
# Working out priority and define fields
if assetChanged:
foundValues = {}
now = time.time()
blacklist = ["lastUpdate", "priority"]
for sourceValue in assetItem.lastSeen:
for key, value in sourceValue.items():
if key not in blacklist:
if key not in foundValues:
foundValues[key] = {
"value": value,
"priority": sourceValue["priority"]
}
else:
if sourceValue["priority"] < foundValues[key][
"priority"] and (
sourceValue["lastUpdate"] +
self.sourcePriorityMaxAge) > now:
foundValues[key] = {
"value": value,
"priority": sourceValue["priority"]
}
assetItem.fields = {}
for key, value in foundValues.items():
assetItem.fields[key] = value["value"]
assetItem.bulkUpdate(["lastSeen", "fields"], self.bulkClass)
del assetData[assetItem.name]
# Adding new
for assetName, assetFields in assetData.items():
assetItem = asset._asset().bulkNew(self.acl, assetName,
assetEntity, assetType,
updateSource, assetFields,
updateTime, self.sourcePriority,
self.bulkClass)
actionResult["result"] = True
actionResult["rc"] = 0
return actionResult
def run(self, data, persistentData, actionResult):
assetName = helpers.evalString(self.assetName, {"data": data})
assetType = helpers.evalString(self.assetType, {"data": data})
assetEntity = helpers.evalString(self.assetEntity, {"data": data})
updateSource = helpers.evalString(self.updateSource, {"data": data})
updateTime = helpers.evalString(self.updateTime, {"data": data})
assetFields = helpers.evalDict(self.assetFields, {"data": data})
if not assetName or not assetType or not updateSource or not assetFields:
actionResult["result"] = False
actionResult["msg"] = "Missing required properties"
actionResult["rc"] = 403
return actionResult
match = "{0}-{1}-{2}".format(assetName, assetType, assetEntity)
assetItem = cache.globalCache.get("assetCache", match, getAssetObject,
assetName, assetType, assetEntity)
if not assetItem:
assetItem = asset._asset().bulkNew(self.acl, assetName,
assetEntity, assetType,
updateSource, assetFields,
updateTime, self.sourcePriority,
self.bulkClass)
cache.globalCache.insert("assetCache", match, [assetItem])
actionResult["result"] = True
actionResult["msg"] = "Created new asset"
actionResult["rc"] = 201
return actionResult
assetChanged = False
# Removing entires if more than one result is found
if len(assetItem) > 1:
newestItem = assetItem[0]
for singleAssetItem in assetItem:
if newestItem.lastUpdateTime < singleAssetItem.lastUpdateTime:
singleAssetItem.delete()
else:
newestItem.delete()
newestItem = singleAssetItem
assetItem = newestItem
cache.globalCache.update("assetCache", match, [assetItem])
assetChanged = True
else:
assetItem = assetItem[0]
if assetItem._id == "":
actionResult["result"] = False
actionResult["msg"] = "Asset not yet added"
actionResult["rc"] = 404
return actionResult
lastSeen = None
for source in assetItem.lastSeen:
if source["source"] == updateSource:
if "lastUpdate" not in lastSeen:
lastSeen["lastUpdate"] = 0
lastSeen = source
break
if not lastSeen:
assetItem.lastSeen.append({
"source": updateSource,
"lastUpdate": 0
})
lastSeen = assetItem.lastSeen[-1]
# Converting millsecond int epoch into epoch floats
currentTimestamp = lastSeen["lastUpdate"]
if len(str(currentTimestamp).split(".")[0]) == 13:
currentTimestamp = currentTimestamp / 1000
if len(str(updateTime).split(".")[0]) == 13:
updateTime = updateTime / 1000
newTimestamp = None
if updateTime:
try:
if updateTime < currentTimestamp:
newTimestamp = False
else:
newTimestamp = updateTime
except (KeyError, ValueError):
pass
if newTimestamp == None:
newTimestamp = time.time()
if newTimestamp != False:
try:
if (time.time() - currentTimestamp) < self.delayedUpdate:
actionResult["result"] = False
actionResult["msg"] = "Delay time not met"
actionResult["rc"] = 300
return actionResult
except KeyError:
pass
assetChanged = True
if newTimestamp > assetItem.lastSeenTimestamp:
assetItem.lastSeenTimestamp = newTimestamp
if self.replaceExisting:
lastSeen = assetFields
else:
for key, value in assetFields.items():
lastSeen[key] = value
lastSeen["priority"] = self.sourcePriority
lastSeen["lastUpdate"] = newTimestamp
# Working out priority and define fields
if assetChanged:
foundValues = {}
now = time.time()
blacklist = ["lastUpdate", "priority"]
for sourceValue in assetItem.lastSeen:
for key, value in sourceValue.items():
if key not in blacklist:
if key not in foundValues:
foundValues[key] = {
"value": value,
"priority": sourceValue["priority"]
}
else:
if sourceValue["priority"] < foundValues[key][
"priority"] and (
sourceValue["lastUpdate"] +
self.sourcePriorityMaxAge) > now:
foundValues[key] = {
"value": value,
"priority": sourceValue["priority"]
}
assetItem.fields = {}
for key, value in foundValues.items():
assetItem.fields[key] = value["value"]
assetItem.bulkUpdate(["lastSeen", "fields"], self.bulkClass)
actionResult["result"] = True
actionResult["msg"] = "Updated asset"
actionResult["rc"] = 302
return actionResult
actionResult["result"] = True
actionResult["msg"] = "Nothing to do"
actionResult["rc"] = 304
return actionResult
def getSearchObject(match, sessionData, search, fields):
return asset._asset().query(query=search, fields=fields)["results"]
