def check(self): latestTime = 0 search = helpers.evalDict(self.search, {"data": {}}) match = "" for key, value in search.items(): match += key if type(value) == str: match += value if not self.cache: # cache does not support nested dicts as these will always be seen as the same assetList = cache.globalCache.get("assetSearchCache", match, getSearchObject, search, self.fields) else: assetList = asset._asset().query(query=search, fields=self.fields)["results"] if assetList is not None: for assetItem in assetList: if len(self.flattenFields) > 0: for flattenField in self.flattenFields: if flattenField in assetItem["fields"]: assetItem[flattenField] = assetItem["fields"][ flattenField] del assetItem["fields"][flattenField] if self.return_one: if "event" in self.result: if self.result["event"]["lastUpdateTime"] > latestTime: self.result["event"] = assetItem else: self.result["event"] = asset else: self.result["events"].append(asset)
def run(self,data,persistentData,actionResult): matches = helpers.evalDict(self.matches,{ "data" : data }) lastSeenWithin = self.lastSeenWithin if lastSeenWithin == 0: lastSeenWithin = 86400 ors = {} finds = {} for matchItem, matchValue in matches.items(): if type(matchValue) is list: ors[matchItem] = matchValue else: finds[matchItem] = matchValue search = {} search["$or"] = [] for matchItem, matchValue in matches.items(): if type(matchValue) is list: for matchValueItem in matchValue: if matchValueItem != None: search["$or"].append( { matchItem : matchValueItem }) else: if matchValue != None: search["$or"].append( { matchItem : matchValue }) search["lastSeenTimestamp"] = { "$gt" : time.time() - lastSeenWithin } assetsList = asset._asset().query(query=search)["results"] actionResult["assets"] = assetsList actionResult["result"] = True actionResult["rc"] = 0 return actionResult
def doughnutEntity(): doughnut = ui.doughnut() entities = asset._asset().groupby(sessionData=api.g.sessionData, field="entity") data = [] for entity in entities: doughnut.addLabel(entity["_id"]) data.append(entity["_count"]) doughnut.addDataset("Assets", data) data = json.loads(jimi.api.request.data) return doughnut.generate(data), 200
def singleAsset(assetID): assetObject = asset._asset().getAsClass(sessionData=api.g.sessionData, id=assetID)[0] assetSources = [] for source in assetObject.lastSeen: assetSources.append(source["source"]) assetSources = ", ".join(assetSources) return render_template("assetItem.html", CSRF=jimi.api.g.sessionData["CSRF"], assetObject=assetObject, assetSources=assetSources)
def radarSources(): radar = ui.radar() sources = asset._asset().groupby(sessionData=api.g.sessionData, field="lastSeen.source") data = {} for source in sources: for _id in source["_id"]: if _id not in data: data[_id] = 0 data[_id] += source["_count"] dataPoints = [] for key, value in data.items(): radar.addLabel(key) dataPoints.append(value) radar.addDataset("Assets", dataPoints) data = json.loads(jimi.api.request.data) return radar.generate(data), 200
def run(self,data,persistentData,actionResult): latestTime = 0 search = helpers.evalDict(self.search,{"data" : data}) order_by = helpers.evalString(self.order_by,{"data" : data}) match = "" for key, value in search.items(): match += key if type(value) == str: match += value sort = None if order_by != "": if self.ascending: sort = [(order_by, 1)] else: sort = [(order_by, -1)] if not self.cache: # cache does not support nested dicts as these will always be seen as the same assetList = cache.globalCache.get("assetSearchCache",match,getSearchObject,search,self.fields,sort) else: assetList = asset._asset().query(query=search,fields=self.fields,sort=sort)["results"] if self.return_one: actionResult["event"] = "" else: actionResult["events"] = [] if assetList is not None: for assetItem in assetList: if len(self.flattenFields) > 0: for flattenField in self.flattenFields: if flattenField in assetItem["fields"]: assetItem[flattenField] = assetItem["fields"][flattenField] del assetItem["fields"][flattenField] if self.return_one: if assetItem["lastUpdateTime"] > latestTime: actionResult["event"] = assetItem else: actionResult["events"].append(assetItem) actionResult["result"] = True actionResult["rc"] = 0 return actionResult
def singleAssetTableFieldsSources(assetID, action): assetObject = asset._asset().getAsClass(sessionData=api.g.sessionData, id=assetID)[0] total = len(assetObject.fields) columns = ["Source", "Fields"] table = ui.table(columns, total, total) if action == "build": return table.getColumns(), 200 elif action == "poll": # Custom table data so it can be vertical data = [] for source in assetObject.lastSeen: data.append([ui.safe(source["source"]), ui.dictTable(source)]) table.data = data return { "draw": int(jimi.api.request.args.get('draw')), "recordsTable": 0, "recordsFiltered": 0, "recordsTotal": 0, "data": data }, 200
def lineCreated(): line = ui.line() aggregateStatement = [{ "$match": { "creationTime": { "$gt": time.time() - (86400 * 365) } } }, { "$project": { "_id": 0, "date": { "$toDate": "$_id" } } }, { "$group": { "_id": { "$month": "$date" }, "_count": { "$sum": 1 } } }, { "$sort": { "_id": 1 } }] timeline = asset._asset().aggregate(sessionData=api.g.sessionData, aggregateStatement=aggregateStatement) data = [] for item in timeline: line.addLabel(item["_id"]) data.append(item["_count"]) line.addDataset("Assets", data) data = json.loads(jimi.api.request.data) return line.generate(data), 200
def singleAssetNetworkRelationships(assetID): assetObject = asset._asset().getAsClass(sessionData=api.g.sessionData, id=assetID)[0] timespan = helpers.roundTime(roundTo=int("86400")).timestamp() relationships = relationship._assetRelationship().getAsClass( sessionData=api.g.sessionData, query={ "timespan": timespan, "$or": [{ "fromAsset": assetObject.fields["ip"] }, { "toAsset": assetObject.fields["ip"] }] }) nodesDict = {} edgesDict = {} for relationshipItem in relationships: if relationshipItem.fromAsset not in nodesDict: nodesDict[relationshipItem.fromAsset] = { "id": relationshipItem.fromAsset, "label": relationshipItem.fromAsset, "shape": "image", "image": "/static/img/computer.svg", "value": 1, "color": { "background": "#C72F1E", "border": "#C72F1E", "highlight": { "background": "#000", "border": "#FFF" } } } if relationshipItem.toAsset not in nodesDict: nodesDict[relationshipItem.toAsset] = { "id": relationshipItem.toAsset, "label": relationshipItem.toAsset, "shape": "image", "image": "/static/img/computer.svg", "value": 1, "color": { "background": "#C72F1E", "border": "#C72F1E", "highlight": { "background": "#000", "border": "#FFF" } } } key = "{0}-{1}".format(relationshipItem.fromAsset, relationshipItem.toAsset) if key not in edgesDict: edgesDict[key] = { "id": key, "from": relationshipItem.fromAsset, "to": relationshipItem.toAsset } nodes = [x for x in nodesDict.values()] edges = [x for x in edgesDict.values()] return {"nodes": nodes, "edges": edges}, 200
def getAssetObject(match, sessionData, assetName, assetType, assetEntity): return asset._asset().getAsClass(query={ "name": assetName, "assetType": assetType, "entity": assetEntity })
def run(self, data, persistentData, actionResult): assetType = helpers.evalString(self.assetType, {"data": data}) assetEntity = helpers.evalString(self.assetEntity, {"data": data}) updateSource = helpers.evalString(self.updateSource, {"data": data}) updateTime = helpers.evalString(self.updateTime, {"data": data}) assetData = helpers.evalDict(self.assetData, {"data": data}) existingAssets = asset._asset().getAsClass( query={ "name": { "$in": list(assetData.keys()) }, "assetType": assetType, "entity": assetEntity }) # Updating existing for assetItem in existingAssets: lastSeen = None for source in assetItem.lastSeen: if source["source"] == updateSource: if "lastUpdate" not in lastSeen: lastSeen["lastUpdate"] = 0 lastSeen = source break if not lastSeen: assetItem.lastSeen.append({ "source": updateSource, "lastUpdate": 0 }) lastSeen = assetItem.lastSeen[-1] # Converting millsecond int epoch into epoch floats currentTimestamp = lastSeen["lastUpdate"] if len(str(currentTimestamp).split(".")[0]) == 13: currentTimestamp = currentTimestamp / 1000 updateTime = assetData[assetItem.name]["lastUpdate"] if len(str(updateTime).split(".")[0]) == 13: updateTime = updateTime / 1000 newTimestamp = None if updateTime: try: if updateTime < currentTimestamp: newTimestamp = False else: newTimestamp = updateTime except (KeyError, ValueError): pass if newTimestamp == None: newTimestamp = time.time() assetChanged = False if newTimestamp != False: try: if (time.time() - currentTimestamp) < self.delayedUpdate: continue except KeyError: pass assetChanged = True if newTimestamp > assetItem.lastSeenTimestamp: assetItem.lastSeenTimestamp = newTimestamp if self.replaceExisting: lastSeen = assetData[assetItem.name] else: for key, value in assetData[assetItem.name].items(): lastSeen[key] = value lastSeen["priority"] = self.sourcePriority lastSeen["lastUpdate"] = newTimestamp # Working out priority and define fields if assetChanged: foundValues = {} now = time.time() blacklist = ["lastUpdate", "priority"] for sourceValue in assetItem.lastSeen: for key, value in sourceValue.items(): if key not in blacklist: if key not in foundValues: foundValues[key] = { "value": value, "priority": sourceValue["priority"] } else: if sourceValue["priority"] < foundValues[key][ "priority"] and ( sourceValue["lastUpdate"] + self.sourcePriorityMaxAge) > now: foundValues[key] = { "value": value, "priority": sourceValue["priority"] } assetItem.fields = {} for key, value in foundValues.items(): assetItem.fields[key] = value["value"] assetItem.bulkUpdate(["lastSeen", "fields"], self.bulkClass) del assetData[assetItem.name] # Adding new for assetName, assetFields in assetData.items(): assetItem = asset._asset().bulkNew(self.acl, assetName, assetEntity, assetType, updateSource, assetFields, updateTime, self.sourcePriority, self.bulkClass) actionResult["result"] = True actionResult["rc"] = 0 return actionResult
def run(self, data, persistentData, actionResult): assetName = helpers.evalString(self.assetName, {"data": data}) assetType = helpers.evalString(self.assetType, {"data": data}) assetEntity = helpers.evalString(self.assetEntity, {"data": data}) updateSource = helpers.evalString(self.updateSource, {"data": data}) updateTime = helpers.evalString(self.updateTime, {"data": data}) assetFields = helpers.evalDict(self.assetFields, {"data": data}) if not assetName or not assetType or not updateSource or not assetFields: actionResult["result"] = False actionResult["msg"] = "Missing required properties" actionResult["rc"] = 403 return actionResult match = "{0}-{1}-{2}".format(assetName, assetType, assetEntity) assetItem = cache.globalCache.get("assetCache", match, getAssetObject, assetName, assetType, assetEntity) if not assetItem: assetItem = asset._asset().bulkNew(self.acl, assetName, assetEntity, assetType, updateSource, assetFields, updateTime, self.sourcePriority, self.bulkClass) cache.globalCache.insert("assetCache", match, [assetItem]) actionResult["result"] = True actionResult["msg"] = "Created new asset" actionResult["rc"] = 201 return actionResult assetChanged = False # Removing entires if more than one result is found if len(assetItem) > 1: newestItem = assetItem[0] for singleAssetItem in assetItem: if newestItem.lastUpdateTime < singleAssetItem.lastUpdateTime: singleAssetItem.delete() else: newestItem.delete() newestItem = singleAssetItem assetItem = newestItem cache.globalCache.update("assetCache", match, [assetItem]) assetChanged = True else: assetItem = assetItem[0] if assetItem._id == "": actionResult["result"] = False actionResult["msg"] = "Asset not yet added" actionResult["rc"] = 404 return actionResult lastSeen = None for source in assetItem.lastSeen: if source["source"] == updateSource: if "lastUpdate" not in lastSeen: lastSeen["lastUpdate"] = 0 lastSeen = source break if not lastSeen: assetItem.lastSeen.append({ "source": updateSource, "lastUpdate": 0 }) lastSeen = assetItem.lastSeen[-1] # Converting millsecond int epoch into epoch floats currentTimestamp = lastSeen["lastUpdate"] if len(str(currentTimestamp).split(".")[0]) == 13: currentTimestamp = currentTimestamp / 1000 if len(str(updateTime).split(".")[0]) == 13: updateTime = updateTime / 1000 newTimestamp = None if updateTime: try: if updateTime < currentTimestamp: newTimestamp = False else: newTimestamp = updateTime except (KeyError, ValueError): pass if newTimestamp == None: newTimestamp = time.time() if newTimestamp != False: try: if (time.time() - currentTimestamp) < self.delayedUpdate: actionResult["result"] = False actionResult["msg"] = "Delay time not met" actionResult["rc"] = 300 return actionResult except KeyError: pass assetChanged = True if newTimestamp > assetItem.lastSeenTimestamp: assetItem.lastSeenTimestamp = newTimestamp if self.replaceExisting: lastSeen = assetFields else: for key, value in assetFields.items(): lastSeen[key] = value lastSeen["priority"] = self.sourcePriority lastSeen["lastUpdate"] = newTimestamp # Working out priority and define fields if assetChanged: foundValues = {} now = time.time() blacklist = ["lastUpdate", "priority"] for sourceValue in assetItem.lastSeen: for key, value in sourceValue.items(): if key not in blacklist: if key not in foundValues: foundValues[key] = { "value": value, "priority": sourceValue["priority"] } else: if sourceValue["priority"] < foundValues[key][ "priority"] and ( sourceValue["lastUpdate"] + self.sourcePriorityMaxAge) > now: foundValues[key] = { "value": value, "priority": sourceValue["priority"] } assetItem.fields = {} for key, value in foundValues.items(): assetItem.fields[key] = value["value"] assetItem.bulkUpdate(["lastSeen", "fields"], self.bulkClass) actionResult["result"] = True actionResult["msg"] = "Updated asset" actionResult["rc"] = 302 return actionResult actionResult["result"] = True actionResult["msg"] = "Nothing to do" actionResult["rc"] = 304 return actionResult
def getSearchObject(match, sessionData, search, fields): return asset._asset().query(query=search, fields=fields)["results"]