def test_scanResultEvent_argument_instanceId_of_invalid_type_should_raise_TypeError(self):
"""
Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False)
"""
sfdb = SpiderFootDb(self.default_options, False)
event_type = 'ALL'
filter_fp = None
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
sfdb.scanResultEvent(invalid_type, event_type, filter_fp)
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
for id in ids.split(','):
scan_name = dbh.scanInstanceGet(id)[0]
if scan_name not in scaninfo:
scaninfo[scan_name] = []
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo[scan_name].append({
"data": event_data,
"type": event_type,
"source_module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"lastseen": lastseen
})
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json"
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo)
def scaneventresultexportmulti(self, ids, dialect="excel"):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow([
"Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"
])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
scaninfo[row[12]][0], lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
def scaneventresults(self, id, eventType, filterfp=False):
"""Event results for a scan
Args:
id (str): scan ID
eventType (str): filter by event type
filterfp: TBD
"""
retdata = []
dbh = SpiderFootDb(self.config)
try:
data = dbh.scanResultEvent(id, eventType, filterfp)
except Exception:
return json.dumps(retdata)
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
escapeddata = html.escape(row[1])
escapedsrc = html.escape(row[2])
retdata.append([
lastseen, escapeddata, escapedsrc, row[3], row[5], row[6],
row[7], row[8], row[13], row[14], row[4]
])
return json.dumps(retdata, ensure_ascii=False)
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
for id in ids.split(','):
scan_name = dbh.scanInstanceGet(id)[0]
if scan_name not in scaninfo:
scaninfo[scan_name] = []
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo[scan_name].append({
"data": event_data,
"type": event_type,
"source_module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"lastseen": lastseen
})
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json"
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo)
def scaneventresultexport(self, id, type, dialect="excel"):
"""Get scan event result data in CSV format
Args:
id (str): scan ID
type (str): TBD
dialect (str): TBD
Returns:
string: results in CSV format
"""
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue().encode("utf-8")
def test_scan_result_event_should_return_a_list(self):
"""
Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False)
"""
sfdb = SpiderFootDb(self.default_options, False)
scan_result_event = sfdb.scanResultEvent("", "", False)
self.assertIsInstance(scan_result_event, list)
def scanelementtypediscovery(self, id, eventType):
keepGoing = True
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
# Get the first round of source IDs for the leafs
nextIds = list()
for row in leafSet:
# these must be unique values!
parentId = row[9]
childId = row[8]
datamap[childId] = row
if pc.has_key(parentId):
if childId not in pc[parentId]:
pc[parentId].append(childId)
else:
pc[parentId] = [ childId ]
# parents of the leaf set
if parentId not in nextIds:
nextIds.append(parentId)
while keepGoing:
parentSet = dbh.scanElementSources(id, nextIds)
nextIds = list()
keepGoing = False
for row in parentSet:
parentId = row[9]
childId = row[8]
datamap[childId] = row
#print childId + " = " + str(row)
if pc.has_key(parentId):
if childId not in pc[parentId]:
pc[parentId].append(childId)
else:
pc[parentId] = [ childId ]
if parentId not in nextIds:
nextIds.append(parentId)
# Prevent us from looping at root
if parentId != "ROOT":
keepGoing = True
datamap[parentId] = row
#print pc
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
def scaneventresults(self, id, eventType):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, eventType)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escaped = cgi.escape(row[1])
retdata.append([lastseen, escaped, row[2], row[3], row[5], row[6], row[7]])
return json.dumps(retdata, ensure_ascii=False)
def scaneventresults(self, id, eventType):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, eventType)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escaped = cgi.escape(row[1])
# For debug
retdata.append([lastseen, escaped, row[2], row[3], row[5], row[6], row[7], row[8]])
#retdata.append([lastseen, escaped, row[2], row[3], row[5], row[6], row[7]])
return json.dumps(retdata, ensure_ascii=False)
def test_scan_result_event_invalid_event_type_should_raise(self):
"""
Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False)
"""
sfdb = SpiderFootDb(self.default_options, False)
invalid_types = [None, list(), dict()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
scan_result_event = sfdb.scanResultEvent(
"", invalid_type, None)
def scaneventresultexport(self, id, type, dialect="excel"):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "Data"])
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
parser.writerow([lastseen, str(row[4]), str(row[3]), str(row[2]), str(row[1])])
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
def scanviz(self, id, gexf="0"):
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf([root], "SpiderFoot Export", data)
else:
return sf.buildGraphJson([root], data)
def scaneventresults(self, id, eventType, filterfp=False):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, eventType, filterfp)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
escapeddata = html.escape(row[1])
escapedsrc = html.escape(row[2])
retdata.append([
lastseen, escapeddata, escapedsrc, row[3], row[5], row[6],
row[7], row[8], row[13], row[14], row[4]
])
return json.dumps(retdata, ensure_ascii=False)
def scanviz(self, id, gexf="0"):
types = list()
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf([root], "SpiderFoot Export", data)
else:
return sf.buildGraphJson([root], data)
def scaneventresultexport(self, id, type):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
blob = "\"Updated\",\"Type\",\"Module\",\"Source\",\"Data\"\n"
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escapedData = cgi.escape(row[1].replace("\n", "#LB#").replace("\r", "#LB#"))
escapedSrc = cgi.escape(row[2].replace("\n", "#LB#").replace("\r", "#LB#"))
blob = blob + "\"" + lastseen + "\",\"" + row[4] + "\",\""
blob = blob + row[3] + "\",\"" + escapedSrc + "\",\"" + escapedData + "\"\n"
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return blob
def scaneventresultexport(self, id, type, dialect="excel"):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "Data"])
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
parser.writerow([lastseen, str(row[4]), str(row[3]), str(row[2]), datafield])
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = list()
scan_name = ""
for id in ids.split(','):
scan = dbh.scanInstanceGet(id)
if scan is None:
continue
scan_name = scan[0]
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo.append({
"data": event_data,
"event_type": event_type,
"module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"last_seen": lastseen,
"scan_name": scan_name,
"scan_target": scan[1]
})
if len(ids.split(',')) > 1 or scan_name == "":
fname = "SpiderFoot.json"
else:
fname = scan_name + "-SpiderFoot.json"
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=" + fname
cherrypy.response.headers[
'Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo).encode("utf-8")
def scanelementtypediscovery(self, id, eventType):
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
def scanelementtypediscovery(self, id, eventType):
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
def scanvizmulti(self, ids, gexf="1"):
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = list()
roots = list()
for id in ids.split(','):
data = data + dbh.scanResultEvent(id, filterFp=True)
roots.append(dbh.scanInstanceGet(id)[1])
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf(roots, "SpiderFoot Export", data)
else:
# Not implemented yet
return None
def scanvizmulti(self, ids, gexf="1"):
types = list()
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = list()
roots = list()
for id in ids.split(','):
data = data + dbh.scanResultEvent(id, filterFp=True)
roots.append(dbh.scanInstanceGet(id)[1])
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf(roots, "SpiderFoot Export", data)
else:
# Not implemented yet
return None
def scaneventresultexportmulti(self, ids, dialect="excel"):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Scan Name", "Updated", "Type", "Module", "Source", "Data"])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
parser.writerow([scaninfo[row[12]][0], lastseen, str(row[4]), str(row[3]), str(row[2]), datafield])
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
