def test_scanResultEvent_argument_eventType_of_invalid_type_should_raise_TypeError(self):
"""
Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False)
"""
sfdb = SpiderFootDb(self.default_options, False)
instance_id = "example instance id"
invalid_types = [None, list(), dict()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError):
sfdb.scanResultEvent(instance_id, invalid_type, None)
def scaneventresultexportmulti(self, ids, dialect="excel"):
"""Get scan event result data in CSV format for multiple scans
Args:
ids (str): comma separated list of scan IDs
dialect (str): TBD
Returns:
string: results in CSV format
"""
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
parser.writerow([scaninfo[row[12]][0], lastseen, str(row[4]), str(row[3]),
str(row[2]), row[13], datafield])
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue().encode('utf-8')
def scaneventresultexport(self, id, type, dialect="excel"):
"""Get scan event result data in CSV format
Args:
id (str): scan ID
type (str): TBD
dialect (str): TBD
Returns:
string: results in CSV format
"""
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
parser.writerow([lastseen, str(row[4]), str(row[3]), str(row[2]), row[13], datafield])
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue().encode('utf-8')
def scanelementtypediscovery(self, id, eventType):
"""scan element type discovery
Args:
id: TBD
eventType (str): filter by event type
"""
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata).encode('utf-8')
def scaneventresults(self, id, eventType, filterfp=False):
"""Event results for a scan
Args:
id (str): scan ID
eventType (str): filter by event type
filterfp: TBD
"""
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
retdata = []
dbh = SpiderFootDb(self.config)
try:
data = dbh.scanResultEvent(id, eventType, filterfp)
except Exception:
return json.dumps(retdata).encode('utf-8')
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escapeddata = html.escape(row[1])
escapedsrc = html.escape(row[2])
retdata.append([lastseen, escapeddata, escapedsrc,
row[3], row[5], row[6], row[7], row[8],
row[13], row[14], row[4]])
return json.dumps(retdata).encode('utf-8')
def scanvizmulti(self, ids, gexf="1"):
"""Export entities results from multiple scans in GEXF format
Args:
ids (str): scan IDs
gexf (str): TBD
Returns:
string: GEXF data
"""
dbh = SpiderFootDb(self.config)
data = list()
roots = list()
if not ids:
return None
for id in ids.split(','):
data = data + dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
if scan:
roots.append(scan[1])
if gexf == "0":
# Not implemented yet
return None
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return SpiderFootHelpers.buildGraphGexf(roots, "SpiderFoot Export", data)
def scanviz(self, id, gexf="0"):
"""Export entities from scan results for visualising
Args:
id (str): scan ID
gexf (str): TBD
Returns:
string: GEXF data
"""
if not id:
return None
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
if not scan:
return None
root = scan[1]
if gexf == "0":
return SpiderFootHelpers.buildGraphJson([root], data)
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return SpiderFootHelpers.buildGraphGexf([root], "SpiderFoot Export", data)
def scanelementtypediscovery(self, id, eventType):
"""Scan element type discovery.
Args:
id (str): scan ID
eventType (str): filter by event type
Returns:
str: JSON
"""
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
retdata = dict()
# Get the events we will be tracing back from
try:
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
except Exception:
return retdata
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata['tree'] = SpiderFootHelpers.dataParentChildToTree(pc)
retdata['data'] = datamap
return retdata
def test_scanResultEvent_should_return_a_list(self):
"""
Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False)
"""
sfdb = SpiderFootDb(self.default_options, False)
instance_id = "example instance id"
scan_result_event = sfdb.scanResultEvent(instance_id, "", False)
self.assertIsInstance(scan_result_event, list)
def scanexportjsonmulti(self, ids):
"""Get scan event result data in JSON format for multiple scans
Args:
ids (str): comma separated list of scan IDs
Returns:
string: results in CSV format
"""
dbh = SpiderFootDb(self.config)
scaninfo = list()
scan_name = ""
for id in ids.split(','):
scan = dbh.scanInstanceGet(id)
if scan is None:
continue
scan_name = scan[0]
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo.append({
"data": event_data,
"event_type": event_type,
"module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"last_seen": lastseen,
"scan_name": scan_name,
"scan_target": scan[1]
})
if len(ids.split(',')) > 1 or scan_name == "":
fname = "SpiderFoot.json"
else:
fname = scan_name + "-SpiderFoot.json"
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=" + fname
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo).encode('utf-8')
def scanviz(self, id, gexf="0"):
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf([root], "SpiderFoot Export", data)
else:
return sf.buildGraphJson([root], data)
def scanvizmulti(self, ids, gexf="1"):
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = list()
roots = list()
for id in ids.split(','):
data = data + dbh.scanResultEvent(id, filterFp=True)
roots.append(dbh.scanInstanceGet(id)[1])
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf(roots, "SpiderFoot Export", data)
else:
# Not implemented yet
return None
def scaneventresults(self, id, eventType, filterfp=False):
"""Event results for a scan.
Args:
id (str): scan ID
eventType (str): filter by event type
filterfp: TBD
Returns:
str: scan results as JSON
"""
retdata = []
dbh = SpiderFootDb(self.config)
try:
data = dbh.scanResultEvent(id, eventType, filterfp)
except Exception:
return retdata
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
retdata.append([
lastseen,
html.escape(row[1]),
html.escape(row[2]),
row[3],
row[5],
row[6],
row[7],
row[8],
row[13],
row[14],
row[4]
])
return retdata
