def stopscanmulti(self, ids):
global globalScanStatus # running scans
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if globalScanStatus.getStatus(id) == "FINISHED" or scaninfo[5] == "FINISHED":
error.append("Scan '" + scaninfo[0] + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and (globalScanStatus.getStatus(id) == "ABORTED" or scaninfo[5] == "ABORTED"):
error.append("Scan '" + scaninfo[0] + "' is already aborted.")
errState = True
if not errState and globalScanStatus.getStatus(id) is None:
error.append("Scan '" + scaninfo[0] + "' is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
errState = True
if not errState:
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST', stoppedscan=True,
errors=error, docroot=self.docroot)
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[modName, modOpt] = key.split(':')
if modName not in self.config['__modules__'].keys():
continue
if modOpt not in self.config['__modules__'][modName]['optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName]['optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
def searchBase(self, id=None, eventType=None, value=None):
regex = ""
if [id, eventType, value].count('') == 2 or \
[id, eventType, value].count(None) == 2:
return None
if value.startswith("/") and value.endswith("/"):
regex = value[1:len(value) - 1]
value = ""
value = value.replace('*', '%')
if value in [None, ""] and regex in [None, ""]:
value = "%"
regex = ""
dbh = SpiderFootDb(self.config)
criteria = {
'scan_id': None if id == '' else id,
'type': None if eventType == '' else eventType,
'value': None if value == '' else value,
'regex': None if regex == '' else regex
}
data = dbh.search(criteria)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escapeddata = cgi.escape(row[1])
escapedsrc = cgi.escape(row[2])
retdata.append([lastseen, escapeddata, escapedsrc,
row[3], row[5], row[6], row[7], row[8], row[10],
row[11], row[4], row[13], row[14]])
return retdata
def savesettingsraw(self, allopts, token):
if str(token) != str(self.token):
return json.dumps(["ERROR", "Invalid token (" + str(self.token) + ")."])
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return json.dumps(["ERROR", "Processing one or more of your inputs failed: " + str(e)])
return json.dumps(["SUCCESS", ""])
def newscan(self):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
modules=self.config['__modules__'], scanname="",
selectedmods="", scantarget="")
def resultsetfp(self, id, resultids, fp):
dbh = SpiderFootDb(self.config)
if fp not in ["0", "1"]:
return json.dumps(["ERROR", "No FP flag set or not set correctly."])
ids = json.loads(resultids)
if not ids:
return json.dumps(["ERROR", "No IDs supplied."])
# Cannot set FPs if a scan is not completed
status = dbh.scanInstanceGet(id)
if status[5] not in [ "ABORTED", "FINISHED", "ERROR-FAILED" ]:
return json.dumps(["WARNING", "Scan must be in a finished state when " + \
"setting False Positives."])
# Make sure the user doesn't set something as non-FP when the
# parent is set as an FP.
if fp == "0":
data = dbh.scanElementSourcesDirect(id, ids)
for row in data:
if str(row[14]) == "1":
return json.dumps(["WARNING",
"You cannot unset an element as False Positive " + \
"if a parent element is still False Positive."]);
# Set all the children as FPs too.. it's only logical afterall, right?
childs = dbh.scanElementChildrenAll(id, ids)
allIds = ids + childs
ret = dbh.scanResultsUpdateFP(id, allIds, fp)
if not ret:
return json.dumps(["ERROR", "Exception encountered."])
else:
return json.dumps(["SUCCESS", ""])
def savesettings(self, allopts, token):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error("Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config, pageid='SETTINGS', updated=True,
docroot=self.docroot, token=self.token)
def clonescan(self, id):
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
targetType = sf.targetType(scantarget)
if targetType == None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
modules=self.config['__modules__'], selectedmods=modlist,
scanname=unicode(scanname, 'utf-8', errors='replace'),
scantarget=unicode(scantarget, 'utf-8', errors='replace'))
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
for id in ids.split(','):
scan_name = dbh.scanInstanceGet(id)[0]
if scan_name not in scaninfo:
scaninfo[scan_name] = []
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo[scan_name].append({
"data": event_data,
"type": event_type,
"source_module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"lastseen": lastseen
})
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json"
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo)
def scaninfo(self, id):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res == None:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0], status=res[5], pageid='SCANLIST')
def scaneventresultsunique(self, id, eventType, filterfp=False):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEventUnique(id, eventType, filterfp)
retdata = []
for row in data:
escaped = cgi.escape(row[0])
retdata.append([escaped, row[1], row[2]])
return json.dumps(retdata, ensure_ascii=False)
def scanlog(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanLogs(id)
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]/1000))
retdata.append([generated, row[1], row[2], cgi.escape(row[3])])
return json.dumps(retdata)
def scansummary(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultSummary(id)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2]))
retdata.append([row[0], row[1], lastseen, row[3], row[4]])
return json.dumps(retdata)
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(config)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(config)
self.config = sf.configUnserialize(dbh.configGet(), config)
def scanelementtypediscovery(self, id, eventType):
keepGoing = True
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
# Get the first round of source IDs for the leafs
nextIds = list()
for row in leafSet:
# these must be unique values!
parentId = row[9]
childId = row[8]
datamap[childId] = row
if pc.has_key(parentId):
if childId not in pc[parentId]:
pc[parentId].append(childId)
else:
pc[parentId] = [ childId ]
# parents of the leaf set
if parentId not in nextIds:
nextIds.append(parentId)
while keepGoing:
parentSet = dbh.scanElementSources(id, nextIds)
nextIds = list()
keepGoing = False
for row in parentSet:
parentId = row[9]
childId = row[8]
datamap[childId] = row
#print childId + " = " + str(row)
if pc.has_key(parentId):
if childId not in pc[parentId]:
pc[parentId].append(childId)
else:
pc[parentId] = [ childId ]
if parentId not in nextIds:
nextIds.append(parentId)
# Prevent us from looping at root
if parentId != "ROOT":
keepGoing = True
datamap[parentId] = row
#print pc
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
def scanstatus(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceGet(id)
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2]))
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3]))
ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4]))
retdata = [data[0], data[1], created, started, ended, data[5]]
return json.dumps(retdata)
def scaninfo(self, id):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, disable_unicode=True, input_encoding='utf-8')
return templ.render(id=id, name=cgi.escape(res[0]), status=res[5], docroot=self.docroot,
pageid="SCANLIST")
def scaneventresults(self, id, eventType):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, eventType)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escaped = cgi.escape(row[1])
retdata.append([lastseen, escaped, row[2], row[3], row[5], row[6], row[7]])
return json.dumps(retdata, ensure_ascii=False)
def scanerrors(self, id, limit=None):
dbh = SpiderFootDb(self.config)
data = dbh.scanErrors(id, limit)
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000))
retdata.append([generated, row[1],
cgi.escape(unicode(row[2], errors='replace'))])
return json.dumps(retdata)
def startscan(self, scanname, scantarget, modulelist, typelist):
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "":
return self.error("Form incomplete.")
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
else:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'],
alreadyRunning=True, runningScan=thread.name[3:],
types=types, pageid="NEWSCAN")
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist,
self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=self.scanner.myId, name=scanname,
status=self.scanner.status, pageid="SCANLIST")
def eventtypes(self):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
ret = list()
for r in types:
ret.append([r[1], r[0]])
ret = sorted(ret, key=itemgetter(0))
return json.dumps(ret)
def scandelete(self, id, confirm=None):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res == None:
return self.error("Scan ID not found.")
if confirm != None:
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0])
def scaneventresultexport(self, id, type, dialect="excel"):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "Data"])
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
parser.writerow([lastseen, str(row[4]), str(row[3]), str(row[2]), str(row[1])])
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
def scandelete(self, id, confirm=None):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
return self.error("Scan ID not found.")
if confirm is not None:
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0], names=list(), ids=list(),
pageid="SCANLIST", docroot=self.docroot)
def scaneventresultexport(self, id, type):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
blob = "\"Updated\",\"Type\",\"Module\",\"Source\",\"Data\"\n"
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escapedData = cgi.escape(row[1].replace("\n", "#LB#").replace("\r", "#LB#"))
escapedSrc = cgi.escape(row[2].replace("\n", "#LB#").replace("\r", "#LB#"))
blob = blob + "\"" + lastseen + "\",\"" + row[4] + "\",\""
blob = blob + row[3] + "\",\"" + escapedSrc + "\",\"" + escapedData + "\"\n"
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return blob
def scanviz(self, id, gexf="0"):
types = list()
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf([root], "SpiderFoot Export", data)
else:
return sf.buildGraphJson([root], data)
def savesettings(self, allopts, token, configFile=None):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
if configFile: # configFile seems to get set even if a file isn't uploaded
if configFile.file:
contents = configFile.file.read()
try:
tmp = dict()
for line in contents.split("\n"):
if "=" not in line:
continue
l = line.strip().split("=")
if len(l) == 1:
l[1] = ""
tmp[l[0]] = l[1]
allopts = json.dumps(tmp)
except BaseException as e:
return self.error("Failed to parse input file. Was it generated from SpiderFoot? (" + str(e) + ")")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return self.error("Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config, pageid='SETTINGS', updated=True,
docroot=self.docroot, token=self.token)
def scanlist(self):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceList()
retdata = []
for row in data:
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[3]))
if row[4] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[4]))
else:
started = "Not yet"
if row[5] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[5]))
else:
finished = "Not yet"
retdata.append([row[0], row[1], row[2], created, started, finished, row[6], row[7]])
return json.dumps(retdata)
def scanelementtypediscovery(self, id, eventType):
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
def clonescan(self, id):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
modules=self.config['__modules__'], selectedmods=modlist,
scanname=scanname, scantarget=scantarget)
def scaneventresultexportmulti(self, ids, dialect="excel"):
"""Get scan event result data in CSV format for multiple scans
Args:
ids (str): comma separated list of scan IDs
dialect (str): TBD
Returns:
string: results in CSV format
"""
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow([
"Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"
])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
scaninfo[row[12]][0], lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue().encode("Utf-8")
def rerunscanmulti(self, ids):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType,
newId, modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True,
docroot=self.docroot,
pageid="SCANLIST")
def test_scan_result_summary_invalid_type_should_raise(self):
"""
Test scanResultSummary(self, instanceId, by="type")
"""
sfdb = SpiderFootDb(self.default_options, False)
invalid_types = [None, list(), dict()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
scan_results_summary = sfdb.scanResultSummary(invalid_type)
instance_id = "example instance id"
invalid_types = [None, list(), dict()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
scan_results_summary = sfdb.scanResultSummary(instance_id, None)
with self.assertRaises(ValueError) as cm:
scan_results_summary = sfdb.scanResultSummary(instance_id, "invalid filter type")
def scaneventresultexport(self, id, type, dialect="excel"):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "Data"])
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow(
[lastseen,
str(row[4]),
str(row[3]),
str(row[2]), datafield])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
def test_scanEventStore_argument_sfEvent_with_empty_module_property_value_should_raise_ValueError(self):
"""
Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)
"""
sfdb = SpiderFootDb(self.default_options, False)
event_type = 'ROOT'
event_data = 'example data'
module = ''
source_event = ''
source_event = SpiderFootEvent(event_type, event_data, module, source_event)
event_type = 'example event type'
event_data = 'example event data'
module = 'example module'
event = SpiderFootEvent(event_type, event_data, module, source_event)
instance_id = "example instance id"
with self.assertRaises(ValueError) as cm:
event.module = ''
sfdb.scanEventStore(instance_id, event)
def scandeletemulti(self, ids, confirm=None):
dbh = SpiderFootDb(self.config)
names = list()
for id in ids.split(','):
res = dbh.scanInstanceGet(id)
names.append(unicode(res[0], 'utf-8', errors='replace'))
if res is None:
return self.error("Scan ID not found (" + id + ").")
if res[5] in [ "RUNNING", "STARTING", "STARTED" ]:
return self.error("You cannot delete running scans.")
if confirm is not None:
for id in ids.split(','):
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect(self.docroot)
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=None, name=None, ids=ids.split(','), names=names,
pageid="SCANLIST", docroot=self.docroot)
def scaneventresultsunique(self, id, eventType, filterfp=False):
"""Unique event results for a scan
Args:
id (str): scan ID
eventType (str): filter by event type
filterfp: TBD
"""
dbh = SpiderFootDb(self.config)
retdata = []
try:
data = dbh.scanResultEventUnique(id, eventType, filterfp)
except Exception:
return json.dumps(retdata)
for row in data:
escaped = html.escape(row[0])
retdata.append([escaped, row[1], row[2]])
return json.dumps(retdata, ensure_ascii=False)
def test_search_invalid_criteria_should_raise(self):
"""
Test search(self, criteria, filterFp=False)
"""
sfdb = SpiderFootDb(self.default_options, False)
invalid_types = [None, "", list()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
search_results = sfdb.search(invalid_type, False)
criteria = {
'scan_id': None,
'type': None,
'value': None,
'regex': None
}
with self.assertRaises(ValueError) as cm:
search_results = sfdb.search(criteria, False)
def scansummary(self, id, by):
retdata = []
dbh = SpiderFootDb(self.config)
try:
scandata = dbh.scanResultSummary(id, by)
except:
return json.dumps(retdata)
try:
statusdata = dbh.scanInstanceGet(id)
except:
return json.dumps(retdata)
for row in scandata:
if row[0] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2]))
retdata.append([row[0], row[1], lastseen, row[3], row[4], statusdata[5]])
return json.dumps(retdata)
def savesettings(self, allopts, token):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error(
"Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config,
pageid='SETTINGS',
updated=True,
docroot=self.docroot,
token=self.token)
def stopscan(self, id, cli=None):
global globalScanStatus
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
if not cli:
return self.error("Invalid scan ID.")
else:
return json.dumps(["ERROR", "Invalid scan ID."])
if globalScanStatus.getStatus(id) is None:
if not cli:
return self.error("That scan is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
else:
return json.dumps(["ERROR", "Scan doesn't appear to be running."])
if globalScanStatus.getStatus(id) == "ABORTED":
if not cli:
return self.error("The scan is already aborted.")
else:
return json.dumps(["ERROR", "Scan already aborted."])
if not globalScanStatus.getStatus(id) == "RUNNING":
if not cli:
return self.error("The running scan is currently in the state '" + \
globalScanStatus.getStatus(id) + "', please try again later or restart " + \
" SpiderFoot.")
else:
return json.dumps(["ERROR", "Scan in an invalid state for stopping."])
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
if not cli:
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST', stoppedscan=True, docroot=self.docroot, errors=list())
else:
return json.dumps(["SUCCESS", ""])
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
for id in ids.split(','):
scan = dbh.scanInstanceGet(id)
if scan is None:
continue
scan_name = scan[0]
if scan_name not in scaninfo:
scaninfo[scan_name] = []
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo[scan_name].append({
"data": event_data,
"type": event_type,
"source_module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"lastseen": lastseen
})
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json"
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo)
def resultsetfp(self, id, resultids, fp):
dbh = SpiderFootDb(self.config)
if fp not in ["0", "1"]:
return json.dumps(
["ERROR", "No FP flag set or not set correctly."])
ids = json.loads(resultids)
if not ids:
return json.dumps(["ERROR", "No IDs supplied."])
# Cannot set FPs if a scan is not completed
status = dbh.scanInstanceGet(id)
if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]:
return json.dumps(["WARNING", "Scan must be in a finished state when " + \
"setting False Positives."])
# Make sure the user doesn't set something as non-FP when the
# parent is set as an FP.
if fp == "0":
data = dbh.scanElementSourcesDirect(id, ids)
for row in data:
if str(row[14]) == "1":
return json.dumps(["WARNING",
"You cannot unset an element as False Positive " + \
"if a parent element is still False Positive."])
# Set all the children as FPs too.. it's only logical afterall, right?
childs = dbh.scanElementChildrenAll(id, ids)
allIds = ids + childs
ret = dbh.scanResultsUpdateFP(id, allIds, fp)
if not ret:
return json.dumps(["ERROR", "Exception encountered."])
else:
return json.dumps(["SUCCESS", ""])
def savesettingsraw(self, allopts, token):
if str(token) != str(self.token):
return json.dumps(
["ERROR", "Invalid token (" + str(self.token) + ")."])
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return json.dumps([
"ERROR",
"Processing one or more of your inputs failed: " + str(e)
])
return json.dumps(["SUCCESS", ""])
def stopscan(self, id, cli=None):
"""Stop a scan.
Args:
id (str): scan ID
cli: TBD
"""
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if not scaninfo:
if cli:
return json.dumps(["ERROR", "Invalid scan ID."])
return self.error("Invalid scan ID.")
scanstatus = scaninfo[5]
if scanstatus == "ABORTED":
if cli:
return json.dumps(["ERROR", "Scan already aborted."])
return self.error("The scan is already aborted.")
if not scanstatus == "RUNNING":
if cli:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
return self.error(
"The running scan is currently in the state '%s', please try again later or restart SpiderFoot."
% scanstatus)
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
if cli:
return json.dumps(["SUCCESS", ""])
raise cherrypy.HTTPRedirect("/")
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
secure_headers = SecureHeaders()
cherrypy.config.update({
"tools.response_headers.on":
True,
"tools.response_headers.headers":
secure_headers.cherrypy()
})
print("")
print("")
print("*************************************************************")
print(" Use SpiderFoot by starting your web browser of choice and ")
print(" browse to http://" + addr + ":" +
str(self.config['__webport']) + self.docroot)
print("*************************************************************")
print("")
print("")
def scanerrors(self, id, limit=None):
"""Scan error data
Args:
id (str): scan ID
limit: TBD
"""
dbh = SpiderFootDb(self.config)
retdata = []
try:
data = dbh.scanErrors(id, limit)
except Exception:
return json.dumps(retdata)
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0] / 1000))
retdata.append([generated, row[1], html.escape(str(row[2]))])
return json.dumps(retdata)
def stopscanmulti(self, ids):
global globalScanStatus # running scans
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
return self.error("Invalid scan ID specified.")
scanname = str(scaninfo[0])
if globalScanStatus.getStatus(
id) == "FINISHED" or scaninfo[5] == "FINISHED":
error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and (globalScanStatus.getStatus(id) == "ABORTED"
or scaninfo[5] == "ABORTED"):
error.append("Scan '" + scanname + "' is already aborted.")
errState = True
if not errState and globalScanStatus.getStatus(id) is None:
error.append("Scan '" + scanname + "' is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
errState = True
if not errState:
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
errors=error,
docroot=self.docroot)
def clonescan(self, id):
"""
Clone an existing scan (pre-selected options in the newscan page)
Args:
id (str): scan ID to clone
"""
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
targetType = sf.targetType(scantarget)
if targetType is None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
selectedmods=modlist,
scanname=str(scanname),
scantarget=str(scantarget))
def clonescan(self, id):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
selectedmods=modlist,
scanname=scanname,
scantarget=scantarget)
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(config)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(config)
self.config = sf.configUnserialize(dbh.configGet(), config)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
print ""
print ""
print "*************************************************************"
print " Use SpiderFoot by starting your web browser of choice and "
print " browse to http://" + addr + ":" + str(self.config['__webport'])
print "*************************************************************"
print ""
print ""
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[modName, modOpt] = key.split(':')
if modName not in self.config['__modules__'].keys():
continue
if modOpt not in self.config['__modules__'][modName][
'optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName][
'optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if not meta:
return json.dumps([])
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
def scanlist(self):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceList()
retdata = []
for row in data:
created = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[3]))
if row[4] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[4]))
else:
started = "Not yet"
if row[5] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[5]))
else:
finished = "Not yet"
retdata.append([
row[0], row[1], row[2], created, started, finished, row[6],
row[7]
])
return json.dumps(retdata)
def query(self, query):
data = None
dbh = SpiderFootDb(self.config)
if not query.lower().startswith("select"):
return json.dumps([
"ERROR", "Non-SELECTs are unpredictable and not recommended."
])
try:
ret = dbh.dbh.execute(query)
data = ret.fetchall()
except BaseException as e:
return json.dumps(["ERROR", str(e)])
return json.dumps(data)
def searchBase(self, id=None, eventType=None, value=None):
regex = ""
if [id, eventType, value].count('') == 2 or \
[id, eventType, value].count(None) == 2:
return None
if value.startswith("/") and value.endswith("/"):
regex = value[1:len(value) - 1]
value = ""
value = value.replace('*', '%')
if value in [None, ""] and regex in [None, ""]:
value = "%"
regex = ""
dbh = SpiderFootDb(self.config)
criteria = {
'scan_id': None if id == '' else id,
'type': None if eventType == '' else eventType,
'value': None if value == '' else value,
'regex': None if regex == '' else regex
}
data = dbh.search(criteria)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
escapeddata = cgi.escape(row[1])
escapedsrc = cgi.escape(row[2])
retdata.append([
lastseen, escapeddata, escapedsrc, row[3], row[5], row[6],
row[7], row[8], row[10], row[11], row[4], row[13], row[14]
])
return retdata
def scanelementtypediscovery(self, id, eventType):
"""scan element type discovery
Args:
id: TBD
eventType (str): filter by event type
"""
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
def scandelete(self, id, confirm=None, raw=False):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
if not raw:
return self.error("Scan ID not found.")
else:
return json.dumps(["ERROR", "Scan ID not found."])
if confirm is not None:
dbh.scanInstanceDelete(id)
if not raw:
raise cherrypy.HTTPRedirect("/")
else:
return json.dumps(["SUCCESS", ""])
else:
templ = Template(filename='dyn/scandelete.tmpl',
lookup=self.lookup)
return templ.render(id=id,
name=str(res[0]),
names=list(),
ids=list(),
pageid="SCANLIST",
docroot=self.docroot)
def query(self, query):
data = None
dbh = SpiderFootDb(self.config)
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
if not query.lower().startswith("select"):
return json.dumps(["ERROR", "Non-SELECTs are unpredictable and not recommended."])
try:
ret = dbh.dbh.execute(query)
data = ret.fetchall()
except BaseException as e:
return json.dumps(["ERROR", str(e)])
return json.dumps(data)
def savesettings(self, allopts, token, configFile=None):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
if configFile: # configFile seems to get set even if a file isn't uploaded
if configFile.file:
contents = configFile.file.read()
if type(contents) == bytes:
contents = contents.decode("utf-8")
try:
tmp = dict()
for line in contents.split("\n"):
if "=" not in line:
continue
l = line.strip().split("=")
if len(l) == 1:
l[1] = ""
tmp[l[0]] = l[1]
allopts = json.dumps(tmp)
except BaseException as e:
return self.error(
"Failed to parse input file. Was it generated from SpiderFoot? ("
+ str(e) + ")")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return self.error(
"Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.SystemRandom().randint(0, 99999999)
return templ.render(opts=self.config,
pageid='SETTINGS',
updated=True,
docroot=self.docroot,
token=self.token)
def test_config_set_invalid_optmap_should_raise(self):
"""
Test configSet(self, optMap=dict())
"""
sfdb = SpiderFootDb(self.default_options, False)
invalid_types = [None, "", list()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
scan_result_event = sfdb.scanResultEventUnique("", invalid_type, None)
sfdb.configSet(invalid_type)
with self.assertRaises(ValueError) as cm:
sfdb.configSet(dict())