def rerunscan(self, id):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, newId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=newId,
name=scanname,
docroot=self.docroot,
status=globalScanStatus.getStatus(newId),
pageid="SCANLIST")
def rerunscan(self, id):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# It must then be a name, as a re-run scan should always have a clean
# target.
targetType = "HUMAN_NAME"
if targetType != "HUMAN_NAME":
scantarget = scantarget.lower()
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget, targetType, newId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=newId, name=unicode(scanname, 'utf-8', errors='replace'), docroot=self.docroot,
status=globalScanStatus.getStatus(newId), pageid="SCANLIST")
def scaneventresultexportmulti(self, ids, dialect="excel"):
"""Get scan event result data in CSV format for multiple scans
Args:
ids (str): comma separated list of scan IDs
dialect (str): TBD
Returns:
string: results in CSV format
"""
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow([
"Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"
])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
scaninfo[row[12]][0], lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue().encode("Utf-8")
def test_scanEventStore_argument_sfEvent_with_empty_module_property_value_should_raise_ValueError(self):
"""
Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)
"""
sfdb = SpiderFootDb(self.default_options, False)
event_type = 'ROOT'
event_data = 'example data'
module = ''
source_event = ''
source_event = SpiderFootEvent(event_type, event_data, module, source_event)
event_type = 'example event type'
event_data = 'example event data'
module = 'example module'
event = SpiderFootEvent(event_type, event_data, module, source_event)
instance_id = "example instance id"
with self.assertRaises(ValueError) as cm:
event.module = ''
sfdb.scanEventStore(instance_id, event)
def scaneventresultexport(self, id, type, dialect="excel"):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "Data"])
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow(
[lastseen,
str(row[4]),
str(row[3]),
str(row[2]), datafield])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
def test_search_invalid_criteria_should_raise(self):
"""
Test search(self, criteria, filterFp=False)
"""
sfdb = SpiderFootDb(self.default_options, False)
invalid_types = [None, "", list()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
search_results = sfdb.search(invalid_type, False)
criteria = {
'scan_id': None,
'type': None,
'value': None,
'regex': None
}
with self.assertRaises(ValueError) as cm:
search_results = sfdb.search(criteria, False)
def scansummary(self, id, by):
retdata = []
dbh = SpiderFootDb(self.config)
try:
scandata = dbh.scanResultSummary(id, by)
except:
return json.dumps(retdata)
try:
statusdata = dbh.scanInstanceGet(id)
except:
return json.dumps(retdata)
for row in scandata:
if row[0] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2]))
retdata.append([row[0], row[1], lastseen, row[3], row[4], statusdata[5]])
return json.dumps(retdata)
def query(self, query):
data = None
dbh = SpiderFootDb(self.config)
cherrypy.response.headers[
'Content-Type'] = "application/json; charset=utf-8"
if not query.lower().startswith("select"):
return json.dumps([
"ERROR", "Non-SELECTs are unpredictable and not recommended."
])
try:
ret = dbh.dbh.execute(query)
data = ret.fetchall()
columnNames = [c[0] for c in dbh.dbh.description]
data = [dict(zip(columnNames, row)) for row in data]
except BaseException as e:
return json.dumps(["ERROR", str(e)])
return json.dumps(data)
def scandeletemulti(self, ids, confirm=None):
dbh = SpiderFootDb(self.config)
names = list()
for id in ids.split(','):
res = dbh.scanInstanceGet(id)
names.append(unicode(res[0], 'utf-8', errors='replace'))
if res is None:
return self.error("Scan ID not found (" + id + ").")
if res[5] in [ "RUNNING", "STARTING", "STARTED" ]:
return self.error("You cannot delete running scans.")
if confirm is not None:
for id in ids.split(','):
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect(self.docroot)
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=None, name=None, ids=ids.split(','), names=names,
pageid="SCANLIST", docroot=self.docroot)
def test_scan_result_summary_invalid_type_should_raise(self):
"""
Test scanResultSummary(self, instanceId, by="type")
"""
sfdb = SpiderFootDb(self.default_options, False)
invalid_types = [None, list(), dict()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
scan_results_summary = sfdb.scanResultSummary(invalid_type)
instance_id = "example instance id"
invalid_types = [None, list(), dict()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
scan_results_summary = sfdb.scanResultSummary(instance_id, None)
with self.assertRaises(ValueError) as cm:
scan_results_summary = sfdb.scanResultSummary(instance_id, "invalid filter type")
def scaneventresultsunique(self, id, eventType, filterfp=False):
"""Unique event results for a scan
Args:
id (str): scan ID
eventType (str): filter by event type
filterfp: TBD
"""
dbh = SpiderFootDb(self.config)
retdata = []
try:
data = dbh.scanResultEventUnique(id, eventType, filterfp)
except Exception:
return json.dumps(retdata)
for row in data:
escaped = html.escape(row[0])
retdata.append([escaped, row[1], row[2]])
return json.dumps(retdata, ensure_ascii=False)
def stopscan(self, id, cli=None):
"""Stop a scan.
Args:
id (str): scan ID
cli: TBD
"""
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if not scaninfo:
if cli:
return json.dumps(["ERROR", "Invalid scan ID."])
return self.error("Invalid scan ID.")
scanstatus = scaninfo[5]
if scanstatus == "ABORTED":
if cli:
return json.dumps(["ERROR", "Scan already aborted."])
return self.error("The scan is already aborted.")
if not scanstatus == "RUNNING":
if cli:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
return self.error(
"The running scan is currently in the state '%s', please try again later or restart SpiderFoot."
% scanstatus)
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
if cli:
return json.dumps(["SUCCESS", ""])
raise cherrypy.HTTPRedirect("/")
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
for id in ids.split(','):
scan = dbh.scanInstanceGet(id)
if scan is None:
continue
scan_name = scan[0]
if scan_name not in scaninfo:
scaninfo[scan_name] = []
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo[scan_name].append({
"data": event_data,
"type": event_type,
"source_module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"lastseen": lastseen
})
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json"
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo)
def stopscan(self, id, cli=None):
global globalScanStatus
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
if not cli:
return self.error("Invalid scan ID.")
else:
return json.dumps(["ERROR", "Invalid scan ID."])
if globalScanStatus.getStatus(id) is None:
if not cli:
return self.error("That scan is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
else:
return json.dumps(["ERROR", "Scan doesn't appear to be running."])
if globalScanStatus.getStatus(id) == "ABORTED":
if not cli:
return self.error("The scan is already aborted.")
else:
return json.dumps(["ERROR", "Scan already aborted."])
if not globalScanStatus.getStatus(id) == "RUNNING":
if not cli:
return self.error("The running scan is currently in the state '" + \
globalScanStatus.getStatus(id) + "', please try again later or restart " + \
" SpiderFoot.")
else:
return json.dumps(["ERROR", "Scan in an invalid state for stopping."])
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
if not cli:
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST', stoppedscan=True, docroot=self.docroot, errors=list())
else:
return json.dumps(["SUCCESS", ""])
def clonescan(self, id):
"""
Clone an existing scan (pre-selected options in the newscan page)
Args:
id (str): scan ID to clone
"""
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
targetType = sf.targetType(scantarget)
if targetType is None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
selectedmods=modlist,
scanname=str(scanname),
scantarget=str(scantarget))
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(config)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(config)
self.config = sf.configUnserialize(dbh.configGet(), config)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
print ""
print ""
print "*************************************************************"
print " Use SpiderFoot by starting your web browser of choice and "
print " browse to http://" + addr + ":" + str(self.config['__webport'])
print "*************************************************************"
print ""
print ""
def scanerrors(self, id, limit=None):
"""Scan error data
Args:
id (str): scan ID
limit: TBD
"""
dbh = SpiderFootDb(self.config)
retdata = []
try:
data = dbh.scanErrors(id, limit)
except Exception:
return json.dumps(retdata)
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0] / 1000))
retdata.append([generated, row[1], html.escape(str(row[2]))])
return json.dumps(retdata)
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
secure_headers = SecureHeaders()
cherrypy.config.update({
"tools.response_headers.on":
True,
"tools.response_headers.headers":
secure_headers.cherrypy()
})
print("")
print("")
print("*************************************************************")
print(" Use SpiderFoot by starting your web browser of choice and ")
print(" browse to http://" + addr + ":" +
str(self.config['__webport']) + self.docroot)
print("*************************************************************")
print("")
print("")
def clonescan(self, id):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
selectedmods=modlist,
scanname=scanname,
scantarget=scantarget)
def stopscanmulti(self, ids):
global globalScanStatus # running scans
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
return self.error("Invalid scan ID specified.")
scanname = str(scaninfo[0])
if globalScanStatus.getStatus(
id) == "FINISHED" or scaninfo[5] == "FINISHED":
error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and (globalScanStatus.getStatus(id) == "ABORTED"
or scaninfo[5] == "ABORTED"):
error.append("Scan '" + scanname + "' is already aborted.")
errState = True
if not errState and globalScanStatus.getStatus(id) is None:
error.append("Scan '" + scanname + "' is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
errState = True
if not errState:
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
errors=error,
docroot=self.docroot)
def savesettings(self, allopts):
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(useropts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error("Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
return templ.render(opts=self.config, pageid='SETTINGS', updated=True)
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[modName, modOpt] = key.split(':')
if modName not in self.config['__modules__'].keys():
continue
if modOpt not in self.config['__modules__'][modName][
'optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName][
'optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if not meta:
return json.dumps([])
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
def resultsetfp(self, id, resultids, fp):
dbh = SpiderFootDb(self.config)
if fp not in ["0", "1"]:
return json.dumps(
["ERROR", "No FP flag set or not set correctly."])
ids = json.loads(resultids)
if not ids:
return json.dumps(["ERROR", "No IDs supplied."])
# Cannot set FPs if a scan is not completed
status = dbh.scanInstanceGet(id)
if not status:
return self.error("Invalid scan ID: %s" % id)
if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]:
return json.dumps(["WARNING", "Scan must be in a finished state when " + \
"setting False Positives."])
# Make sure the user doesn't set something as non-FP when the
# parent is set as an FP.
if fp == "0":
data = dbh.scanElementSourcesDirect(id, ids)
for row in data:
if str(row[14]) == "1":
return json.dumps(["WARNING",
"You cannot unset an element as False Positive " + \
"if a parent element is still False Positive."])
# Set all the children as FPs too.. it's only logical afterall, right?
childs = dbh.scanElementChildrenAll(id, ids)
allIds = ids + childs
ret = dbh.scanResultsUpdateFP(id, allIds, fp)
if not ret:
return json.dumps(["ERROR", "Exception encountered."])
else:
return json.dumps(["SUCCESS", ""])
def scanlist(self):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceList()
retdata = []
for row in data:
created = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[3]))
if row[4] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[4]))
else:
started = "Not yet"
if row[5] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[5]))
else:
finished = "Not yet"
retdata.append([
row[0], row[1], row[2], created, started, finished, row[6],
row[7]
])
return json.dumps(retdata)
def searchBase(self, id=None, eventType=None, value=None):
regex = ""
if [id, eventType, value].count('') == 2 or \
[id, eventType, value].count(None) == 2:
return None
if value.startswith("/") and value.endswith("/"):
regex = value[1:len(value) - 1]
value = ""
value = value.replace('*', '%')
if value in [None, ""] and regex in [None, ""]:
value = "%"
regex = ""
dbh = SpiderFootDb(self.config)
criteria = {
'scan_id': None if id == '' else id,
'type': None if eventType == '' else eventType,
'value': None if value == '' else value,
'regex': None if regex == '' else regex
}
data = dbh.search(criteria)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
escapeddata = cgi.escape(row[1])
escapedsrc = cgi.escape(row[2])
retdata.append([
lastseen, escapeddata, escapedsrc, row[3], row[5], row[6],
row[7], row[8], row[10], row[11], row[4], row[13], row[14]
])
return retdata
def scandelete(self, id, confirm=None, raw=False):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
if not raw:
return self.error("Scan ID not found.")
else:
return json.dumps(["ERROR", "Scan ID not found."])
if confirm is not None:
dbh.scanInstanceDelete(id)
if not raw:
raise cherrypy.HTTPRedirect("/")
else:
return json.dumps(["SUCCESS", ""])
else:
templ = Template(filename='dyn/scandelete.tmpl',
lookup=self.lookup)
return templ.render(id=id,
name=str(res[0]),
names=list(),
ids=list(),
pageid="SCANLIST",
docroot=self.docroot)
def scanelementtypediscovery(self, id, eventType):
"""scan element type discovery
Args:
id: TBD
eventType (str): filter by event type
"""
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
def scanhistory(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultHistory(id)
return json.dumps(data, ensure_ascii=False)
def startscan(self, scanname, scantarget, modulelist, typelist, usecase):
global globalScanStatus
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
types = dbh.eventTypes()
targetType = None
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "" and usecase == "":
return self.error("Form incomplete.")
# User selected modules
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
# User selected types
if len(modlist) == 0 and typelist != "":
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# User selected a use case
if len(modlist) == 0 and usecase != "":
for mod in self.config['__modules__']:
if usecase == 'all' or usecase in self.config['__modules__'][
mod]['cats']:
modlist.append(mod)
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
targetType = sf.targetType(scantarget)
if targetType is None:
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(scanId) is None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=scanId,
name=scanname,
docroot=self.docroot,
status=globalScanStatus.getStatus(scanId),
pageid="SCANLIST")
sfModules[modName]['opts'] = sfModules[modName]['object'].opts
if hasattr(sfModules[modName]['object'], 'optdescs'):
sfModules[modName]['optdescs'] = sfModules[modName][
'object'].optdescs
if len(list(sfModules.keys())) < 1:
print("No modules found in the modules directory.")
sys.exit(-1)
# Add module info to sfConfig so it can be used by the UI
sfConfig['__modules__'] = sfModules
# Add descriptions of the global config options
sfConfig['__globaloptdescs__'] = sfOptdescs
sf = SpiderFoot(sfConfig)
dbh = SpiderFootDb(sfConfig, init=True)
if not args.l:
if args.modules:
print("Modules available:")
for m in sorted(sfModules.keys()):
if "__" in m:
continue
print(('{0:25} {1}'.format(m, sfModules[m]['descr'])))
sys.exit(0)
if args.types:
print("Types available:")
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
